diff --git a/CHANGELOG.md b/CHANGELOG.md index 84121d9..ee065e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ ## Unreleased — 0.1.2 +- Added compatible non-destructive generation archival on desktop schema v22 + and Android schema v21. Only exactly acknowledged generations older than the + two active recovery tips leave admission after a verified durable payload + copy and audit record; all identities, manifests, ACKs, lineage and causal + evidence remain stored. Older Android stores can recover exact missing + producer-side ACK rows from the desktop consumer's durable ledger, with + strict run, peer, generation and manifest revalidation. A correlated Android + capacity error now reaches the Web without waiting for a transport timeout, + and the prepared-items tile can inspect every proposal rather than only its + first compact summary. + - Fixed a production full-generation timeout caused by recursively uploading retained staging and unrelated transport history on every MTP push. Each phase now publishes a private bounded outbox containing only its request, diff --git a/README.md b/README.md index 88fec83..c62c9a0 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,7 @@ not contain cloud credentials and does not run `rclone`. As of 2026-09-17: - `TRAINLOG_FORMAT_V1=PASS/FROZEN`; -- desktop SQLite schema v21 and Android SQLite schema v20; +- desktop SQLite schema v22 and Android SQLite schema v21; - Notcurses is the only active desktop terminal backend; - direct storage is `/storage/emulated/0/Documents/Trainlog` under Android's all-files access setting; diff --git a/android/app/src/main/java/com/labfytools/trainlog/data/AndroidBackupService.kt b/android/app/src/main/java/com/labfytools/trainlog/data/AndroidBackupService.kt index 33694c7..2fdab0b 100644 --- a/android/app/src/main/java/com/labfytools/trainlog/data/AndroidBackupService.kt +++ b/android/app/src/main/java/com/labfytools/trainlog/data/AndroidBackupService.kt @@ -219,7 +219,7 @@ internal class AndroidBackupService(private val context: Context) { manifest.getInt("version") == VERSION && manifest.getString("package") == context.packageName && manifest.getBoolean("plaintext") && - manifest.getInt("schema") in 17..20 + manifest.getInt("schema") in 17..21 ) { "Unsupported backup." } @@ -313,7 +313,7 @@ internal class AndroidBackupService(private val context: Context) { } ) require(db.rawQuery("PRAGMA foreign_key_check", null).use { !it.moveToFirst() }) - require(db.version in 17..20) { "Unsupported Android schema ${db.version}." } + require(db.version in 17..21) { "Unsupported Android schema ${db.version}." } return db.version } finally { db.close() diff --git a/android/app/src/main/java/com/labfytools/trainlog/data/SyncGenerationForegroundCoordinator.kt b/android/app/src/main/java/com/labfytools/trainlog/data/SyncGenerationForegroundCoordinator.kt index e49ebbc..061e747 100644 --- a/android/app/src/main/java/com/labfytools/trainlog/data/SyncGenerationForegroundCoordinator.kt +++ b/android/app/src/main/java/com/labfytools/trainlog/data/SyncGenerationForegroundCoordinator.kt @@ -14,8 +14,65 @@ internal sealed interface ForegroundGenerationResult { } /** Owns the bounded, user-initiated Android side of a generation conversation. */ -internal class SyncGenerationForegroundCoordinator(private val repository: TrainlogRepository) { - private val service = SyncGenerationService(repository) +internal class SyncGenerationForegroundCoordinator( + private val repository: TrainlogRepository, + archiveDirectorySync: ((File) -> Unit)? = null, +) { + private val service = + archiveDirectorySync?.let { SyncGenerationService(repository, it) } + ?: SyncGenerationService(repository) + + private fun requireExactKeys(value: JSONObject, expected: Set, label: String) { + val actual = value.keys().asSequence().toSet() + if (actual != expected) throw SyncGenerationException("invalid $label fields") + } + + private fun reconcileArchiveAcknowledgements( + raw: ByteArray, + runId: String, + androidPeerId: String, + desktopPeerId: String, + ) { + /* WHY: releases before archive-v1 advanced generation status without + * retaining the consumer's ACK locally. The desktop consumer may + * return its exact durable ACK so the Android producer can repair + * that evidence gap without fabricating or weakening proof. + * CONTRACT: the envelope is strictly correlated and contains at most + * 32 ACKs; acceptAcknowledgement revalidates every ACK against the + * immutable local manifest before any archive can become eligible. + * INVARIANT: a missing, rejected, or ambiguous ACK leaves its + * generation active and protected. */ + val envelope = JSONObject(raw.toString(Charsets.UTF_8)) + requireExactKeys( + envelope, + setOf( + "format", + "version", + "run_id", + "android_peer_id", + "desktop_peer_id", + "acknowledgements", + ), + "archive acknowledgement envelope", + ) + if ( + envelope.getString("format") != "trainlog-sync-archive-acknowledgements" || + envelope.getInt("version") != 1 || + envelope.getString("run_id") != runId || + envelope.getString("android_peer_id") != androidPeerId || + envelope.getString("desktop_peer_id") != desktopPeerId + ) { + throw SyncGenerationException("archive acknowledgement envelope is not correlated") + } + val acknowledgements = envelope.getJSONArray("acknowledgements") + if (acknowledgements.length() > 32) + throw SyncGenerationException("too many archive acknowledgements") + for (index in 0 until acknowledgements.length()) { + service.acceptAcknowledgement( + acknowledgements.getJSONObject(index).toString().toByteArray(Charsets.UTF_8) + ) + } + } private fun publish(path: File, value: String) { val temporary = File(path.parentFile, ".${path.name}.tmp-${UUID.randomUUID()}") @@ -81,15 +138,41 @@ internal class SyncGenerationForegroundCoordinator(private val repository: Train "causal-delete-v1", "mobile-history-v4", "execution-draft-v1", + "generation-archive-v1", ) ), ) .toString(), ) val deadline = System.nanoTime() + timeout.toNanos() - val request = JSONObject(await(File(directory, "request-v1.json"), deadline).readText()) + val requestRaw = + awaitCorrelated(File(directory, "request-v1.json"), deadline) { candidate -> + val runId = candidate.optString("run_id") + runId.startsWith("sy_") && + !repository.inSyncGenerationTransaction { db -> + db.rawQuery( + "SELECT 1 FROM sync_generations WHERE run_id=? LIMIT 1", + arrayOf(runId), + ).use { it.moveToFirst() } + } + } + val request = JSONObject(requestRaw.toString(Charsets.UTF_8)) if (request.getString("android_peer_id") != peer) throw SyncGenerationException("request targets another Android peer") + val archiveAckRaw = + awaitCorrelated( + File(directory, "desktop-archive-acknowledgements-v1.json"), + deadline, + ) { + it.optString("run_id") == request.getString("run_id") && + it.optString("android_peer_id") == peer + } + reconcileArchiveAcknowledgements( + archiveAckRaw, + request.getString("run_id"), + peer, + request.getString("desktop_peer_id"), + ) val captured = service.capture( directory, @@ -128,6 +211,25 @@ internal class SyncGenerationForegroundCoordinator(private val repository: Train publish(File(directory, "android-consumption-ack-v1.json"), acknowledgement) ForegroundGenerationResult.Completed(captured.runId) } catch (error: Exception) { + if (error.message == "generation retention capacity exhausted") { + try { + val request = JSONObject(File(directory, "request-v1.json").readText()) + publish( + File(directory, "android-generation-error-v1.json"), + JSONObject() + .put("format", "trainlog-sync-generation-error") + .put("version", 1) + .put("run_id", request.getString("run_id")) + .put("android_peer_id", service.peerId()) + .put("code", "peer_capacity_exhausted") + .put("action", "Archive acknowledged generations, then retry explicitly.") + .toString(), + ) + } catch (_: Exception) { + // Preserve the original local failure when error reporting + // cannot itself be durably published. + } + } ForegroundGenerationResult.Error(error.message ?: "generation synchronization failed") } } diff --git a/android/app/src/main/java/com/labfytools/trainlog/data/SyncGenerationService.kt b/android/app/src/main/java/com/labfytools/trainlog/data/SyncGenerationService.kt index 4ab7c27..019207a 100644 --- a/android/app/src/main/java/com/labfytools/trainlog/data/SyncGenerationService.kt +++ b/android/app/src/main/java/com/labfytools/trainlog/data/SyncGenerationService.kt @@ -1,7 +1,11 @@ package com.labfytools.trainlog.data +import android.system.Os +import android.system.OsConstants import java.io.File +import java.nio.ByteBuffer import java.nio.charset.StandardCharsets +import java.nio.file.Files import java.security.MessageDigest import java.time.OffsetDateTime import java.util.UUID @@ -23,7 +27,18 @@ internal data class CapturedSyncGeneration( * CONTRACT: artifacts are immutable, bounded and published before the manifest marker. INVARIANT: * causal operation payloads are never rewritten to attach generation identity. */ -internal class SyncGenerationService(private val repository: TrainlogRepository) { +internal class SyncGenerationService( + private val repository: TrainlogRepository, + private val syncArchiveDirectory: (File) -> Unit = { directory -> + val descriptor = + Os.open(directory.absolutePath, OsConstants.O_RDONLY or O_DIRECTORY, 0) + try { + Os.fsync(descriptor) + } finally { + Os.close(descriptor) + } + }, +) { companion object { const val MAX_MANIFEST_BYTES = 64 * 1024 const val MAX_ARTIFACTS = 32 @@ -33,6 +48,10 @@ internal class SyncGenerationService(private val repository: TrainlogRepository) const val MAX_PATH_BYTES = 240 const val MAX_DIAGNOSTIC_BYTES = 1024 const val MAX_RETAINED_PER_PEER = 8 + // Android's public OsConstants omits Linux O_DIRECTORY even though + // Os.open forwards the platform flag. Keep the fixed Linux ABI value + // local to the directory-fsync boundary. + private const val O_DIRECTORY = 0x10000 private val ID = Regex( "^(gen|peer|sy)_[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" @@ -160,6 +179,105 @@ internal class SyncGenerationService(private val repository: TrainlogRepository) * callers cannot supply or replace it. */ internal fun peerId(): String = stablePeerId("android") + private fun archiveDigest(directory: File): String { + val digest = MessageDigest.getInstance("SHA-256") + directory.walkTopDown().filter { it.isFile }.sortedBy { it.relativeTo(directory).path } + .forEach { file -> + if (java.nio.file.Files.isSymbolicLink(file.toPath())) + throw SyncGenerationException("archive contains symbolic link") + val name = file.relativeTo(directory).invariantSeparatorsPath.toByteArray() + digest.update(ByteBuffer.allocate(4).putInt(name.size).array()) + digest.update(name) + digest.update(MessageDigest.getInstance("SHA-256").digest(file.readBytes())) + } + return digest.digest().joinToString("") { "%02x".format(it) } + } + + internal fun archiveAcknowledged(ownedRoot: File, consumerPeerId: String): Int { + /* WHY: acknowledged payloads must not occupy active admission forever. + * CONTRACT: an exact consumed ACK is required and the newest two + * acknowledged lineage members remain active for recovery. + * INVARIANT: generation, artifact, ACK and causal rows remain intact; + * the ledger is committed only after a verified atomic archive copy. */ + val rows = repository.inSyncGenerationTransaction { db -> + db.rawQuery( + "SELECT g.generation_id,g.run_id,g.producer_peer_id,g.consumer_peer_id," + + "g.manifest_sha256,g.staging_path,g.manifest_json FROM sync_generations g " + + "WHERE g.consumer_peer_id=? AND g.status='acknowledged' AND NOT EXISTS(" + + "SELECT 1 FROM sync_generation_archives a WHERE a.generation_id=g.generation_id) " + + "ORDER BY g.generated_at DESC,g.generation_id DESC", + arrayOf(consumerPeerId), + ).use { cursor -> + buildList { + while (cursor.moveToNext()) add(List(7) { cursor.getString(it) }) + } + } + } + val root = File(File(ownedRoot, "archives"), "generations") + if (!root.exists() && !root.mkdirs()) + throw SyncGenerationException("cannot create generation archive") + var count = 0 + rows.drop(2).forEach { row -> + val eligible = repository.inSyncGenerationTransaction { db -> + db.rawQuery( + "SELECT ack_id FROM sync_acknowledgements WHERE generation_id=? AND run_id=? " + + "AND producer_peer_id=? AND consumer_peer_id=? AND manifest_sha256=? " + + "AND result='consumed' AND durability='sqlite-commit-full' LIMIT 2", + arrayOf(row[0], row[1], row[2], row[3], row[4]), + ).use { cursor -> + if (!cursor.moveToFirst()) null + else cursor.getString(0).also { + if (cursor.moveToNext()) throw SyncGenerationException("ambiguous ACK evidence") + } + } + } ?: return@forEach + val source = File(row[5]) + if (Files.isSymbolicLink(source.toPath())) + throw SyncGenerationException("generation staging is a symbolic link") + validatePublished(source, row[3]) + val destination = File(root, row[0]) + val temporary = File(root, ".${row[0]}.tmp") + if (Files.isSymbolicLink(destination.toPath())) + throw SyncGenerationException("generation archive is a symbolic link") + if (!destination.exists()) { + if (Files.isSymbolicLink(temporary.toPath())) + throw SyncGenerationException("temporary generation archive is a symbolic link") + if (temporary.exists() && !temporary.deleteRecursively()) + throw SyncGenerationException("cannot reset interrupted generation archive") + if (!source.copyRecursively(temporary, overwrite = false)) + throw SyncGenerationException("cannot copy generation archive") + validatePublished(temporary, row[3]) + temporary.walkTopDown().filter { it.isFile }.forEach { file -> + java.io.FileOutputStream(file, true).use { it.fd.sync() } + } + if (!temporary.renameTo(destination)) + throw SyncGenerationException("cannot commit generation archive") + syncArchiveDirectory(root) + } + validatePublished(destination, row[3]) + val checksum = archiveDigest(destination) + val audit = + canonical( + JSONObject() + .put("format", "trainlog-sync-generation-archive-audit") + .put("version", 1) + .put("generation_id", row[0]) + .put("run_id", row[1]) + .put("manifest_sha256", row[4]) + .put("ack_id", eligible) + .put("archive_sha256", checksum), + ) + repository.inSyncGenerationTransaction { db -> + db.execSQL( + "INSERT OR IGNORE INTO sync_generation_archives VALUES(?,?,?,?,?,?)", + arrayOf(row[0], destination.absolutePath, row[4], checksum, OffsetDateTime.now().toString(), audit), + ) + } + count += 1 + } + return count + } + fun capture( ownedRoot: File, consumerPeerId: String, @@ -169,10 +287,11 @@ internal class SyncGenerationService(private val repository: TrainlogRepository) requireId(consumerPeerId, "peer") requireId(runId, "sy") requireId(generationId, "gen") + archiveAcknowledged(ownedRoot, consumerPeerId) repository.inSyncGenerationTransaction { db -> val retained = db.rawQuery( - "SELECT COUNT(*) FROM sync_generations WHERE consumer_peer_id=? AND status IN('captured','published','waiting_acknowledgement','acknowledged')", + "SELECT COUNT(*) FROM sync_generations g WHERE consumer_peer_id=? AND status IN('captured','published','waiting_acknowledgement','acknowledged') AND NOT EXISTS(SELECT 1 FROM sync_generation_archives a WHERE a.generation_id=g.generation_id)", arrayOf(consumerPeerId), ) .use { @@ -869,6 +988,32 @@ internal class SyncGenerationService(private val repository: TrainlogRepository) throw SyncGenerationException("ACK does not match pending generation") val target = if (ack.getString("result") == "consumed") "acknowledged" else "rejected" + /* CONTRACT: the correlated ACK is permanent causal + * evidence and is committed atomically with generation + * state. Replay is byte-identical or rejected below. */ + db.execSQL( + "INSERT OR IGNORE INTO sync_acknowledgements VALUES(?,?,?,?,?,?,?,?,?,?,?)", + arrayOf( + ack.getString("ack_id"), + ack.getString("generation_id"), + ack.getString("run_id"), + ack.getString("producer_peer_id"), + ack.getString("consumer_peer_id"), + ack.getString("manifest_sha256"), + ack.getString("result"), + ack.getString("durability"), + ack.getString("consumed_at"), + ack.getString("diagnostic"), + claimed, + ), + ) + db.rawQuery( + "SELECT payload_sha256 FROM sync_acknowledgements WHERE ack_id=?", + arrayOf(ack.getString("ack_id")), + ).use { stored -> + if (!stored.moveToFirst() || stored.getString(0) != claimed) + throw SyncGenerationException("conflicting ACK replay") + } if (row.getString(4) in setOf("acknowledged", "rejected")) { if (row.getString(4) != target) throw SyncGenerationException("conflicting ACK replay") diff --git a/android/app/src/main/java/com/labfytools/trainlog/data/TrainlogRepository.kt b/android/app/src/main/java/com/labfytools/trainlog/data/TrainlogRepository.kt index 37ce918..0f5ddc3 100644 --- a/android/app/src/main/java/com/labfytools/trainlog/data/TrainlogRepository.kt +++ b/android/app/src/main/java/com/labfytools/trainlog/data/TrainlogRepository.kt @@ -7934,7 +7934,7 @@ private class TrainlogDatabaseHelper( appContext, databaseName, null, - 20, + 21, ) { override fun onConfigure( db: SQLiteDatabase, @@ -7979,6 +7979,7 @@ private class TrainlogDatabaseHelper( createSyncDataLifecycleTables(db) createCausalDeleteTables(db) createSyncGenerationTables(db) + createSyncGenerationArchiveTable(db) seedEquipment(db) } @@ -8146,6 +8147,14 @@ private class TrainlogDatabaseHelper( createSyncGenerationTables(db) version = 20 } + if (version < 21 && newVersion >= 21) { + /* WHY: acknowledged payloads must leave bounded active admission + * without losing immutable identity, lineage or ACK evidence. + * CONTRACT: v21 adds only an append-only verified archive ledger. + * INVARIANT: migration archives nothing and invents no ACK. */ + createSyncGenerationArchiveTable(db) + version = 21 + } if (version != newVersion) { error( @@ -8244,6 +8253,15 @@ private class TrainlogDatabaseHelper( db.execSQL("CREATE TABLE IF NOT EXISTS sync_causal_publications(operation_id TEXT NOT NULL REFERENCES sync_causal_operations(operation_id),generation_id TEXT NOT NULL,first_emission INTEGER NOT NULL CHECK(first_emission IN(0,1)),PRIMARY KEY(operation_id,generation_id));") } + private fun createSyncGenerationArchiveTable(db: SQLiteDatabase) { + db.execSQL( + "CREATE TABLE IF NOT EXISTS sync_generation_archives(" + + "generation_id TEXT PRIMARY KEY REFERENCES sync_generations(generation_id)," + + "archive_path TEXT NOT NULL UNIQUE,manifest_sha256 TEXT NOT NULL," + + "archive_sha256 TEXT NOT NULL,archived_at TEXT NOT NULL,audit_json TEXT NOT NULL);", + ) + } + private fun createAiSessionDraftTables(db: SQLiteDatabase) { db.execSQL( "CREATE TABLE IF NOT EXISTS ai_session_drafts(" + diff --git a/android/app/src/test/java/com/labfytools/trainlog/data/AndroidV10PlanningMigrationTest.kt b/android/app/src/test/java/com/labfytools/trainlog/data/AndroidV10PlanningMigrationTest.kt index 0761881..dc29876 100644 --- a/android/app/src/test/java/com/labfytools/trainlog/data/AndroidV10PlanningMigrationTest.kt +++ b/android/app/src/test/java/com/labfytools/trainlog/data/AndroidV10PlanningMigrationTest.kt @@ -66,7 +66,7 @@ class AndroidV10PlanningMigrationTest { TrainlogRepository(context, name).useForTest { it.listSessions() } SQLiteDatabase.openDatabase(path.path, null, SQLiteDatabase.OPEN_READONLY).use { db -> - assertEquals(20, scalar(db, "PRAGMA user_version")) + assertEquals(21, scalar(db, "PRAGMA user_version")) for (table in listOf("session_exercises", "draft_session_exercises")) { assertEquals(6, scalar(db, "SELECT COUNT(*) FROM pragma_table_info('$table') WHERE name IN ('load_mode','rest_seconds','target_sets','target_reps','target_duration_seconds','target_weight_kg')")) assertEquals(0, scalar(db, "SELECT COUNT(*) FROM $table WHERE load_mode<>'none' OR rest_seconds<>0 OR target_sets IS NOT NULL OR target_reps IS NOT NULL OR target_duration_seconds IS NOT NULL OR target_weight_kg IS NOT NULL")) diff --git a/android/app/src/test/java/com/labfytools/trainlog/data/AndroidV11ExerciseAliasMigrationTest.kt b/android/app/src/test/java/com/labfytools/trainlog/data/AndroidV11ExerciseAliasMigrationTest.kt index 39adcfd..fc8d091 100644 --- a/android/app/src/test/java/com/labfytools/trainlog/data/AndroidV11ExerciseAliasMigrationTest.kt +++ b/android/app/src/test/java/com/labfytools/trainlog/data/AndroidV11ExerciseAliasMigrationTest.kt @@ -69,7 +69,7 @@ class AndroidV11ExerciseAliasMigrationTest { migrated.close() } SQLiteDatabase.openDatabase(path, null, SQLiteDatabase.OPEN_READONLY).use { db -> - assertEquals(20, scalar(db, "PRAGMA user_version")) + assertEquals(21, scalar(db, "PRAGMA user_version")) /* v13 intentionally adds the frozen machine identities and * exercise-owned metadata; all non-exercise facts stay exact. */ val after = snapshot(db).filterKeys { it !in setOf("exercise_aliases", "exercise_profile_state", "exercise_profile_revisions") } diff --git a/android/app/src/test/java/com/labfytools/trainlog/data/AndroidV16AiSessionDraftMigrationTest.kt b/android/app/src/test/java/com/labfytools/trainlog/data/AndroidV16AiSessionDraftMigrationTest.kt index 549c504..cfb5aa2 100644 --- a/android/app/src/test/java/com/labfytools/trainlog/data/AndroidV16AiSessionDraftMigrationTest.kt +++ b/android/app/src/test/java/com/labfytools/trainlog/data/AndroidV16AiSessionDraftMigrationTest.kt @@ -78,7 +78,7 @@ class AndroidV16AiSessionDraftMigrationTest { migrated.close() } SQLiteDatabase.openDatabase(path, null, SQLiteDatabase.OPEN_READONLY).use { db -> - assertEquals(20, scalar(db, "PRAGMA user_version")) + assertEquals(21, scalar(db, "PRAGMA user_version")) assertEquals(before, listOf( scalar(db, "SELECT COUNT(*) FROM sessions"), scalar(db, "SELECT COUNT(*) FROM performed_sets"), diff --git a/android/app/src/test/java/com/labfytools/trainlog/data/BodyZonesTest.kt b/android/app/src/test/java/com/labfytools/trainlog/data/BodyZonesTest.kt index c710d14..85c34e0 100644 --- a/android/app/src/test/java/com/labfytools/trainlog/data/BodyZonesTest.kt +++ b/android/app/src/test/java/com/labfytools/trainlog/data/BodyZonesTest.kt @@ -328,7 +328,7 @@ class BodyZonesTest { SQLiteDatabase.openDatabase(context.getDatabasePath(name).path, null, SQLiteDatabase.OPEN_READONLY).use { db -> db.rawQuery("PRAGMA user_version", null).use { cursor -> - assertTrue(cursor.moveToFirst()); assertEquals(20, cursor.getInt(0)) + assertTrue(cursor.moveToFirst()); assertEquals(21, cursor.getInt(0)) } db.rawQuery("SELECT id FROM exercises", null).use { cursor -> assertTrue(cursor.moveToFirst()); assertEquals(42L, cursor.getLong(0)) diff --git a/android/app/src/test/java/com/labfytools/trainlog/data/BridgeBackupMigrationTest.kt b/android/app/src/test/java/com/labfytools/trainlog/data/BridgeBackupMigrationTest.kt index b9f647e..c121ab3 100644 --- a/android/app/src/test/java/com/labfytools/trainlog/data/BridgeBackupMigrationTest.kt +++ b/android/app/src/test/java/com/labfytools/trainlog/data/BridgeBackupMigrationTest.kt @@ -43,7 +43,7 @@ class BridgeBackupMigrationTest { null, android.database.sqlite.SQLiteDatabase.OPEN_READONLY, ) - .use { assertEquals(20, it.version) } + .use { assertEquals(21, it.version) } } finally { repository.close() context.deleteDatabase(name) diff --git a/android/app/src/test/java/com/labfytools/trainlog/data/SyncDeploymentConversationTest.kt b/android/app/src/test/java/com/labfytools/trainlog/data/SyncDeploymentConversationTest.kt index 9d33d8b..031fade 100644 --- a/android/app/src/test/java/com/labfytools/trainlog/data/SyncDeploymentConversationTest.kt +++ b/android/app/src/test/java/com/labfytools/trainlog/data/SyncDeploymentConversationTest.kt @@ -1,6 +1,7 @@ package com.labfytools.trainlog.data import android.content.Context +import android.database.sqlite.SQLiteDatabase import androidx.test.core.app.ApplicationProvider import com.labfytools.trainlog.model.ActiveSessionDraft import com.labfytools.trainlog.model.NewExerciseProfile @@ -86,18 +87,9 @@ class SyncDeploymentConversationTest { ActiveSessionDraft(exercises = listOf(occurrence)) ), ) - val coordinator = SyncGenerationForegroundCoordinator(repository) val release = System.getenv("TRAINLOG_SYNC_TEST_RELEASE_FILE") - val executor = java.util.concurrent.Executors.newSingleThreadExecutor() - val result = - executor.submit { - coordinator.run(transport, Duration.ofSeconds(30)) { - release?.let { awaitFile(File(it)) } - } - } - val peer = JSONObject(awaitFile(File(transport, "android-peer-v1.json")).readText()) + val desktop = File(transport, "desktop.sqlite") if (suppliedTransport == null) { - val desktop = File(transport, "desktop.sqlite") val fixture = ProcessBuilder( File(repositoryRoot, "build/tui/sync-generation-fixture").absolutePath, @@ -106,7 +98,24 @@ class SyncDeploymentConversationTest { .directory(repositoryRoot) .start() assertEquals(0, fixture.waitFor()) - localWorker = + } + val conversations = if (suppliedTransport == null) 24 else 1 + repeat(conversations) { + // Robolectric cannot open a directory file descriptor. The + // production default performs the real directory fsync; this + // test substitutes only that kernel primitive while retaining + // the production copy, validation, rename and ledger path. + val coordinator = SyncGenerationForegroundCoordinator(repository) {} + val executor = java.util.concurrent.Executors.newSingleThreadExecutor() + val result = + executor.submit { + coordinator.run(transport, Duration.ofSeconds(30)) { + release?.let { awaitFile(File(it)) } + } + } + val peer = JSONObject(awaitFile(File(transport, "android-peer-v1.json")).readText()) + if (suppliedTransport == null) { + localWorker = ProcessBuilder( "python3", File(repositoryRoot, "tools/sync_peer_worker.py").absolutePath, @@ -126,19 +135,56 @@ class SyncDeploymentConversationTest { .directory(repositoryRoot) .redirectErrorStream(true) .start() + } + val completed = result.get(35, java.util.concurrent.TimeUnit.SECONDS) + executor.shutdownNow() + assertTrue( + "conversation ${it + 1}: $completed", + completed is ForegroundGenerationResult.Completed, + ) + var workerOutput = "" + localWorker?.let { worker -> + workerOutput = worker.inputStream.bufferedReader().readText() + assertEquals("worker failed:\n$workerOutput", 0, worker.waitFor()) + assertTrue(workerOutput.contains("\"result\":\"completed\"")) + } + if (suppliedTransport == null && it == 7) { + // Reproduce the deployed pre-archive state: those builds + // advanced producer status but did not retain received + // ACK rows. The next real worker conversation must return + // the desktop consumer's exact evidence and unblock the + // full active quota without deleting any generation. + repository.inSyncGenerationTransaction { database -> + database.delete("sync_acknowledgements", null, null) + database.rawQuery( + "SELECT COUNT(*) FROM sync_generations WHERE status='acknowledged'", + null, + ).use { cursor -> + assertTrue(cursor.moveToFirst()) + assertEquals(8, cursor.getInt(0)) + } + } + } + localWorker = null } - assertTrue( - result.get(35, java.util.concurrent.TimeUnit.SECONDS) - is ForegroundGenerationResult.Completed - ) - executor.shutdownNow() repository.close() repository = TrainlogRepository(context, databaseName) assertTrue(repository.loadActiveSessionDraft() is ActiveDraftLoadResult.Loaded) - localWorker?.let { worker -> - val output = worker.inputStream.bufferedReader().readText() - assertEquals("worker failed:\n$output", 0, worker.waitFor()) - assertTrue(output.contains("\"result\":\"completed\"")) + if (suppliedTransport == null) { + SQLiteDatabase.openDatabase( + context.getDatabasePath(databaseName).absolutePath, + null, + SQLiteDatabase.OPEN_READONLY, + ).use { database -> + database.rawQuery("SELECT COUNT(*) FROM sync_generation_archives", null).use { + assertTrue(it.moveToFirst()) + assertTrue(it.getInt(0) >= 16) + } + database.rawQuery("SELECT COUNT(*) FROM sync_generations", null).use { + assertTrue(it.moveToFirst()) + assertEquals(24, it.getInt(0)) + } + } } } finally { localWorker?.destroyForcibly() diff --git a/android/app/src/test/java/com/labfytools/trainlog/data/SyncGenerationServiceTest.kt b/android/app/src/test/java/com/labfytools/trainlog/data/SyncGenerationServiceTest.kt index cdb9cf1..7d8592b 100644 --- a/android/app/src/test/java/com/labfytools/trainlog/data/SyncGenerationServiceTest.kt +++ b/android/app/src/test/java/com/labfytools/trainlog/data/SyncGenerationServiceTest.kt @@ -46,6 +46,7 @@ class SyncGenerationServiceTest { SQLiteDatabase.openDatabase(path.absolutePath, null, SQLiteDatabase.OPEN_READWRITE).use { db -> listOf( + "sync_generation_archives", "sync_causal_publications", "sync_acknowledgements", "sync_consumed_generations", @@ -62,7 +63,7 @@ class SyncGenerationServiceTest { .use { db -> db.rawQuery("PRAGMA user_version", null).use { assertTrue(it.moveToFirst()) - assertEquals(20, it.getInt(0)) + assertEquals(21, it.getInt(0)) } db.rawQuery("SELECT COUNT(*) FROM exercises", null).use { assertTrue(it.moveToFirst()) diff --git a/android/app/src/test/java/com/labfytools/trainlog/data/TrainlogRepositoryDraftTest.kt b/android/app/src/test/java/com/labfytools/trainlog/data/TrainlogRepositoryDraftTest.kt index 3ebeb2d..18d0565 100644 --- a/android/app/src/test/java/com/labfytools/trainlog/data/TrainlogRepositoryDraftTest.kt +++ b/android/app/src/test/java/com/labfytools/trainlog/data/TrainlogRepositoryDraftTest.kt @@ -2260,7 +2260,7 @@ class TrainlogRepositoryDraftTest { ).use { db -> db.rawQuery("PRAGMA user_version;", null).use { cursor -> assertTrue(cursor.moveToFirst()) - assertEquals(20, cursor.getInt(0)) + assertEquals(21, cursor.getInt(0)) } db.rawQuery( "SELECT eq.equipment_id, ps.reps, ps.weight_kg FROM session_exercises se " + @@ -2387,7 +2387,7 @@ class TrainlogRepositoryDraftTest { ).use { db -> db.rawQuery("PRAGMA user_version;", null).use { cursor -> assertTrue(cursor.moveToFirst()) - assertEquals(20, cursor.getInt(0)) + assertEquals(21, cursor.getInt(0)) } db.rawQuery("SELECT weight_kg FROM performed_sets WHERE id = 1;", null).use { cursor -> assertTrue(cursor.moveToFirst()) diff --git a/docs/android.md b/docs/android.md index ef74fa7..d9955e7 100644 --- a/docs/android.md +++ b/docs/android.md @@ -961,6 +961,8 @@ The authorized private rollout used the bridge only to add the backup surface without clearing package data, verified the resulting plaintext archive, then installed the current signed release variant through a strictly increasing version-code chain. The final installed private candidate retains product -version 0.1.2 and schema v20. Its full-generation foreground coordinator waits -for correlated objects, tolerates stale objects left by interrupted runs, and +version 0.1.2 and schema v21. Schema v21 adds the verified generation archive +ledger without changing any domain identity. Its full-generation foreground +coordinator waits for correlated objects, tolerates stale objects left by +interrupted runs, strictly revalidates desktop-retained historical ACKs, and shows completion only after both durable peer acknowledgements. diff --git a/docs/current_state.md b/docs/current_state.md index 5a6c21e..e120df8 100644 --- a/docs/current_state.md +++ b/docs/current_state.md @@ -50,8 +50,8 @@ parallel implementation of its rules. | Boundary | Current state | |---|---| | Frozen project exchange | `TRAINLOG_FORMAT_V1=PASS/FROZEN` | -| Desktop SQLite | schema v21 | -| Android SQLite | schema v20 | +| Desktop SQLite | schema v22 | +| Android SQLite | schema v21 | | Mobile snapshot | V3 active; V1/V2 readable legacy inputs; explicit V4 codec staged, not selected by transport | | Desktop terminal backend | Notcurses only | | Trainlog product version | `0.1.2` development, synchronized across Android and desktop; latest stable release: `v0.1.1` | @@ -359,7 +359,7 @@ The focused causal closeout additionally proves child-data delete/update conflicts including change-and-revert, imported built-in and alias protection, active/pending/desktop draft-revision equivalence, protected companion refusal, current-view retirement and bounded pre-effect admission. The later generation -slice additively advances schemas to desktop v21/Android v20 without changing +slice additively advances schemas to desktop v22/Android v21 without changing active V3 transport selection. The residual causal evidence also proves legitimate Android and desktop diff --git a/docs/database.md b/docs/database.md index 95f30b1..9d2a831 100644 --- a/docs/database.md +++ b/docs/database.md @@ -532,6 +532,12 @@ foreign-key checks. Reconciliation validation also uses coherent Android and desktop v9 copies before applying migrations to the real stores. SQLite files are never synchronization artifacts. +Desktop schema v22 adds `sync_generation_archives` as an append-only archive +ledger. A row retains the generation identity, verified manifest digest, +deterministic archive digest, durable location, timestamp and canonical audit +record. Generation, artifact, ACK, consumption and causal-publication rows are +not deleted or rewritten when a payload becomes archived. + The schemas deliberately differ where ownership differs. Desktop sessions own `ended_at`, notes, planned targets and session-specific load semantics; the Android capture schema does not. Desktop body observations may link to a diff --git a/docs/roadmap.md b/docs/roadmap.md index 4b81c7b..55f826d 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -24,7 +24,7 @@ distinction between measured values and estimates. ## Current baseline -Desktop schema v21, Android schema v20, Notcurses, direct +Desktop schema v22, Android schema v21, Notcurses, direct `Documents/Trainlog` storage, mobile export V3, Training Knowledge V1, Body Zones V1, Training Feedback V1/V2, STATS V1, and Session Generator V1 are implemented. Session Generator V1 is hidden pending V2. AI session-draft diff --git a/docs/sync_exchange.md b/docs/sync_exchange.md index df66af1..5c17565 100644 --- a/docs/sync_exchange.md +++ b/docs/sync_exchange.md @@ -18,7 +18,7 @@ available because availability is independently guarded by causal state. ## Machine-exercise Phase 1 compatibility -Desktop schema v21 and Android schema v20 retain mobile export V3, readable V1/V2 imports, +Desktop schema v22 and Android schema v21 retain mobile export V3, readable V1/V2 imports, equipment definitions V1, equipment associations V2, exercise aliases V1, and the BODY ZONES companion without wire-format changes. Machine metadata is not silently added to a frozen artifact: stable exercise IDs and canonical names @@ -756,8 +756,32 @@ transaction while calling the existing domain exporters. Both validate domain and outer limits before recording a captured generation. Publication copies verified immutable bytes into `generations//`, resumes only byte-identical partial publication, and writes `manifest.json` last. Capacity -admits at most eight retained outgoing generations per consumer; exhaustion is -recoverable and never evicts an unacknowledged generation or causal proof. +admits at most eight active outgoing generations per consumer. Acknowledged +generations older than the two newest lineage members leave active admission +only after their exact consumed ACK, manifest, artifact sizes and digests have +been reconciled and a complete atomic archive copy has been verified and +journaled. Archive rows preserve identity, lineage, late-ACK and replay +protection; pending, rejected, ambiguous and recovery-tip generations remain +active. Archive directories are outside bounded MTP polling and are never +uploaded recursively. Failure or interruption leaves admission closed rather +than fabricating an ACK or discarding evidence. + +Before Android captures for a negotiated `generation-archive-v1` conversation, +the desktop consumer publishes +`desktop-archive-acknowledgements-v1.json`. Its strictly correlated envelope +contains at most 32 exact consumed ACK documents already retained in the +desktop consumption ledger. This repairs the evidence gap left by older +Android producers that advanced generation status without retaining the +received ACK. Android revalidates every document against its immutable local +generation and manifest before storing it; missing, mismatched, rejected, or +ambiguous evidence cannot make a generation archivable. The object is a +bounded coordination artifact, not a new ACK source. + +If Android still cannot admit a generation, it durably publishes the +run-correlated `android-generation-error-v1.json` with +`peer_capacity_exhausted`. The desktop worker and Web surface preserve that +cause and its archive/retry action instead of converting it to a later +transport timeout. The required artifact order is causal deletions, catalog/profile identity, custom-equipment definitions, V4 history, aliases and occurrence equipment, diff --git a/docs/tests.md b/docs/tests.md index b14c026..26f49ea 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -1323,11 +1323,23 @@ Android generation service, desktop consumers and durable ACK path remain real. backup including an active draft, raw partial input, stable peer identity and preferences, and rejects corrupt and traversal entries. The bridge build runs the analogous test on the exact schema-v17 source and feeds its archive to -`BridgeBackupMigrationTest`, which verifies normal migration to schema v20. +`BridgeBackupMigrationTest`, which verifies normal migration to schema v21. These are software proofs. They do not claim physical USB/MTP behavior, installation over the signed release, a real user backup, or a real-device restore. +The production Android coordinator and desktop peer worker also complete 24 +successive bidirectional conversations against their real SQLite services with +no cleanup between runs. The test crosses multiple archive cycles and checks +that the active draft survives. After the eighth exchange it also reproduces +the deployed legacy evidence gap by retaining acknowledged generations while +removing only their producer-side ACK rows; the next production conversation +must restore the desktop consumer's exact correlated ACKs, archive eligible +generations, and complete without deleting or duplicating any generation. +Focused archive tests cover interrupted-copy resume, invalid archives, +simulated insufficient space, missing and late ACKs, and idempotent ACK replay +after archival. + ## Readability validation The normalized C scope is checked non-mutatively with the explicit diff --git a/tests/test_generation_archival.py b/tests/test_generation_archival.py new file mode 100644 index 0000000..1d5a138 --- /dev/null +++ b/tests/test_generation_archival.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Failure-boundary tests for the production generation archive ledger.""" + +import sqlite3 +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) +import sync_generation_exchange as generation # noqa: E402 + + +FIXTURE = ROOT / "build" / "tui" / "sync-generation-fixture" +PEER = "peer_11111111-1111-4111-8111-111111111111" + + +class GenerationArchiveTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="trainlog-archive-test-") + self.root = Path(self.temporary.name) + self.database = self.root / "desktop.sqlite" + subprocess.run([str(FIXTURE), str(self.database)], check=True) + self.owned = self.root / "owned" + self.objects = self.root / "objects" + + def tearDown(self): + self.temporary.cleanup() + + def acknowledge(self, generation_id: str, manifest: dict, digest: str) -> Path: + generation.publish(self.database, generation_id, self.objects) + with sqlite3.connect(self.database) as database: + producer = database.execute( + "SELECT producer_peer_id FROM sync_generations WHERE generation_id=?", + (generation_id,), + ).fetchone()[0] + ack = generation.ack_document( + manifest["run_id"], generation_id, producer, PEER, digest, + "consumed", generation.now(), "", + ) + path = self.root / f"{generation_id}.ack.json" + path.write_bytes(generation.canonical(ack)) + self.assertEqual("acknowledged", generation.accept_ack(self.database, path)) + return path + + def create_acknowledged(self): + values = [] + for _ in range(3): + stage, manifest, digest = generation.capture_desktop( + self.database, self.owned, PEER + ) + del stage + ack = self.acknowledge(manifest["generation_id"], manifest, digest) + values.append((manifest, ack)) + return values + + def test_interrupted_archive_resumes_and_replay_stays_idempotent(self): + values = self.create_acknowledged() + oldest = values[0][0]["generation_id"] + temporary = self.owned / "archives" / "generations" / f".{oldest}.tmp" + temporary.mkdir(parents=True) + (temporary / "interrupted").write_text("partial") + self.assertEqual(1, generation.archive_acknowledged(self.database, self.owned, PEER)) + self.assertFalse(temporary.exists()) + self.assertEqual("unchanged", generation.accept_ack(self.database, values[0][1])) + with sqlite3.connect(self.database) as database: + self.assertEqual( + 1, + database.execute( + "SELECT COUNT(*) FROM sync_generation_archives WHERE generation_id=?", + (oldest,), + ).fetchone()[0], + ) + + def test_invalid_archive_and_no_space_never_commit_ledger(self): + values = self.create_acknowledged() + oldest = values[0][0]["generation_id"] + destination = self.owned / "archives" / "generations" / oldest + destination.mkdir(parents=True) + (destination / "manifest.json").write_text("{}") + with self.assertRaises(generation.GenerationError): + generation.archive_acknowledged(self.database, self.owned, PEER) + with sqlite3.connect(self.database) as database: + self.assertEqual(0, database.execute("SELECT COUNT(*) FROM sync_generation_archives").fetchone()[0]) + destination.unlink(missing_ok=True) if destination.is_file() else None + for child in destination.iterdir(): + child.unlink() + destination.rmdir() + with mock.patch.object(generation.shutil, "copytree", side_effect=OSError(28, "no space")): + with self.assertRaises(OSError): + generation.archive_acknowledged(self.database, self.owned, PEER) + with sqlite3.connect(self.database) as database: + self.assertEqual(0, database.execute("SELECT COUNT(*) FROM sync_generation_archives").fetchone()[0]) + + def test_missing_then_late_ack_is_protected_until_correlated(self): + values = self.create_acknowledged() + stage, manifest, digest = generation.capture_desktop(self.database, self.owned, PEER) + del stage + generation.publish(self.database, manifest["generation_id"], self.objects) + generation.archive_acknowledged(self.database, self.owned, PEER) + with sqlite3.connect(self.database) as database: + self.assertEqual( + "waiting_acknowledgement", + database.execute( + "SELECT status FROM sync_generations WHERE generation_id=?", + (manifest["generation_id"],), + ).fetchone()[0], + ) + self.assertEqual( + 0, + database.execute( + "SELECT COUNT(*) FROM sync_generation_archives WHERE generation_id=?", + (manifest["generation_id"],), + ).fetchone()[0], + ) + self.acknowledge(manifest["generation_id"], manifest, digest) + self.assertGreaterEqual(generation.archive_acknowledged(self.database, self.owned, PEER), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sync_generation_exchange.py b/tests/test_sync_generation_exchange.py index 509910d..2caa50f 100644 --- a/tests/test_sync_generation_exchange.py +++ b/tests/test_sync_generation_exchange.py @@ -21,8 +21,9 @@ CREATE TABLE sync_generations(generation_id TEXT PRIMARY KEY,run_id TEXT,produce CREATE TABLE sync_generation_artifacts(generation_id TEXT,logical_name TEXT,format TEXT,version INTEGER,filename TEXT,size_bytes INTEGER,sha256 TEXT,required INTEGER,PRIMARY KEY(generation_id,logical_name),UNIQUE(generation_id,filename)); CREATE TABLE sync_consumed_generations(generation_id TEXT PRIMARY KEY,run_id TEXT,producer_peer_id TEXT,consumer_peer_id TEXT,parent_generation_id TEXT,manifest_sha256 TEXT,consumed_at TEXT,result TEXT,durability TEXT,diagnostic TEXT,ack_json TEXT,UNIQUE(producer_peer_id,generation_id)); CREATE TABLE sync_acknowledgements(ack_id TEXT PRIMARY KEY,generation_id TEXT,run_id TEXT,producer_peer_id TEXT,consumer_peer_id TEXT,manifest_sha256 TEXT,result TEXT,durability TEXT,created_at TEXT,diagnostic TEXT,payload_sha256 TEXT); +CREATE TABLE sync_generation_archives(generation_id TEXT PRIMARY KEY,archive_path TEXT UNIQUE,manifest_sha256 TEXT,archive_sha256 TEXT,archived_at TEXT,audit_json TEXT); CREATE TABLE sync_causal_publications(operation_id TEXT,generation_id TEXT,first_emission INTEGER,PRIMARY KEY(operation_id,generation_id)); -PRAGMA user_version=21; +PRAGMA user_version=22; """ class GenerationTest(unittest.TestCase): diff --git a/tools/causal_delete_exchange.py b/tools/causal_delete_exchange.py index 3769026..ac817fa 100644 --- a/tools/causal_delete_exchange.py +++ b/tools/causal_delete_exchange.py @@ -38,8 +38,8 @@ def digest(value) -> str: def require_schema(db: sqlite3.Connection) -> None: version = db.execute("PRAGMA user_version").fetchone()[0] - if version not in (20, 21): - raise CausalError(f"desktop schema v20/v21 required, found v{version}") + if version not in (20, 21, 22): + raise CausalError(f"desktop schema v20-v22 required, found v{version}") def target_snapshot(db: sqlite3.Connection, kind: str, target: str): diff --git a/tools/execution_draft_exchange.py b/tools/execution_draft_exchange.py index 5b50ff0..497d7a3 100644 --- a/tools/execution_draft_exchange.py +++ b/tools/execution_draft_exchange.py @@ -182,8 +182,8 @@ def causal_payload(draft): def require_schema(connection): version = connection.execute("PRAGMA user_version").fetchone()[0] - if version not in (19, 20, 21): - raise LifecycleError(f"desktop schema v19/v20/v21 required, found v{version}") + if version not in (19, 20, 21, 22): + raise LifecycleError(f"desktop schema v19-v22 required, found v{version}") def import_document(connection, document, own_transaction=True): diff --git a/tools/export_ai_history.py b/tools/export_ai_history.py index a241735..4f82b37 100644 --- a/tools/export_ai_history.py +++ b/tools/export_ai_history.py @@ -12,7 +12,7 @@ from pathlib import Path from trainlog_sqlite import configure_connection -SUPPORTED_SCHEMA_VERSIONS = (15, 16, 17, 18, 19, 20, 21) +SUPPORTED_SCHEMA_VERSIONS = (15, 16, 17, 18, 19, 20, 21, 22) MEASUREMENT_FIELDS = ( "body_weight_kg", "neck_cm", "shoulders_cm", "chest_cm", "waist_cm", "hips_cm", "left_arm_cm", "right_arm_cm", "left_forearm_cm", diff --git a/tools/export_ai_session_drafts.py b/tools/export_ai_session_drafts.py index 267b5fa..04a8157 100644 --- a/tools/export_ai_session_drafts.py +++ b/tools/export_ai_session_drafts.py @@ -20,8 +20,8 @@ ID_PREFIX = "aid_" def ensure_publication_state(connection): """Install the additive v18 cursor for direct syncd/tool invocations.""" owned_transaction = not connection.in_transaction - if connection.execute("PRAGMA user_version").fetchone()[0] not in (18, 19, 20, 21): - raise ValueError("schema desktop v18-v21 requis") + if connection.execute("PRAGMA user_version").fetchone()[0] not in (18, 19, 20, 21, 22): + raise ValueError("schema desktop v18-v22 requis") columns = {row[1] for row in connection.execute( "PRAGMA table_info(ai_session_drafts)")} if "published_at" not in columns: diff --git a/tools/export_equipment_associations.py b/tools/export_equipment_associations.py index f559deb..ac8d2b4 100644 --- a/tools/export_equipment_associations.py +++ b/tools/export_equipment_associations.py @@ -28,7 +28,7 @@ def main(): args = parser.parse_args() connection = connect_database(args.database) try: - if connection.execute("PRAGMA user_version;").fetchone()[0] not in (8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21): + if connection.execute("PRAGMA user_version;").fetchone()[0] not in (8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22): raise ValueError("schema desktop v8 à v16 requis") known_equipment = load_supplied_equipment_ids(args.catalog) known_equipment.update(row[0] for row in connection.execute( diff --git a/tools/export_equipment_definitions.py b/tools/export_equipment_definitions.py index bf04596..630f19a 100644 --- a/tools/export_equipment_definitions.py +++ b/tools/export_equipment_definitions.py @@ -24,7 +24,7 @@ def main(complete_causal_envelope=False): connection = connect_database(args.database) connection.row_factory = sqlite3.Row try: - if connection.execute("PRAGMA user_version").fetchone()[0] not in (8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21): + if connection.execute("PRAGMA user_version").fetchone()[0] not in (8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22): raise ValueError("schema desktop v8 à v16 requis") if not complete_causal_envelope and connection.execute("PRAGMA user_version").fetchone()[0] >= 20 and connection.execute("SELECT 1 FROM sync_causal_state WHERE target_kind='custom_equipment' AND deleted=1 LIMIT 1").fetchone(): raise ValueError("causal equipment protection requires the staged artifact") diff --git a/tools/export_exercise_aliases.py b/tools/export_exercise_aliases.py index b631201..009ce61 100644 --- a/tools/export_exercise_aliases.py +++ b/tools/export_exercise_aliases.py @@ -16,7 +16,7 @@ def main(): args = parser.parse_args() con = connect_database(args.database) try: - if con.execute("PRAGMA user_version").fetchone()[0] not in (12, 13, 14, 15, 16, 17, 18, 19, 20, 21): + if con.execute("PRAGMA user_version").fetchone()[0] not in (12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22): raise ValueError("schema desktop v12-v16 requis") rows = con.execute( "SELECT source_exercise_id,canonical_exercise_id FROM exercise_aliases " diff --git a/tools/export_exercise_body_zones.py b/tools/export_exercise_body_zones.py index a2358d6..6a092d9 100644 --- a/tools/export_exercise_body_zones.py +++ b/tools/export_exercise_body_zones.py @@ -41,7 +41,7 @@ def main(complete_causal_envelope=False): connection = connect_database(args.database) connection.row_factory = sqlite3.Row try: - if connection.execute("PRAGMA user_version").fetchone()[0] not in (11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21): + if connection.execute("PRAGMA user_version").fetchone()[0] not in (11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22): raise ValueError("schema desktop v11-v16 requis") if not complete_causal_envelope and connection.execute("PRAGMA user_version").fetchone()[0] >= 20 and connection.execute("SELECT 1 FROM sync_causal_state WHERE target_kind='body_zone_relation' AND deleted=1 LIMIT 1").fetchone(): raise ValueError("causal BODY ZONES protection requires the staged artifact") diff --git a/tools/export_exercise_profile_state.py b/tools/export_exercise_profile_state.py index a7573a4..ef52064 100644 --- a/tools/export_exercise_profile_state.py +++ b/tools/export_exercise_profile_state.py @@ -25,7 +25,7 @@ def main(): con.row_factory = sqlite3.Row try: version = con.execute("PRAGMA user_version").fetchone()[0] - if version not in (17, 18, 19, 20, 21): + if version not in (17, 18, 19, 20, 21, 22): raise SystemExit(f"EXERCISE_PROFILE_STATE_EXPORT=FAIL schema={version}") rows = con.execute(""" SELECT e.exercise_id,e.recording_mode,e.tracking_mode,e.data_fields, diff --git a/tools/export_pc_catalog.py b/tools/export_pc_catalog.py index d834e13..c1d39d8 100755 --- a/tools/export_pc_catalog.py +++ b/tools/export_pc_catalog.py @@ -71,7 +71,7 @@ def main(): # CONTRACT: v8 adds only desktop-local custom equipment. The PC # catalogue artifact is unchanged, but it must read the current # canonical desktop schema rather than accept a stale pre-v8 database. - if version not in (8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21): + if version not in (8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22): raise SystemExit( "PC_CATALOG_EXPORT=FAIL " f"schema={version}" diff --git a/tools/export_pc_mobile.py b/tools/export_pc_mobile.py index 90c1ac7..2e36164 100644 --- a/tools/export_pc_mobile.py +++ b/tools/export_pc_mobile.py @@ -83,9 +83,9 @@ def main(complete_causal_envelope=False): con.row_factory = sqlite3.Row try: schema_version = con.execute("PRAGMA user_version").fetchone()[0] - if args.version == 4 and schema_version not in (19, 20, 21): + if args.version == 4 and schema_version not in (19, 20, 21, 22): raise ValueError("mobile V4 exige le schéma desktop v19 ou v20") - if schema_version not in (11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21) and not (args.version == 2 and schema_version == 10): + if schema_version not in (11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22) and not (args.version == 2 and schema_version == 10): raise ValueError("schema desktop v11-v16 requis (v10 accepté pour export V2 explicite)") if not complete_causal_envelope and schema_version >= 20 and con.execute( "SELECT 1 FROM sync_causal_state WHERE deleted=1 LIMIT 1").fetchone(): diff --git a/tools/export_training_feedback.py b/tools/export_training_feedback.py index 5f0ed4e..f16c0d5 100644 --- a/tools/export_training_feedback.py +++ b/tools/export_training_feedback.py @@ -10,7 +10,7 @@ def main(complete_causal_envelope=False): parser=argparse.ArgumentParser(); parser.add_argument("output",type=Path); parser.add_argument("--database",type=Path,required=True); args=parser.parse_args() db=connect_database(args.database) try: - if db.execute("PRAGMA user_version").fetchone()[0] not in (15, 16, 17, 18, 19, 20, 21): raise ValueError("schema desktop v15-v21 requis") + if db.execute("PRAGMA user_version").fetchone()[0] not in (15, 16, 17, 18, 19, 20, 21, 22): raise ValueError("schema desktop v15-v22 requis") if not complete_causal_envelope and db.execute("PRAGMA user_version").fetchone()[0] >= 20 and db.execute("SELECT 1 FROM sync_causal_state WHERE target_kind='feedback' AND deleted=1 LIMIT 1").fetchone(): raise ValueError("causal feedback protection requires the staged artifact") roots=db.execute("SELECT f.feedback_id,s.session_id,se.entry_id,e.exercise_id,f.observed_at FROM exercise_feedback f JOIN session_exercises se ON se.id=f.session_exercise_row_id JOIN sessions s ON s.id=se.session_row_id JOIN exercises e ON e.id=se.exercise_row_id").fetchall() followups=db.execute("SELECT f.followup_id,s.session_id,f.observed_at FROM session_followups f JOIN sessions s ON s.id=f.session_row_id").fetchall() diff --git a/tools/import_ai_session_draft.py b/tools/import_ai_session_draft.py index 4b7df37..863260a 100644 --- a/tools/import_ai_session_draft.py +++ b/tools/import_ai_session_draft.py @@ -193,8 +193,8 @@ def import_payload(connection, payload, imported_at=None): INVARIANT: validation and every child insert share one IMMEDIATE transaction; any failure leaves no draft, child, or import-identity row behind. """ - if connection.execute("PRAGMA user_version").fetchone()[0] not in (18, 19, 20, 21): - raise ImportFailure("schema desktop v18-v21 requis") + if connection.execute("PRAGMA user_version").fetchone()[0] not in (18, 19, 20, 21, 22): + raise ImportFailure("schema desktop v18-v22 requis") draft, entries = validate(payload, connection) digest = canonical_digest(payload) imported_at = imported_at or dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") diff --git a/tools/import_equipment_associations.py b/tools/import_equipment_associations.py index 1beb773..614ee33 100644 --- a/tools/import_equipment_associations.py +++ b/tools/import_equipment_associations.py @@ -187,7 +187,7 @@ def main(): payload = json.loads(args.artifact.read_text(encoding="utf-8")) connection = connect_database(args.database) try: - if connection.execute("PRAGMA user_version;").fetchone()[0] not in (8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21): + if connection.execute("PRAGMA user_version;").fetchone()[0] not in (8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22): fail("schema desktop v8 à v21 requis") proof_path = args.mobile_export if proof_path is None: diff --git a/tools/import_equipment_definitions.py b/tools/import_equipment_definitions.py index b203abe..d9be1f3 100644 --- a/tools/import_equipment_definitions.py +++ b/tools/import_equipment_definitions.py @@ -53,7 +53,7 @@ def validate(payload, reserved): def apply_definitions(connection, definitions, complete_causal_envelope=False): """Apply validated definitions without owning the surrounding transaction.""" - if connection.execute("PRAGMA user_version").fetchone()[0] not in (8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21): + if connection.execute("PRAGMA user_version").fetchone()[0] not in (8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22): fail("schema desktop v8 à v21 requis") if not complete_causal_envelope and connection.execute("PRAGMA user_version").fetchone()[0] >= 20 and connection.execute("SELECT 1 FROM sync_causal_state WHERE target_kind='custom_equipment' AND deleted=1 LIMIT 1").fetchone(): fail("causal equipment protection requires the staged artifact") diff --git a/tools/import_exercise_aliases.py b/tools/import_exercise_aliases.py index e989168..f672781 100644 --- a/tools/import_exercise_aliases.py +++ b/tools/import_exercise_aliases.py @@ -56,7 +56,7 @@ def load(path): def apply_aliases(con, aliases): """Apply parsed aliases inside the caller-owned transaction.""" - if con.execute("PRAGMA user_version").fetchone()[0] not in (12, 13, 14, 15, 16, 17, 18, 19, 20, 21): + if con.execute("PRAGMA user_version").fetchone()[0] not in (12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22): fail("schema desktop v12-v21 requis") for source, canonical in aliases: target = con.execute("SELECT id,tracking_mode,recording_mode,data_fields FROM exercises WHERE exercise_id=?", (canonical,)).fetchone() diff --git a/tools/import_exercise_body_zones.py b/tools/import_exercise_body_zones.py index da98d45..bda9ed9 100644 --- a/tools/import_exercise_body_zones.py +++ b/tools/import_exercise_body_zones.py @@ -151,7 +151,7 @@ def replace(connection, row_id, primary, secondary): def apply_body_zones(connection, parsed, mobile_proof, complete_causal_envelope=False): """Apply validated zone rows without owning transaction boundaries.""" - if connection.execute("PRAGMA user_version").fetchone()[0] not in (11,12,13,14,15,16,17,18,19,20,21): raise ImportFailure("schema desktop v11-v21 requis") + if connection.execute("PRAGMA user_version").fetchone()[0] not in (11,12,13,14,15,16,17,18,19,20,21,22): raise ImportFailure("schema desktop v11-v22 requis") if not complete_causal_envelope and connection.execute("PRAGMA user_version").fetchone()[0]>=20 and connection.execute("SELECT 1 FROM sync_causal_state WHERE target_kind='body_zone_relation' AND deleted=1 LIMIT 1").fetchone(): raise ImportFailure("causal BODY ZONES protection requires the staged artifact") connection.execute("PRAGMA foreign_keys=ON");grouped={} # CONTRACT: resolve complete alias groups before the first write; direct diff --git a/tools/import_exercise_profile_state.py b/tools/import_exercise_profile_state.py index d2c809e..a70db81 100644 --- a/tools/import_exercise_profile_state.py +++ b/tools/import_exercise_profile_state.py @@ -49,7 +49,7 @@ def validate(item): def apply_profile_state(con, root, allow_pending=False): if not isinstance(root,dict) or set(root)!={"format","version","generated_at","exercises"} or root["format"]!="trainlog-exercise-profile-state" or type(root["version"]) is not int or root["version"]!=1 or not isinstance(root["exercises"],list): fail("enveloppe profile-state invalide") parse_timestamp(root["generated_at"], "generated_at");con.row_factory=sqlite3.Row - if con.execute("PRAGMA user_version").fetchone()[0] not in (17,18,19,20,21): fail("schema desktop v17-v21 requis") + if con.execute("PRAGMA user_version").fetchone()[0] not in (17,18,19,20,21,22): fail("schema desktop v17-v22 requis") applied=equal=ancestor=pending=0;seen=set() for item in root["exercises"]: validate(item);eid=item["exercise_id"] diff --git a/tools/import_mobile_export.py b/tools/import_mobile_export.py index b2e8e11..e2efbb4 100755 --- a/tools/import_mobile_export.py +++ b/tools/import_mobile_export.py @@ -846,9 +846,9 @@ def require_supported_schema(connection): # CONTRACT: v9 owns explicit max_results; earlier supported schemas remain # readable for legacy artifacts and are never made to fake that table. - if version not in (5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21): + if version not in (5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22): raise ImportFailure( - f"base desktop schema v5 à v21 attendue, version trouvée: {version}" + f"base desktop schema v5 à v22 attendue, version trouvée: {version}" ) @@ -1978,8 +1978,8 @@ def apply_payload(connection, payload, trace_exercises=False): require_supported_schema(connection) schema_version = connection.execute("PRAGMA user_version;").fetchone()[0] - if payload["version"] == 4 and schema_version not in (19, 20, 21): - raise ImportFailure("mobile V4 exige le schéma desktop v19, v20 ou v21") + if payload["version"] == 4 and schema_version not in (19, 20, 21, 22): + raise ImportFailure("mobile V4 exige le schéma desktop v19 à v22") if schema_version >= 20: protected = {(row[0], row[1]) for row in connection.execute( "SELECT target_kind,target_id FROM sync_causal_state WHERE deleted=1")} diff --git a/tools/import_training_feedback.py b/tools/import_training_feedback.py index 02f09ae..cacf45e 100644 --- a/tools/import_training_feedback.py +++ b/tools/import_training_feedback.py @@ -52,7 +52,7 @@ def merge_revisions(db,table,parent_column,parent_id,revisions): def apply_feedback(db,root,complete_causal_envelope=False): roots_added=roots_skipped=revisions_added=revisions_skipped=0 db.execute("PRAGMA foreign_keys=ON") - if db.execute("PRAGMA user_version").fetchone()[0] not in (15,16,17,18,19,20,21):fail("schema desktop v15-v21 requis") + if db.execute("PRAGMA user_version").fetchone()[0] not in (15,16,17,18,19,20,21,22):fail("schema desktop v15-v22 requis") if not complete_causal_envelope and db.execute("PRAGMA user_version").fetchone()[0]>=20 and db.execute("SELECT 1 FROM sync_causal_state WHERE target_kind='feedback' AND deleted=1 LIMIT 1").fetchone():fail("causal feedback protection requires the staged artifact") seen=set() for index,item in enumerate(root["exercise_feedback"]): diff --git a/tools/package_sync_candidate.py b/tools/package_sync_candidate.py index 640a852..12bc4d5 100644 --- a/tools/package_sync_candidate.py +++ b/tools/package_sync_candidate.py @@ -95,8 +95,8 @@ def main() -> int: ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True ).strip(), "product_version": "0.1.2", - "desktop_schema": 21, - "android_schema": 20, + "desktop_schema": 22, + "android_schema": 21, "protocols": [ "mobile-export-v3", "mobile-history-v4", @@ -104,6 +104,7 @@ def main() -> int: "execution-draft-v1", "generation-manifest-v1", "generation-ack-v1", + "generation-archive-v1", ], "entry_points": [ "bin/trainlog", diff --git a/tools/sync_generation_exchange.py b/tools/sync_generation_exchange.py index 98cab0e..33285da 100644 --- a/tools/sync_generation_exchange.py +++ b/tools/sync_generation_exchange.py @@ -199,8 +199,8 @@ def validate_manifest_bytes(raw: bytes, expected_consumer: str | None = None): def require_schema(db: sqlite3.Connection) -> None: - if db.execute("PRAGMA user_version").fetchone()[0] != 21: - raise GenerationError("desktop schema v21 required") + if db.execute("PRAGMA user_version").fetchone()[0] != 22: + raise GenerationError("desktop schema v22 required") def peer_identity(db: sqlite3.Connection, kind: str) -> str: @@ -215,6 +215,106 @@ def peer_identity(db: sqlite3.Connection, kind: str) -> str: return identity +def archive_digest(directory: Path) -> str: + digest = hashlib.sha256() + for path in sorted(value for value in directory.rglob("*") if value.is_file()): + if path.is_symlink(): + raise GenerationError("archive contains symbolic link") + relative = path.relative_to(directory).as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(4, "big")) + digest.update(relative) + digest.update(bytes.fromhex(digest_file(path)[1])) + return digest.hexdigest() + + +def archive_acknowledged(database: Path, owned_root: Path, consumer: str) -> int: + """Durably archive eligible acknowledged payloads without deleting evidence. + + WHY: acknowledged generations otherwise consume the bounded active window + forever. CONTRACT: exact correlated consumed ACK evidence is required and + the newest two acknowledged lineage members remain active for recovery. + INVARIANT: generation, artifact, ACK and causal rows are never removed or + rewritten; a ledger row is committed only after a verified atomic copy. + """ + archive_root = owned_root / "archives" / "generations" + archive_root.mkdir(parents=True, exist_ok=True, mode=0o700) + with closing(connect_database(database)) as db: + require_schema(db) + rows = db.execute( + "SELECT g.generation_id,g.run_id,g.producer_peer_id,g.consumer_peer_id," + "g.manifest_sha256,g.staging_path,g.manifest_json FROM sync_generations g " + "WHERE g.consumer_peer_id=? AND g.status='acknowledged' " + "AND NOT EXISTS(SELECT 1 FROM sync_generation_archives a WHERE a.generation_id=g.generation_id) " + "ORDER BY g.generated_at DESC,g.generation_id DESC", + (consumer,), + ).fetchall() + archived = 0 + for generation, run, producer, target, manifest_sha, staging, manifest_json in rows[2:]: + with closing(connect_database(database)) as db: + ack = db.execute( + "SELECT ack_id FROM sync_acknowledgements WHERE generation_id=? AND run_id=? " + "AND producer_peer_id=? AND consumer_peer_id=? AND manifest_sha256=? " + "AND result='consumed' AND durability='sqlite-commit-full' LIMIT 2", + (generation, run, producer, target, manifest_sha), + ).fetchall() + if len(ack) != 1: + continue + source = Path(staging) + if source.is_symlink(): + raise GenerationError("generation staging is a symbolic link") + manifest = validate_manifest_bytes(manifest_json.encode(), target) + validate_published(source, target) + destination = archive_root / generation + temporary = archive_root / ("." + generation + ".tmp") + if destination.exists(): + if destination.is_symlink(): + raise GenerationError("generation archive is a symbolic link") + validate_published(destination, target) + checksum = archive_digest(destination) + else: + if temporary.exists(): + if temporary.is_symlink(): + raise GenerationError("temporary generation archive is a symbolic link") + shutil.rmtree(temporary) + shutil.copytree(source, temporary, copy_function=shutil.copy2) + validate_published(temporary, target) + checksum = archive_digest(temporary) + for copied in temporary.rglob("*"): + if copied.is_file(): + with copied.open("rb") as stream: + os.fsync(stream.fileno()) + os.replace(temporary, destination) + directory_fd = os.open(archive_root, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + audit = canonical({ + "format": "trainlog-sync-generation-archive-audit", + "version": 1, + "generation_id": generation, + "run_id": run, + "manifest_sha256": manifest_sha, + "ack_id": ack[0][0], + "archive_sha256": checksum, + }).decode() + with closing(connect_database(database)) as db: + db.execute("BEGIN IMMEDIATE") + current = db.execute( + "SELECT status,manifest_sha256 FROM sync_generations WHERE generation_id=?", + (generation,), + ).fetchone() + if current != ("acknowledged", manifest_sha): + raise GenerationError("generation changed during archive") + db.execute( + "INSERT OR IGNORE INTO sync_generation_archives VALUES(?,?,?,?,?,?)", + (generation, str(destination), manifest_sha, checksum, now(), audit), + ) + db.commit() + archived += 1 + return archived + + def run_export(tool: str, extra: tuple[str, ...], output: Path, snapshot: Path) -> None: command = [sys.executable, str(ROOT / "tools" / tool)] if tool in ("execution_draft_exchange.py", "causal_delete_exchange.py"): @@ -249,9 +349,10 @@ def capture_desktop(database: Path, owned_root: Path, consumer: str, run = run_id or "sy_" + str(uuid.uuid4()) if not valid_id(generation, "gen") or not valid_id(run, "sy"): raise GenerationError("invalid generation/run") + archive_acknowledged(database, owned_root, consumer) with closing(connect_database(database)) as admission: require_schema(admission) - retained = admission.execute("SELECT COUNT(*) FROM sync_generations WHERE consumer_peer_id=? AND status IN('captured','published','waiting_acknowledgement','acknowledged')", (consumer,)).fetchone()[0] + retained = admission.execute("SELECT COUNT(*) FROM sync_generations g WHERE consumer_peer_id=? AND status IN('captured','published','waiting_acknowledgement','acknowledged') AND NOT EXISTS(SELECT 1 FROM sync_generation_archives a WHERE a.generation_id=g.generation_id)", (consumer,)).fetchone()[0] if retained >= MAX_RETAINED_PER_PEER: raise GenerationError("generation retention capacity exhausted") stage = owned_root / "staging" / generation @@ -539,10 +640,23 @@ def accept_ack(database: Path, path: Path) -> str: raise GenerationError("ACK has no pending generation") ack = validate_ack(raw, row) status = "acknowledged" if ack["result"] == "consumed" else "rejected" + db.execute( + "INSERT OR IGNORE INTO sync_acknowledgements VALUES(?,?,?,?,?,?,?,?,?,?,?)", + (ack["ack_id"], ack["generation_id"], ack["run_id"], + ack["producer_peer_id"], ack["consumer_peer_id"], + ack["manifest_sha256"], ack["result"], ack["durability"], + ack["consumed_at"], ack["diagnostic"], ack["payload_sha256"]), + ) + stored = db.execute( + "SELECT payload_sha256 FROM sync_acknowledgements WHERE ack_id=?", + (ack["ack_id"],), + ).fetchone() + if stored != (ack["payload_sha256"],): + raise GenerationError("conflicting ACK replay") if row[5] in ("acknowledged", "rejected"): if row[5] != status: raise GenerationError("conflicting ACK replay") - db.rollback(); return "unchanged" + db.commit(); return "unchanged" if row[5] != "waiting_acknowledgement": raise GenerationError("generation is not awaiting ACK") db.execute("UPDATE sync_generations SET status=?,acknowledged_at=? WHERE generation_id=?", diff --git a/tools/sync_orchestrator.py b/tools/sync_orchestrator.py index 41a16b0..cd1fd23 100644 --- a/tools/sync_orchestrator.py +++ b/tools/sync_orchestrator.py @@ -123,6 +123,8 @@ def failure_code(error: BaseException) -> str: diagnostic = str(error).lower() if "transport_timeout:" in diagnostic or "peer worker timed out" in diagnostic: return "transport_timeout" + if "peer_capacity_exhausted:" in diagnostic: + return "peer_capacity_exhausted" if "expected android peer not found" in diagnostic: return "device_unavailable" if "capacity exhausted" in diagnostic or "retained generation capacity" in diagnostic: diff --git a/tools/sync_peer_worker.py b/tools/sync_peer_worker.py index edde883..e91b7c6 100644 --- a/tools/sync_peer_worker.py +++ b/tools/sync_peer_worker.py @@ -30,6 +30,7 @@ CAPS = { "causal-delete-v1", "mobile-history-v4", "execution-draft-v1", + "generation-archive-v1", } @@ -53,9 +54,12 @@ def bounded(path: Path, limit: int = 65536) -> bytes: def publish_json(path: Path, value: dict) -> None: """Publish a coordination object only after all bytes are durable.""" + raw = generation.canonical(value) + if len(raw) > 65536: + raise RuntimeError(f"{path.name} exceeds bound") temporary = path.with_name("." + path.name + ".tmp-" + str(os.getpid())) with temporary.open("wb") as stream: - stream.write(generation.canonical(value)) + stream.write(raw) stream.flush() os.fsync(stream.fileno()) os.chmod(temporary, 0o600) @@ -166,6 +170,27 @@ def wait_json(path: Path, deadline: float, predicate, pump=None) -> dict: time.sleep(0.05) +def correlated_peer_error(root: Path, run_id: str) -> None: + path = root / "android-generation-error-v1.json" + if not path.is_file(): + return + try: + value = json.loads(bounded(path)) + except (OSError, json.JSONDecodeError): + return + if ( + isinstance(value, dict) + and value.get("format") == "trainlog-sync-generation-error" + and value.get("version") == 1 + and value.get("run_id") == run_id + and value.get("code") == "peer_capacity_exhausted" + ): + raise RuntimeError( + "peer_capacity_exhausted: Android active generation capacity is exhausted; " + "archive acknowledged generations, then retry explicitly" + ) + + def load_peer(root: Path, expected: str) -> dict: value = json.loads(bounded(root / "android-peer-v1.json")) if ( @@ -225,12 +250,28 @@ def main() -> int: } request_path = args.transport_root / "request-v1.json" publish_json(request_path, request) + with closing(generation.connect_database(args.database)) as db: + acknowledgement_rows = db.execute( + "SELECT ack_json FROM sync_consumed_generations WHERE producer_peer_id=? " + "AND consumer_peer_id=? AND result='consumed' ORDER BY consumed_at,generation_id LIMIT 32", + (peer["peer_id"], desktop_peer), + ).fetchall() + archive_acknowledgements = { + "format": "trainlog-sync-archive-acknowledgements", + "version": 1, + "run_id": args.run_id, + "android_peer_id": peer["peer_id"], + "desktop_peer_id": desktop_peer, + "acknowledgements": [json.loads(row[0]) for row in acknowledgement_rows], + } + archive_ack_path = args.transport_root / "desktop-archive-acknowledgements-v1.json" + publish_json(archive_ack_path, archive_acknowledgements) if args.mode == "mtp": push_adapter( args.mtp_adapter, args.expected_peer, args.transport_root, - ["request-v1.json"], + ["request-v1.json", "desktop-archive-acknowledgements-v1.json"], deadline, ) emit( @@ -240,8 +281,13 @@ def main() -> int: consumer_peer_id=desktop_peer, ) inbound_ref = args.transport_root / "android-generation-v1.json" + def pump_inbound(): + if pump is not None: + pump() + correlated_peer_error(args.transport_root, args.run_id) + inbound = wait_json( - inbound_ref, deadline, lambda value: value.get("run_id") == args.run_id, pump + inbound_ref, deadline, lambda value: value.get("run_id") == args.run_id, pump_inbound ) inbound_dir = args.transport_root / inbound["relative_path"] ack = generation.consume_desktop(args.database, inbound_dir) @@ -252,7 +298,7 @@ def main() -> int: args.mtp_adapter, args.expected_peer, args.transport_root, - ["request-v1.json", "desktop-consumption-ack-v1.json"], + ["request-v1.json", "desktop-archive-acknowledgements-v1.json", "desktop-consumption-ack-v1.json"], deadline, ) if ack["result"] != "consumed": @@ -291,6 +337,7 @@ def main() -> int: args.transport_root, [ "request-v1.json", + "desktop-archive-acknowledgements-v1.json", "desktop-consumption-ack-v1.json", "desktop-generation-v1.json", ref["relative_path"], diff --git a/tui/include/trainlog/database.h b/tui/include/trainlog/database.h index 94dc3aa..4ebc245 100644 --- a/tui/include/trainlog/database.h +++ b/tui/include/trainlog/database.h @@ -16,7 +16,7 @@ #include "trainlog/model.h" #include "trainlog/status.h" -#define TRAINLOG_DATABASE_SCHEMA_VERSION 21 +#define TRAINLOG_DATABASE_SCHEMA_VERSION 22 typedef struct TrainlogDatabase TrainlogDatabase; diff --git a/tui/meson.build b/tui/meson.build index 2a60475..4fd449a 100644 --- a/tui/meson.build +++ b/tui/meson.build @@ -770,6 +770,12 @@ test( args: [meson.project_source_root() / 'tests/test_sync_generation_exchange.py'], timeout: 120, ) +test( + 'generation_archival', + find_program('python3'), + args: [meson.project_source_root() / 'tests/test_generation_archival.py'], + timeout: 120, +) test( 'sync_orchestrator', find_program('python3'), diff --git a/tui/src/database.c b/tui/src/database.c index 58bd2cf..d044a5d 100644 --- a/tui/src/database.c +++ b/tui/src/database.c @@ -870,6 +870,20 @@ static const char *const MIGRATE_V20_TO_V21_SQL = "CREATE TABLE sync_causal_publications(operation_id TEXT NOT NULL REFERENCES sync_causal_operations(operation_id),generation_id TEXT NOT NULL,first_emission INTEGER NOT NULL CHECK(first_emission IN(0,1)),PRIMARY KEY(operation_id,generation_id));" "PRAGMA user_version=21;COMMIT;"; +/* WHY: acknowledged payload copies cannot consume active admission forever, + * but generation identity, lineage and ACK evidence must remain permanent. + * CONTRACT: v22 adds an append-only archive ledger; it neither fabricates an + * ACK nor changes a generation status. INVARIANT: an archive row is valid only + * after production code has durably copied and verified the complete payload. */ +static const char *const MIGRATE_V21_TO_V22_SQL = + "BEGIN IMMEDIATE;" + "CREATE TABLE sync_generation_archives(" + "generation_id TEXT PRIMARY KEY REFERENCES sync_generations(generation_id)," + "archive_path TEXT NOT NULL UNIQUE,manifest_sha256 TEXT NOT NULL," + "archive_sha256 TEXT NOT NULL,archived_at TEXT NOT NULL," + "audit_json TEXT NOT NULL);" + "PRAGMA user_version=22;COMMIT;"; + /* CONTRACT: v18 was already deployed before bounded draft publication was * added. Keep its version number and add only the nullable lifecycle cursor; * NULL deliberately means retryable/not yet published. */ @@ -1565,7 +1579,7 @@ static TrainlogStatus initialize_or_validate_schema( status = TRAINLOG_STATUS_OK; } else if (version == 11 || version == 12 || version == 13 || version == 14 || version == 15 || version == 16 || version == 17 || version == 18 || - version == 19 || version == 20 || version == 21) { + version == 19 || version == 20 || version == 21 || version == 22) { status = TRAINLOG_STATUS_OK; } else { if (version == 1) { @@ -1738,6 +1752,9 @@ static TrainlogStatus initialize_or_validate_schema( if (status == TRAINLOG_STATUS_OK && version < 21) { status = execute_sql(database, MIGRATE_V20_TO_V21_SQL); } + if (status == TRAINLOG_STATUS_OK && version < 22) { + status = execute_sql(database, MIGRATE_V21_TO_V22_SQL); + } if (status == TRAINLOG_STATUS_OK) { status = ensure_v18_ai_draft_publication_state(database); } @@ -1753,7 +1770,7 @@ static TrainlogStatus initialize_or_validate_schema( set_open_diagnostic( output_diagnostic, output_diagnostic_capacity, - version == 0 ? "create schema v21" : "migrate database to schema v21", + version == 0 ? "create schema v22" : "migrate database to schema v22", database->connection, SQLITE_ERROR ); diff --git a/tui/src/generation_mtp.c b/tui/src/generation_mtp.c index da6a1f7..22e39e0 100644 --- a/tui/src/generation_mtp.c +++ b/tui/src/generation_mtp.c @@ -598,6 +598,10 @@ TrainlogStatus trainlog_generation_mtp_pull(const TrainlogGenerationMtpIo *io, status = pull_named_file( io, &selection, selection.root, "android-consumption-ack-v1.json", local, false); } + if (status == TRAINLOG_STATUS_OK) { + status = pull_named_file( + io, &selection, selection.root, "android-generation-error-v1.json", local, false); + } if (status == TRAINLOG_STATUS_OK && referenced_android_generation(local, generation, sizeof(generation))) { status = pull_referenced_generation(io, &selection, local, generation); diff --git a/tui/tests/test_schema_v21_migration.c b/tui/tests/test_schema_v21_migration.c index e4c8193..fb088a4 100644 --- a/tui/tests/test_schema_v21_migration.c +++ b/tui/tests/test_schema_v21_migration.c @@ -27,16 +27,17 @@ int main(void) TRAINLOG_TRACKING_REPS,TRAINLOG_RECORDING_SETS,0U)==TRAINLOG_STATUS_OK); trainlog_database_close(database);database=NULL; CHECK(sqlite3_open(path,&raw)==SQLITE_OK); - CHECK(sqlite3_exec(raw,"PRAGMA foreign_keys=OFF;DROP TABLE sync_causal_publications;DROP TABLE sync_acknowledgements;DROP TABLE sync_consumed_generations;DROP TABLE sync_generation_artifacts;DROP TABLE sync_generations;DROP TABLE sync_peer_identity;PRAGMA user_version=20;",NULL,NULL,NULL)==SQLITE_OK); + CHECK(sqlite3_exec(raw,"PRAGMA foreign_keys=OFF;DROP TABLE sync_generation_archives;DROP TABLE sync_causal_publications;DROP TABLE sync_acknowledgements;DROP TABLE sync_consumed_generations;DROP TABLE sync_generation_artifacts;DROP TABLE sync_generations;DROP TABLE sync_peer_identity;PRAGMA user_version=20;",NULL,NULL,NULL)==SQLITE_OK); CHECK(sqlite3_close(raw)==SQLITE_OK); CHECK(trainlog_database_open(path,&database)==TRAINLOG_STATUS_OK); CHECK(trainlog_database_exercise_count(database,&exercises)==TRAINLOG_STATUS_OK);CHECK(exercises>0U); trainlog_database_close(database);database=NULL; CHECK(sqlite3_open(path,&raw)==SQLITE_OK); - CHECK(scalar(raw,"PRAGMA user_version")==21); + CHECK(scalar(raw,"PRAGMA user_version")==22); CHECK(scalar(raw,"SELECT COUNT(*) FROM sync_generations")==0); CHECK(scalar(raw,"SELECT COUNT(*) FROM sync_consumed_generations")==0); CHECK(scalar(raw,"SELECT COUNT(*) FROM sync_acknowledgements")==0); + CHECK(scalar(raw,"SELECT COUNT(*) FROM sync_generation_archives")==0); CHECK(sqlite3_close(raw)==SQLITE_OK);CHECK(unlink(path)==0); return 0; } diff --git a/tui/tests/test_schema_v7_migration.c b/tui/tests/test_schema_v7_migration.c index f3420fd..c1bf49b 100644 --- a/tui/tests/test_schema_v7_migration.c +++ b/tui/tests/test_schema_v7_migration.c @@ -134,8 +134,8 @@ static bool test_v7_migration_sqlite_failure_has_diagnostic(void) CHECK(trainlog_database_open_with_diagnostic(path, &database, diagnostic, sizeof(diagnostic)) == TRAINLOG_STATUS_DATABASE_ERROR); CHECK(database == NULL); - CHECK(strncmp(diagnostic, "migrate database to schema v21: SQLite rc=", - strlen("migrate database to schema v21: SQLite rc=")) == 0); + CHECK(strncmp(diagnostic, "migrate database to schema v22: SQLite rc=", + strlen("migrate database to schema v22: SQLite rc=")) == 0); CHECK(strstr(diagnostic, "extended_rc=") != NULL); CHECK(strstr(diagnostic, "custom_equipment") != NULL); CHECK(strstr(diagnostic, "already exists") != NULL); @@ -154,12 +154,12 @@ static bool test_newer_schema_has_application_diagnostic(void) CHECK(fd >= 0); CHECK(close(fd) == 0); CHECK(sqlite3_open(path, &raw) == SQLITE_OK); - CHECK(sqlite3_exec(raw, "PRAGMA user_version=22;", NULL, NULL, NULL) == SQLITE_OK); + CHECK(sqlite3_exec(raw, "PRAGMA user_version=23;", NULL, NULL, NULL) == SQLITE_OK); CHECK(sqlite3_close(raw) == SQLITE_OK); CHECK(trainlog_database_open_with_diagnostic(path, &database, diagnostic, sizeof(diagnostic)) == TRAINLOG_STATUS_SCHEMA_UNSUPPORTED); CHECK(database == NULL); - CHECK(strstr(diagnostic, "schema version 22 is newer") != NULL); + CHECK(strstr(diagnostic, "schema version 23 is newer") != NULL); CHECK(strstr(diagnostic, "SQLite") == NULL); CHECK(unlink(path) == 0); return true; diff --git a/tui/tests/test_schema_v9_migration.c b/tui/tests/test_schema_v9_migration.c index 226d2bc..6af49ee 100644 --- a/tui/tests/test_schema_v9_migration.c +++ b/tui/tests/test_schema_v9_migration.c @@ -151,7 +151,7 @@ static bool test_v9_to_v10_failure_rolls_back(void) CHECK(trainlog_database_open_with_diagnostic(path, &database, diagnostic, sizeof(diagnostic)) == TRAINLOG_STATUS_DATABASE_ERROR); CHECK(database == NULL); - CHECK(strstr(diagnostic, "migrate database to schema v21") != NULL); + CHECK(strstr(diagnostic, "migrate database to schema v22") != NULL); CHECK(strstr(diagnostic, "performed_sets_v9") != NULL); CHECK(sqlite3_open(path, &raw) == SQLITE_OK); diff --git a/web/src/components/SyncControl.tsx b/web/src/components/SyncControl.tsx index f6c681e..2425334 100644 --- a/web/src/components/SyncControl.tsx +++ b/web/src/components/SyncControl.tsx @@ -34,6 +34,9 @@ function statusAction(status: SyncStatus | null): string { if (status?.phase === 'failed' && status.error_code === 'transport_timeout') { return 'Vérifiez le dialogue USB/MTP sur Android avant de relancer.' } + if (status?.phase === 'failed' && status.error_code === 'peer_capacity_exhausted') { + return 'Conservez l’application Android ouverte : les générations acquittées doivent être archivées avant une relance explicite.' + } return '' } diff --git a/web/src/dashboard/DashboardTiles.test.tsx b/web/src/dashboard/DashboardTiles.test.tsx index aaabd26..3fc3d72 100644 --- a/web/src/dashboard/DashboardTiles.test.tsx +++ b/web/src/dashboard/DashboardTiles.test.tsx @@ -40,6 +40,17 @@ describe('tuiles Dashboard alimentées par le contrat', () => { expect(screen.queryByText('Aucun élément disponible')).not.toBeInTheDocument() }) + it('rend chaque proposition consultable sans la transformer en brouillon', () => { + render() + const expected = document.querySelector('[data-prepared-identity="aid_expected"]') + expect(expected).toHaveTextContent('Attendue') + expect(expected).toHaveAttribute('data-prepared-identity', 'aid_expected') + expect(document.body).not.toHaveTextContent('Brouillon d’exécution · à reprendre') + }) + it.each(['compact', 'medium', 'large'] as const)('rend les 90 jours et les sommes factuelles en taille %s', (size) => { const { container } = render() const heatmap = screen.getByRole('img') diff --git a/web/src/dashboard/DashboardTiles.tsx b/web/src/dashboard/DashboardTiles.tsx index fb16322..c48934e 100644 --- a/web/src/dashboard/DashboardTiles.tsx +++ b/web/src/dashboard/DashboardTiles.tsx @@ -41,10 +41,25 @@ export function NextSessionTile({

{heading}

{title} {size !== 'compact' && ( -
- - -
+ <> +
+ + +
+ {preparedItems.items.length > 1 && ( +
+ {preparedItems.items.length} éléments préparés +
    + {preparedItems.items.map((prepared) => ( +
  • + {prepared.title || (prepared.kind === 'ai_proposal' ? 'Proposition sans titre' : 'Séance préparée')} + {' · '}{prepared.kind === 'ai_proposal' ? 'proposition' : 'brouillon d’exécution'} +
  • + ))} +
+
+ )} + )} {failed && Dernière lecture conservée · actualisation indisponible} {pending && !failed && Actualisation…}