From 0f99415317ac869c46506e266fb13b74218afacb Mon Sep 17 00:00:00 2001 From: fy59 Date: Sat, 8 Aug 2026 12:27:02 +0200 Subject: [PATCH] feat(catalog): add persistent scan sets and image assets --- README.md | 4 +- docs/architecture/overview.md | 16 +- docs/architecture/persistence.md | 55 +- docs/architecture/project_database.md | 108 +++- docs/architecture/reconstruction_pipeline.md | 23 +- docs/architecture/resource_governor.md | 3 +- docs/architecture/runtime.md | 4 +- docs/architecture/task_kind_registry.md | 2 +- docs/architecture/task_system.md | 2 +- docs/concepts/matching_and_tracks.md | 5 + docs/concepts/scan_sets.md | 169 ++++--- docs/concepts/visual_index.md | 5 + docs/roadmap/roadmap.md | 4 +- include/lardon3d/image_catalog.h | 25 + include/lardon3d/import.h | 11 + include/lardon3d/import_task.h | 2 + include/lardon3d/project.h | 3 +- include/lardon3d/project_db.h | 79 ++- meson.build | 42 +- src/image_catalog_persistent.c | 219 ++++++++ src/import.c | 104 ++++ src/import_task.c | 42 +- src/project.c | 15 +- src/project_db.c | 500 ++++++++++++++++++- tests/test_import_task.c | 33 +- tests/test_persistent_image_catalog.c | 278 +++++++++++ tests/test_project_db.c | 126 ++++- 27 files changed, 1686 insertions(+), 193 deletions(-) create mode 100644 src/image_catalog_persistent.c create mode 100644 tests/test_persistent_image_catalog.c diff --git a/README.md b/README.md index 8809dc7..9276dd4 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,11 @@ Lardon3D ne vise pas simplement "dossier de photos → objet 3D", mais "ensemble - **Project** : cycle de vie persistant, identité stable et Project Database ouverte - **Import** : premier task kind de production, exécuté par la file générique en lots bornés et reprenables -- **Image Catalog** : indexage des métadonnées d'images +- **ScanSet / Image Catalog v1** : acquisitions, images logiques, provenance et assets SHA-256 persistants et paginés - **Image View** : vues triées et filtrées pour la TUI - **Task** : moteur de tâches avec pause/reprise, annulation et séquences - **Task Checkpoint v1** : snapshot durable, fichier atomique et reprise sûre -- **Project Database v3** : identité, tâches typées/checkpoints, paramètres d'import et inventaire d'artefacts SQLite +- **Project Database v4** : tâches, ScanSets, catalogue image, assets et migrations SQLite - **Task Kind Registry** : identité métier durable et reconstruction runtime explicite - **Recovery projet** : reprise automatique sélective et bornée des imports récupérables - **Task Queue** : file FIFO avec sélection adaptative et backpressure diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index b318473..8d341ab 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -44,15 +44,15 @@ des fichiers admissibles et maintenance d'un manifeste cohérent. **Statut :** IMPLEMENTED ### Import Task -Wrapper asynchrone de l'import avec états, progression et annulation -coopérative. Exécute l'import dans un worker dédié (non encore migré vers -le scheduler générique). +Premier type métier persistant. Il s'exécute par lots bornés dans le scheduler +générique, cible explicitement un ScanSet et peut être reconstruit puis repris. **Statut :** IMPLEMENTED -### Image Catalog -Chargement et indexage en mémoire des métadonnées d'images depuis le -manifeste du projet. Fournit un accès structuré aux images. +### ScanSet et Image Catalog +Catalogue SQLite persistant séparant acquisition, image logique et asset +physique SHA-256. Les parcours persistants sont paginés ; l'ancien catalogue +mémoire depuis `manifest.tsv` reste une façade legacy pour la TUI. **Statut :** IMPLEMENTED @@ -120,6 +120,6 @@ frontière de séquence connue. - Absence de DAG de dépendances. - Absence de priorités. - Absence de pools de workers multiples (CPU/GPU/IO). -- Import non migré vers le scheduler générique. -- Persistance des tâches et checkpoints non implémentée. +- La TUI legacy ne sélectionne pas encore explicitement ses ScanSets. +- La réconciliation globale des assets/checkpoints orphelins n'est pas implémentée. - Viewer et publication live non implémentés. diff --git a/docs/architecture/persistence.md b/docs/architecture/persistence.md index fb64d9f..a314f55 100644 --- a/docs/architecture/persistence.md +++ b/docs/architecture/persistence.md @@ -108,7 +108,7 @@ par le `time_t` local avant conversion. Le format reste donc lisible entre plateformes uniquement pour les valeurs communes à leurs domaines `size_t` et `time_t`. -## Project Database v3 +## Project Database v4 SQLite contient l'état logique interrogable et les références aux fichiers ; les checkpoints et artefacts volumineux restent externes. L'enregistrement du @@ -136,7 +136,7 @@ non durable, absent, invalide, version inconnue et erreur d'I/O. Aucune réparat ou suppression silencieuse n'est effectuée. Le format checkpoint reste en version 1 et ne contient pas de `task_kind`. Le -schéma SQLite v3 conserve `task_kind` et `task_kind_version` dans le résumé +schéma SQLite v4 conserve `task_kind` et `task_kind_version` dans le résumé logique interrogable. La migration v1→v2 laisse ces deux colonnes à `NULL` : une tâche legacy reste inspectable mais ne peut pas être reconstruite ou resoumise. Un kind inconnu ou une version non supportée est diagnostiqué sans exécuter de @@ -147,8 +147,8 @@ code. **IMPLEMENTED** — modèle durable, codec v1, lecture validée, publication atomique et restauration sûre d'une tâche isolée. -**IMPLEMENTED** — Project Database v3 pour identité, résumés de tâches typées, -références checkpoint et artefacts génériques. +**IMPLEMENTED** — Project Database v4 pour identité, tâches typées, ScanSets, +images logiques, assets SHA-256, références checkpoint et artefacts génériques. **IMPLEMENTED** — registry statique bornée et reconstruction explicite avec ownership du userdata. @@ -156,21 +156,39 @@ ownership du userdata. **IMPLEMENTED** — API projet de sauvegarde fichier+DB et inventaire validé au redémarrage. -**IMPLEMENTED** — `import.images` persiste son chemin source absolu dans une -table dédiée et publie un checkpoint après chaque lot validé. Le manifeste -publié rend le rejeu idempotent à la granularité d'une image. +**IMPLEMENTED** — `import.images` persiste son chemin source absolu et son +`scanset_id`, puis publie un checkpoint après chaque lot validé. Le catalogue +SQLite rend le rejeu idempotent à la granularité du contenu dans un ScanSet. Le chemin source absolu est l'intention durable v1 : il doit rester accessible après redémarrage et un projet déplacé ne rend pas une source externe portable. Une source absente ou devenue non-répertoire fait échouer proprement la -reconstruction. Une image déjà inscrite au manifeste est un résultat validé et -les modifications ultérieures de sa source sont ignorées. Pour la fenêtre -« copie publiée, manifeste non publié », la reprise n'adopte la destination -orpheline qu'après comparaison octet par octet avec la source ; une collision -différente est une erreur. Le manifeste est republié atomiquement avant le -checkpoint de fin de lot. Si ce checkpoint ou sa transaction DB échoue, le -manifeste demeure la frontière idempotente plus récente et un checkpoint -orphelin peut subsister selon le protocole filesystem puis SQLite. +reconstruction. Après import terminé, l'image dépend de l'asset géré, plus de la +source. Le SHA-256 est calculé pendant la copie avec un tampon fixe de 64 Kio. +L'asset est publié sans écrasement sous +`assets/images//`, puis seulement enregistré `READY` dans une +transaction SQLite. Un asset concurrent déjà présent n'est adopté qu'après +rehash complet et vérification de taille. Si SQLite échoue après publication, +le fichier reste orphelin pour une future réconciliation ; aucune transaction +FS+SQLite n'est revendiquée. + +Les identités publiées `scanset_id`, `image_id` et `asset_id` utilisent les +séquences SQLite `AUTOINCREMENT` : une valeur issue d'une transaction validée +n'est jamais réattribuée à un autre objet, même après suppression de la ligne. +Une valeur réservée par une transaction annulée n'est pas une identité publiée. + +`manifest.tsv` reste supporté par l'ancien chemin d'import/catalogue. Le chemin +persistant entretient une projection best-effort par hardlinks pour la TUI, +mais SQLite est le commit logique canonique : la correction de la reprise ne +dépend plus de l'ordre de `readdir()` ni du nom de fichier. + +La migration v3 vers v4 ne transforme pas les lignes du manifeste historique +en images cataloguées : elles ne contiennent pas toutes les preuves exigées par +le modèle v4. Le marqueur durable `legacy_image_catalog_pending` rend cette +situation visible. Une tâche récupérable peut repeupler le catalogue par rejeu +si sa source existe encore ; sinon les fichiers et le manifeste restent une +projection legacy, explicitement non cataloguée. Une tâche v3 déjà terminée +n'est pas rejouée automatiquement. **IMPLEMENTED** — reprise automatique sélective à l'ouverture : pagination bornée, validation checkpoint/kind, reconstruction production et enqueue sans @@ -184,5 +202,8 @@ terminale n'appartient pas à la requête de reprise. **NOT_YET_WIRED** — autosave complet, réconciliation des fichiers orphelins et retry piloté par l'utilisateur pour les sources indisponibles. -**PLANNED** — catalogue d'artefacts photogrammétriques réels, migrations v4+ et -reprise avec dépendances. +**NOT_YET_WIRED** — migration de la vue TUI en mémoire vers la pagination +SQLite, scrub des assets et réconciliation globale des orphelins. + +**PLANNED** — Feature Store, Visual Index, migrations v5+ et reprise avec +dépendances. diff --git a/docs/architecture/project_database.md b/docs/architecture/project_database.md index 626bf05..6accae0 100644 --- a/docs/architecture/project_database.md +++ b/docs/architecture/project_database.md @@ -131,7 +131,8 @@ La base de données projet stocke les métadonnées de reconstruction et les rel ## Relations - Project → Scan Set (1:N) -- Scan Set → Image (N:M) +- Scan Set → Image logique (1:N) +- Image logique → Image Asset (N:1) - Image → Feature Set (1:N) - Image → Visual Signature (1:N) - Candidate Pair → Image (2) @@ -167,9 +168,9 @@ référence vers le fichier checkpoint. Le fichier checkpoint validé reste la source complète pour `lardon3d_task_restore()` ; la DB seule ne reconstruit jamais une tâche. Un écart ou un fichier invalide interdit la reprise. -## Schéma v3 implémenté +## Schéma v4 implémenté -- `metadata(key PRIMARY KEY, value)` contient `schema_version=3` et +- `metadata(key PRIMARY KEY, value)` contient `schema_version=4` et `next_task_id`, prochain ID durable allouable. - `project(singleton=1, stable_id UNIQUE, name, created_at, updated_at)` décrit l'unique identité logique de la DB. @@ -182,22 +183,41 @@ jamais une tâche. Un écart ou un fichier invalide interdit la reprise. - `artifacts(artifact_id PRIMARY KEY, kind, path, state, size_bytes, producer_task_id REFERENCES tasks, created_at, updated_at)` inventorie des fichiers externes. Les états v1 sont `STAGED` et `READY`. +- `scansets(scanset_id INTEGER PRIMARY KEY AUTOINCREMENT, name, created_at, updated_at)` + représente les acquisitions logiques, y compris les ScanSets vides. +- `image_assets(asset_id INTEGER PRIMARY KEY AUTOINCREMENT, sha256 UNIQUE, path UNIQUE, + size_bytes, state, created_at)` décrit les contenus physiques `READY`. +- `images(image_id INTEGER PRIMARY KEY AUTOINCREMENT, scanset_id REFERENCES scansets, + asset_id REFERENCES image_assets, original_name, source_path, + producer_task_id REFERENCES tasks, imported_at)` décrit les images logiques. + `UNIQUE(scanset_id,asset_id)` interdit les doublons de contenu dans une même + acquisition sans fusionner deux acquisitions différentes. - `image_import_tasks(task_id PRIMARY KEY REFERENCES tasks ON DELETE CASCADE, - source_path)` conserve l'unique paramètre métier v1 de `import.images`. + source_path, scanset_id REFERENCES scansets)` conserve les paramètres + métier immuables de `import.images`. -Les indexes portent uniquement sur `tasks(recovery_state, task_id)`, -`artifacts(state, artifact_id)` et `artifacts(producer_task_id)`. +`AUTOINCREMENT` est volontairement limité à ces trois identités publiées. Il +empêche la réutilisation d'un ID issu d'une transaction validée même si sa ligne +maximale est supprimée plus tard. Le coût de `sqlite_sequence` est accepté pour +garantir qu'un futur Feature Store, match ou track ne voie jamais son identifiant +désigner un autre objet. Les IDs de transactions rollbackées ne sont pas +considérés publiés et peuvent être réutilisés. + +Les indexes ajoutés en v4 sont `images(scanset_id,image_id)`, pagination réelle, +et `images(producer_task_id,image_id)`, recherche par tâche productrice. Le +SHA-256 et le chemin asset sont déjà indexés par leurs contraintes `UNIQUE`. ## Ouverture et migrations -Une DB vide reçoit directement le schéma v3 dans une transaction +Une DB vide reçoit directement le schéma v4 dans une transaction `BEGIN IMMEDIATE`. Une DB v1 reçoit transactionnellement les colonnes nullable `task_kind` et `task_kind_version`, puis les migrations v2→v3. Les anciennes lignes restent `NULL/NULL`, sans type inventé et sans perte des projets, tâches, checkpoints ou -artefacts. Une interruption ou erreur provoque un rollback complet. Une DB v2 -est migrée vers v3 ; une DB v3 est validée puis ouverte. Une version future est refusée et une DB contenant +artefacts. Une interruption ou erreur provoque un rollback complet. Les DB v1, +v2 et v3 sont migrées séquentiellement vers v4. Une version future est refusée et une DB contenant des tables sans métadonnée de version est considérée corrompue. La fonction -interne de migration ne connaît que `0 → 3`, `1 → 2 → 3` et `2 → 3`. +interne de migration ne connaît que `0 → 4`, `1 → 2 → 3 → 4`, +`2 → 3 → 4` et `3 → 4`. Migration v1→v2 exacte, exécutée entre `BEGIN IMMEDIATE` et `COMMIT` : @@ -227,7 +247,53 @@ UPDATE metadata SET value=3 WHERE key='schema_version' AND value=2; ``` -Configuration v3 : `foreign_keys=ON`, `journal_mode=DELETE`, +Migration v3→v4 exacte, dans la même transaction : + +```sql +CREATE TABLE scansets( + scanset_id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(scanset_id>0), + name TEXT NOT NULL CHECK(length(name)>0 AND length(name)<256), + created_at INTEGER NOT NULL CHECK(created_at>=0), + updated_at INTEGER NOT NULL CHECK(updated_at>=created_at) +); +CREATE TABLE image_assets( + asset_id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(asset_id>0), + sha256 BLOB NOT NULL UNIQUE CHECK(length(sha256)=32), + path TEXT NOT NULL UNIQUE CHECK(length(path)>0 AND length(path)<4096), + size_bytes INTEGER NOT NULL CHECK(size_bytes>=0), + state INTEGER NOT NULL CHECK(state=1), + created_at INTEGER NOT NULL CHECK(created_at>=0) +); +CREATE TABLE images( + image_id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(image_id>0), + scanset_id INTEGER NOT NULL REFERENCES scansets(scanset_id), + asset_id INTEGER NOT NULL REFERENCES image_assets(asset_id), + original_name TEXT NOT NULL CHECK(length(original_name)>0 AND length(original_name)<256), + source_path TEXT NOT NULL CHECK(length(source_path)>0 AND length(source_path)<4096), + producer_task_id INTEGER REFERENCES tasks(task_id), + imported_at INTEGER NOT NULL CHECK(imported_at>=0), + UNIQUE(scanset_id,asset_id) +); +CREATE INDEX images_scanset_idx ON images(scanset_id,image_id); +CREATE INDEX images_producer_idx ON images(producer_task_id,image_id); +ALTER TABLE image_import_tasks + ADD COLUMN scanset_id INTEGER REFERENCES scansets(scanset_id); +INSERT INTO scansets(name,created_at,updated_at) + SELECT 'Imports antérieurs à ScanSet v1',0,0 + WHERE EXISTS(SELECT 1 FROM image_import_tasks); +UPDATE image_import_tasks + SET scanset_id=(SELECT scanset_id FROM scansets + WHERE name='Imports antérieurs à ScanSet v1' + ORDER BY scanset_id LIMIT 1) + WHERE scanset_id IS NULL; +INSERT INTO metadata(key,value) + VALUES('legacy_image_catalog_pending', + CASE WHEN EXISTS(SELECT 1 FROM image_import_tasks) THEN 1 ELSE 0 END); +UPDATE metadata SET value=4 + WHERE key='schema_version' AND value=3; +``` + +Configuration v4 : `foreign_keys=ON`, `journal_mode=DELETE`, `synchronous=FULL`, `busy_timeout=5000`. Le mode DELETE convient au propriétaire unique actuel, évite les fichiers WAL/SHM durables et conserve la synchronisation forte. Le timeout borne l'attente d'un verrou externe à cinq secondes. @@ -288,7 +354,7 @@ ouvert. ## Statut -**IMPLEMENTED** — SQLite système, schéma v3 et migrations v1→v2→v3, identité +**IMPLEMENTED** — SQLite système, schéma v4 et migrations v1→v2→v3→v4, identité projet, transactions tâche+checkpoint, pagination de reprise et artefacts génériques. @@ -304,9 +370,19 @@ immuables de `import.images` et reconstruction production explicite. **IMPLEMENTED** — reprise automatique sélective, pagination de 8, ordre par ID, fenêtre de queue non bloquante et résumé consultable. -**NOT_YET_WIRED** — autosave à toutes les transitions, retry UI des sources -indisponibles et réconciliation des checkpoints orphelins, ScanSet -et catalogue image persistants, Feature Store et Visual Index. +**IMPLEMENTED** — ScanSets, images logiques, assets SHA-256 et pagination +bornée à 256. Les identités sont des `INTEGER PRIMARY KEY AUTOINCREMENT` SQLite +allouées sous transaction ; aucun `SELECT MAX()+1` n'est utilisé en +fonctionnement normal et une identité validée n'est jamais réutilisée. -**PLANNED** — migrations v4+, dépendances d'artefacts, graphe géométrique et +La migration v3→v4 ne lit pas `manifest.tsv`. Elle crée le ScanSet legacy et +positionne `metadata.legacy_image_catalog_pending=1` dès qu'une ancienne tâche +d'import existe. Cet indicateur signifie « données legacy potentiellement non +cataloguées », pas « images migrées ». + +**NOT_YET_WIRED** — autosave à toutes les transitions, retry UI des sources +indisponibles, migration de la TUI legacy et réconciliation des fichiers +orphelins, Feature Store et Visual Index. + +**PLANNED** — migrations v5+, dépendances d'artefacts, graphe géométrique et reconstruction incrémentale. diff --git a/docs/architecture/reconstruction_pipeline.md b/docs/architecture/reconstruction_pipeline.md index 2cbb7c8..e5f21d8 100644 --- a/docs/architecture/reconstruction_pipeline.md +++ b/docs/architecture/reconstruction_pipeline.md @@ -22,10 +22,10 @@ viewer, puis reprendre ultérieurement exactement où il s'était arrêté. |--------|-------------| | **Définition** | Un *scan set* (ou acquisition) regroupe un ensemble d'images capturées dans un contexte donné : même lieu, même session, même objectif de reconstruction. | | **Enrichissement progressif** | Un scan set peut être alimenté par vagues successives : images initiales, images de relèvement, images de contrôle. Chaque vague est horodatée et traçable. | -| **Identification stable** | Chaque scan set reçoit un identifiant UUID qui ne change jamais, même si le nom lisible est renommé. Les références internes utilisent cet UUID. | +| **Identification stable** | Chaque ScanSet reçoit un `scanset_id` SQLite positif qui ne dépend pas de son nom. | -**Statut :** PLANNED — le concept existe dans le manifeste projet mais n'est -pas encore structuré avec UUID et vague d'import. +**Statut :** IMPLEMENTED v1 — identité, nom et horodatages persistants. Les +vagues et relations géométriques entre acquisitions restent planifiées. --- @@ -33,14 +33,12 @@ pas encore structuré avec UUID et vague d'import. | Aspect | Description | |--------|-------------| -| **Identité stable** | Chaque image possède un identifiant interne stable (UUID), distinct du nom de fichier sur disque. Le fichier peut être renommé ou déplacé sans casser les références. | -| **Provenance** | Le catalogue enregistre le scan set d'origine, la date d'import, le chemin original et le chemin local. | -| **État de traitement** | Pour chaque image, le catalogue maintient un état : RAW, FEATURES_EXTRACTED, MATCHED, REGISTERED. Cet état est lu par le pipeline pour décider de l'étape suivante. | +| **Identité stable** | Chaque image possède un `image_id` stable, distinct du nom et de l'asset physique. | +| **Provenance** | Le catalogue enregistre le ScanSet, la date d'import, le nom et le chemin source, ainsi que la tâche productrice éventuelle. | +| **Asset** | Le contenu physique est identifié par `asset_id`, SHA-256, taille et chemin géré. Plusieurs images logiques peuvent partager cet asset. | -**Statut :** PARTIELLEMENT IMPLEMENTÉ — le `image_catalog` gère les -métadonnées de base mais pas encore l'état de traitement ni l'UUID. L'import -`import.images` est exécuté par le scheduler générique, copie au plus 32 images -par lot et reprend à partir du manifeste publié sans dupliquer une entrée. +**Statut :** IMPLEMENTED v1 — catalogue SQLite paginé et assets +content-addressed. Les états Feature/Matching/Reconstruction restent planifiés. --- @@ -64,8 +62,7 @@ par lot et reprend à partir du manifeste publié sans dupliquer une entrée. | **Pipeline conceptuel** | Extraction de features globales → construction de l'index → requête par similarité → retour des K plus proches voisins. | | **Proximité temporelle comme signal secondaire** | Lorsque les images portent un horodatage EXIF, la proximité temporelle sert de signal complémentaire au contenu visuel, mais ne remplace jamais l'analyse visuelle. | -**Statut :** PLANNED — le pipeline d'import利用 déjà la proximité temporelle -pour les paires candidats, mais aucun index visuel n'existe. +**Statut :** PLANNED — aucun index visuel ni générateur de paires n'existe. --- @@ -97,7 +94,7 @@ candidate generator n'existe pas encore. | Aspect | Description | |--------|-------------| | **Observation 2D → track → point3D** | Un *track* est une chaîne d'observations 2D cohérentes d'un même point 3D à travers plusieurs images. Chaque observation est un keypoint indexé par image. | -| **Lien avec le catalog** | Les tracks référencent les images par leur UUID interne, pas par nom de fichier. | +| **Lien avec le catalogue** | Les tracks référenceront les images par `image_id`, jamais par nom ou chemin. | | **Persistance** | Les tracks sont persistés entre les sessions de traitement. Un track ne peut être détruit que par une action explicite de l'utilisateur. | **Statut :** PLANNED — aucune implémentation existante. diff --git a/docs/architecture/resource_governor.md b/docs/architecture/resource_governor.md index 0dc053b..4d265d9 100644 --- a/docs/architecture/resource_governor.md +++ b/docs/architecture/resource_governor.md @@ -66,7 +66,8 @@ Le Resource Governor est l'unique propriétaire des budgets (RAM, GPU, CPU, IO). - Pas d'adaptation basée sur le débit (duration_ns non encore utilisé) - L'import `import.images` est admis avec 128 Kio fixes, un coût borné par item, un thread CPU, un slot I/O et des lots de 1 à 32. Il enregistre le nombre - d'images nouvellement publiées dans le manifeste et la durée réelle du lot. + d'images logiques nouvellement enregistrées dans le ScanSet et la durée + réelle du lot. Cela inclut une copie orpheline identique adoptée, même si aucun octet n'est recopié. `peak_memory_bytes == 0` signifie explicitement « mesure inconnue » : l'échantillon peut conserver taille/durée mais n'alimente jamais l'adaptation diff --git a/docs/architecture/runtime.md b/docs/architecture/runtime.md index 1a9d65c..f14add6 100644 --- a/docs/architecture/runtime.md +++ b/docs/architecture/runtime.md @@ -76,7 +76,9 @@ qu'à la terminaison de cette tentative. **IMPLEMENTED** — snapshot, codec v1 et restauration isolée. **IMPLEMENTED** — l'import `import.images` se sauvegarde à chaque fin de lot et -se reconstruit explicitement avec un userdata neuf lié au projet rouvert. +se reconstruit explicitement avec un userdata neuf lié au projet rouvert. Son +intention durable contient `source_path + scanset_id`; le hash/copie et la +transaction catalogue restent hors mutex Task et hors mutex DB pendant l'I/O. **IMPLEMENTED** — `project_open()` inventorie par pages de 8, restaure puis resoumet automatiquement les tâches production valides. Il retourne après diff --git a/docs/architecture/task_kind_registry.md b/docs/architecture/task_kind_registry.md index da2a343..5a80c9a 100644 --- a/docs/architecture/task_kind_registry.md +++ b/docs/architecture/task_kind_registry.md @@ -26,7 +26,7 @@ fin de l'exécution. Le constructeur métier n'est jamais appelé sous mutex DB. ## Persistance et legacy -Le checkpoint générique reste en version 1. Project Database v3 conserve le +Le checkpoint générique reste en version 1. Project Database v4 conserve le kind/version ; les lignes migrées depuis v1 restent `NULL/NULL` et sont classées `LEGACY_UNTYPED`. Un kind inconnu ou une version non supportée reste inspectable mais inexécutable. Aucun type n'est inventé et aucun code n'est sélectionné par diff --git a/docs/architecture/task_system.md b/docs/architecture/task_system.md index b93ea23..3d5a9a2 100644 --- a/docs/architecture/task_system.md +++ b/docs/architecture/task_system.md @@ -106,7 +106,7 @@ localement : son userdata est détruit, sans callback terminal ni écriture durable d'une fausse annulation. Une annulation explicitement demandée conserve le contrat de notification terminale. -La Project Database v3 peut enregistrer transactionnellement un résumé +La Project Database v4 peut enregistrer transactionnellement un résumé `Lardon3DTaskDurableSnapshot` et la référence de son checkpoint. Elle ne stocke ni estimation sérialisée complète, ni callback, ni réservation, et ne remplace pas la validation du fichier checkpoint avant `task_restore()`. diff --git a/docs/concepts/matching_and_tracks.md b/docs/concepts/matching_and_tracks.md index 2c3d0ff..abd83b1 100644 --- a/docs/concepts/matching_and_tracks.md +++ b/docs/concepts/matching_and_tracks.md @@ -28,6 +28,11 @@ Reconstruction Layers (triangulation) Le matching est le pont entre les caractéristiques 2D des images et la structure 3D de la scène. +La génération future de paires candidates combinera `image_id`, appartenance +au ScanSet, résultats du Visual Index et provenance. Une proximité temporelle +pourra servir de signal secondaire ; la proximité dans un dossier et le nom de +fichier ne constituent jamais l'identité principale. + ## Concepts clés ### Étapes du matching diff --git a/docs/concepts/scan_sets.md b/docs/concepts/scan_sets.md index 6ba8f9c..36e7cb6 100644 --- a/docs/concepts/scan_sets.md +++ b/docs/concepts/scan_sets.md @@ -1,91 +1,120 @@ -# Scan Sets & Acquisitions - -## Définition - -Un **scan set** (ou ensemble de scans) regroupe l'ensemble des images capturées lors d'une session d'acquisition unique, correspondant à un seul objet ou site reconstruit. Chaque scan set constitue l'unité atomique d'entrée du pipeline de photogrammétrie. - -Une **acquisition** désigne le processus physique de capture des images : positioning des capteurs, paramètres d'exposition, et conditions d'éclairage. Le scan set est le résultat numérique de cette acquisition. +# ScanSets et catalogue d'images ## Statut -**PLANNED** — Concept définissant la structure d'entrée, pas encore implémenté comme module distinct. Actuellement, le module `import` et `image_catalog` gèrent les images individuellement sans regroupement en scan sets. -Le chemin source durable de `import.images` ne constitue donc pas encore une -identité de ScanSet. +**IMPLEMENTED — ScanSet v1 et Image Catalog persistant v1.** -## Place dans le pipeline +Un projet contient zéro ou plusieurs acquisitions logiques appelées +`ScanSet`. Un ScanSet vide est valide : il peut être créé avant la capture ou +l'import. -``` -Acquisition physique - ↓ -Scan Set (import groupé) - ↓ -Image Catalog (indexation) - ↓ -Visual Index (indexation visuelle) - ↓ -Matching & Tracks - ↓ -Reconstruction Layers +## Modèle durable + +```text +Project + └── ScanSet (acquisition logique) + └── Image (observation logique) + └── Image Asset (contenu physique géré) ``` -Le scan set est la première structure organisée après l'import brut. Il fournit le contexte de regroupement nécessaire pour les étapes suivantes. +- `scanset_id` identifie durablement une acquisition, indépendamment de son + nom. +- `image_id` identifie durablement une observation dans un ScanSet, + indépendamment du nom du fichier et de la tâche d'import. +- `asset_id` identifie un contenu physique. Son SHA-256, sa taille et son + chemin relatif décrivent le fichier géré par Lardon3D. +- La provenance d'une image conserve son nom original, son chemin source, son + instant d'import et, s'il existe, le `task_id` importeur. -## Concepts clés +Un contenu identique n'implique pas une identité logique unique. Dans un même +ScanSet, le couple `(scanset_id, asset_id)` est unique : réimporter le même +contenu retourne `ALREADY_PRESENT`. Dans deux ScanSets distincts, deux images +logiques possèdent deux `image_id`, mais peuvent partager le même `asset_id` et +le même fichier physique. -### Structure d'un scan set +Un ScanSet peut documenter un objet complet, une sous-zone ou une pièce +démontée. Aucune pose ni transformation 3D entre ScanSets n'est inventée en v1. -| Champ | Type | Description | -|-------|------|-------------| -| `id` | `uint64_t` | Identifiant unique du scan set | -| `name` | `char[]` | Nom lisible (ex: "table_statue_01") | -| `project_ref` | `uint64_t` | Référence au projet parent | -| `image_count` | `uint32_t` | Nombre d'images dans le set | -| `acquisition_date` | `int64_t` | Timestamp de la session | -| `camera_model` | `char[]` | Modèle de caméra utilisé | -| `pixel_size_um` | `double` | Taille de pixel en micromètres | -| `focal_length_mm` | `double` | Distance focale nominale | +Ces trois identifiants utilisent `AUTOINCREMENT`. Un ID issu d'une transaction +validée ne sera jamais réattribué, même après une future suppression physique. +Cette garantie est nécessaire avant que Feature Store, Visual Index, matches et +tracks ne commencent à les référencer. -### Relations entre images +## Stockage physique -Au sein d'un scan set, les images entretiennent des relations spatiales : +Les assets image sont content-addressed : -- **Overlap horizontal** : chevauchement entre images adjacentes d'une même ligne de capture (typiquement 60-80%) -- **Overlap vertical** : chevauchement entre lignes de capture successives (typiquement 30-50%) -- **Baseline** : distance physique entre deux positions de capture consécutives -- **Convergence angle** : angle entre les axes optiques de deux caméras pour un même point +```text +assets/images/<2 premiers hex>/ +``` -### Types de scan sets +Le hash est un SHA-256 binaire de 32 octets dans SQLite et est encodé par le +programme pour construire le chemin. Le nom utilisateur ne participe jamais au +chemin de stockage. Le calcul et la copie utilisent un tampon fixe de 64 Kio ; +une image entière n'est jamais chargée en mémoire. -- **Grid scan** : captures organisées en grille régulière, adapté aux objets de taille moyenne -- **Orbital scan** : captures circulaires autour d'un objet, adapté à la sculpture et aux artefacts -- **Linear scan** : captures le long d'un axe linéaire, adapté aux façades et structures longues -- **Free-form scan** : captures sans contrainte géométrique, nécessite plus d'overlap pour compenser +La publication crée un temporaire, le synchronise, publie sans écrasement puis +synchronise le répertoire. Un asset existant n'est adopté qu'après vérification +complète de sa taille et de son SHA-256. La transaction SQLite vient ensuite. +Un échec SQLite peut donc laisser un fichier orphelin, mais jamais une ligne +`READY` créée par le chemin métier avant publication. -## Relations avec les autres modules +## Import et reprise -| Module | Relation | -|--------|----------| -| **Project** | Un scan set appartient à un projet. Le projet contient la structure de répertoires pour les images du set. | -| **Import** | L'import copie les images dans le projet. Le scan set représente un groupe logique d'images importées. | -| **Image Catalog** | Le catalogue indexe les métadonnées de chaque image du scan set. | -| **Image View** | Les vues peuvent filtrer ou trier les images par scan set. | -| **Resource Governor** | Le gouverneur estime les ressources nécessaires pour traiter un scan set complet. | -| **Task** | Le traitement d'un scan set est décomposé en tâches unitaires par le scheduler. | +`import.images` persiste désormais `source_path + scanset_id`. Le ScanSet est +immuable pour un `task_id`. L'import parcourt le dossier en streaming et traite +des lots bornés. Sa correction ne dépend pas de l'ordre de `readdir()` : la +présence logique est décidée par `(scanset_id, SHA-256)` dans la base. -## Contraintes de conception +Une source externe reste nécessaire tant que la tâche est récupérable. Après +`COMPLETED`, le catalogue et l'asset géré ne dépendent plus de sa présence. -- Un scan set ne doit jamais être modifié après le début du traitement (immutabilité partielle). -- L'ajout d'images à un scan set existant doit déclencher un recalcul incrémental, pas un retraitement complet. -- La taille maximale d'un scan set est bornée par la RAM disponible : pas plus de N images en mémoire simultanément. -- Chaque image appartient à exactement un scan set (relation 1:N). +Les anciennes tâches v3 sont rattachées par migration à un ScanSet explicite +nommé `Imports antérieurs à ScanSet v1`. Ce rattachement exprime seulement +l'absence historique de regroupement ; aucune provenance de capture n'est +inventée. -## Terminologie +La migration ne transforme pas les lignes de `manifest.tsv` en images v4 : elle +n'a ni hash ni transaction catalogue historique permettant de le faire sans +rejouer les fichiers. Elle pose donc l'indicateur durable +`legacy_image_catalog_pending=1` lorsqu'une ancienne tâche d'import existe. -| Terme | Définition | -|-------|------------| -| **Scan set** | Groupe d'images d'une même session d'acquisition | -| **Acquisition** | Processus physique de capture | -| **Overlap** | Pourcentage de superficie commune entre deux images | -| **Baseline** | Distance entre deux positions de capture | -| **Capture session** | Période continue d'acquisition d'images | -| **Footprint** | Zone physique couverte par une image au sol | +- Si une tâche v3 récupérable retrouve sa source, sa reprise relit les sources, + publie les assets content-addressed et remplit le catalogue. Les anciennes + copies sous `images/originals` ne sont ni écrasées ni supprimées. +- Si sa source a disparu, la reconstruction échoue proprement et les anciennes + données restent uniquement legacy ; aucune provenance ni image v4 n'est + inventée. +- Un import v3 déjà terminal n'est pas rejoué automatiquement. Ses images du + manifeste restent accessibles par la projection legacy mais sont marquées + conceptuellement **LEGACY DATA NOT YET CATALOGUED** via l'indicateur DB. + +L'indicateur reste conservateur en v1 et n'est pas effacé automatiquement : une +future commande de migration/reconciliation devra vérifier l'intégralité des +données historiques avant de le lever. + +## Accès borné + +Les listes de ScanSets et d'images utilisent un curseur par ID et une limite de +1 à 256. Aucun `get_all_images()` persistant n'existe. Des milliers d'images ne +nécessitent donc pas autant de records simultanément en mémoire. + +## Transition du manifeste + +`images/manifest.tsv` reste pris en charge par l'ancien catalogue en mémoire et +les anciennes API d'import. Le chemin production maintient une projection +best-effort par hardlinks pour que la TUI existante continue d'afficher les +nouvelles images sans seconde copie physique. `project.db + assets/images` +reste toutefois la vérité canonique : le manifeste est legacy et diagnostique, +et n'est plus une condition de reprise. Deux images de ScanSets différents qui +partagent un nom ne peuvent pas toutes deux être représentées dans cette vue +legacy. La migration de la TUI vers les pages SQLite reste donc nécessaire. + +## Futures étapes + +**NOT_YET_WIRED** — sélection de ScanSet dans la TUI, migration de l'ancienne +vue mémoire, vérification/scrub des assets et réconciliation globale des +orphelins. + +**PLANNED** — Feature Store, Visual Index, paires candidates, matching, tracks, +SfM, MVS et relations géométriques entre ScanSets. diff --git a/docs/concepts/visual_index.md b/docs/concepts/visual_index.md index f6289aa..65218f7 100644 --- a/docs/concepts/visual_index.md +++ b/docs/concepts/visual_index.md @@ -26,6 +26,11 @@ Matching & Tracks (appariement détaillé) L'index visuel se situe entre l'extraction de features et la sélection de paires candidates. Il réduit drastiquement l'espace de recherche pour l'appariement. +Le futur index référencera exclusivement les `image_id` stables du catalogue +persistant. Un chemin d'asset ou un nom source n'est ni une identité d'image ni +une clé d'index durable. Le partage d'un asset entre deux ScanSets n'empêche pas +leurs deux images logiques d'être des observations distinctes. + ## Concepts clés ### Descripteurs visuels diff --git a/docs/roadmap/roadmap.md b/docs/roadmap/roadmap.md index 98bba67..a832ae0 100644 --- a/docs/roadmap/roadmap.md +++ b/docs/roadmap/roadmap.md @@ -29,12 +29,12 @@ Lardon3D suit une feuille de route ordonnée qui privilégie la stabilité et la ### Phase 3 : Persistance - ✅ Fondation versionnée des checkpoints de tâches -- ✅ Project Database v3 (paramètres d'import et allocation durable des task IDs) +- ✅ Project Database v4 (tâches, ScanSets, images, assets et migrations) - ✅ Branchement Project Database au cycle de vie projet et inventaire de reprise - ✅ Registry durable des types métier de tâches - ✅ Premier type métier reconstructible (`import.images`) - ✅ Resoumission automatique contrôlée et bornée des tâches récupérables -- 📋 ScanSet et Image Catalog persistants +- ✅ ScanSet v1 et Image Catalog persistant v1 ### Phase 4 : Pipeline avancé - 📋 Feature Store diff --git a/include/lardon3d/image_catalog.h b/include/lardon3d/image_catalog.h index 021bf88..be95416 100644 --- a/include/lardon3d/image_catalog.h +++ b/include/lardon3d/image_catalog.h @@ -5,6 +5,31 @@ #include #include +#include + +typedef enum { + LARDON3D_IMAGE_CATALOG_IMPORTED = 0, + LARDON3D_IMAGE_CATALOG_ALREADY_PRESENT, + LARDON3D_IMAGE_CATALOG_INVALID_ARGUMENT, + LARDON3D_IMAGE_CATALOG_SOURCE_ERROR, + LARDON3D_IMAGE_CATALOG_PUBLICATION_ERROR, + LARDON3D_IMAGE_CATALOG_DB_ERROR +} Lardon3DImageCatalogImportResult; + +bool lardon3d_image_catalog_create_scanset( + Lardon3DAppState *state, const char *name, + Lardon3DProjectDbScanSet *scanset +); +Lardon3DImageCatalogImportResult lardon3d_image_catalog_import_file( + Lardon3DAppState *state, uint64_t scanset_id, const char *source_path, + uint64_t producer_task_id, Lardon3DProjectDbImage *image, + Lardon3DProjectDbImageAsset *asset +); +Lardon3DProjectDbResult lardon3d_image_catalog_list( + Lardon3DAppState *state, uint64_t scanset_id, uint64_t after_image_id, + Lardon3DProjectDbImage *images, Lardon3DProjectDbImageAsset *assets, + size_t capacity, size_t *count +); typedef struct { char *filename; diff --git a/include/lardon3d/import.h b/include/lardon3d/import.h index a48ed19..d99586f 100644 --- a/include/lardon3d/import.h +++ b/include/lardon3d/import.h @@ -64,4 +64,15 @@ Lardon3DImportOutcome lardon3d_import_directory_batch( bool *complete ); +Lardon3DImportOutcome lardon3d_import_directory_batch_to_scanset( + Lardon3DAppState *state, + uint64_t scanset_id, + uint64_t producer_task_id, + const char *source_directory, + size_t batch_size, + Lardon3DImportResult *result, + const Lardon3DImportControl *control, + bool *complete +); + #endif diff --git a/include/lardon3d/import_task.h b/include/lardon3d/import_task.h index 3678cca..4b35b4f 100644 --- a/include/lardon3d/import_task.h +++ b/include/lardon3d/import_task.h @@ -39,11 +39,13 @@ typedef struct { Lardon3DTask *lardon3d_project_create_image_import_task( Lardon3DAppState *state, + uint64_t scanset_id, const char *source_directory, uint64_t *task_id ); bool lardon3d_project_enqueue_image_import( Lardon3DAppState *state, + uint64_t scanset_id, const char *source_directory, uint64_t *task_id ); diff --git a/include/lardon3d/project.h b/include/lardon3d/project.h index 779a16a..d0f6a4e 100644 --- a/include/lardon3d/project.h +++ b/include/lardon3d/project.h @@ -67,7 +67,8 @@ Lardon3DProjectTaskCheckpointResult lardon3d_project_checkpoint_image_import_task( Lardon3DAppState *state, const Lardon3DTask *task, - const char *source_path + const char *source_path, + uint64_t scanset_id ); Lardon3DProjectDbResult lardon3d_project_list_recoverable( Lardon3DAppState *state, diff --git a/include/lardon3d/project_db.h b/include/lardon3d/project_db.h index 29d0a17..af3582e 100644 --- a/include/lardon3d/project_db.h +++ b/include/lardon3d/project_db.h @@ -9,12 +9,16 @@ #include enum { - LARDON3D_PROJECT_DB_SCHEMA_VERSION = 3, + LARDON3D_PROJECT_DB_SCHEMA_VERSION = 4, LARDON3D_PROJECT_DB_ID_CAPACITY = 65, LARDON3D_PROJECT_DB_KIND_CAPACITY = 65, LARDON3D_PROJECT_DB_PATH_CAPACITY = 4096, LARDON3D_PROJECT_DB_ERROR_CAPACITY = 256, LARDON3D_PROJECT_DB_RECOVERY_PAGE_MAX = 256, + LARDON3D_PROJECT_DB_CATALOG_PAGE_MAX = 256, + LARDON3D_PROJECT_DB_SCANSET_NAME_CAPACITY = 256, + LARDON3D_PROJECT_DB_IMAGE_NAME_CAPACITY = 256, + LARDON3D_PROJECT_DB_SHA256_SIZE = 32, }; typedef struct Lardon3DProjectDb Lardon3DProjectDb; @@ -86,8 +90,45 @@ typedef struct { typedef struct { uint64_t task_id; char source_path[LARDON3D_PROJECT_DB_PATH_CAPACITY]; + uint64_t scanset_id; } Lardon3DProjectDbImageImport; +typedef struct { + uint64_t scanset_id; + char name[LARDON3D_PROJECT_DB_SCANSET_NAME_CAPACITY]; + int64_t created_at; + int64_t updated_at; +} Lardon3DProjectDbScanSet; + +typedef enum { + LARDON3D_DB_IMAGE_ASSET_READY = 1 +} Lardon3DProjectDbImageAssetState; + +typedef struct { + uint64_t asset_id; + unsigned char sha256[LARDON3D_PROJECT_DB_SHA256_SIZE]; + char path[LARDON3D_PROJECT_DB_PATH_CAPACITY]; + uint64_t size_bytes; + Lardon3DProjectDbImageAssetState state; + int64_t created_at; +} Lardon3DProjectDbImageAsset; + +typedef struct { + uint64_t image_id; + uint64_t scanset_id; + uint64_t asset_id; + char original_name[LARDON3D_PROJECT_DB_IMAGE_NAME_CAPACITY]; + char source_path[LARDON3D_PROJECT_DB_PATH_CAPACITY]; + bool has_producer_task; + uint64_t producer_task_id; + int64_t imported_at; +} Lardon3DProjectDbImage; + +typedef enum { + LARDON3D_PROJECT_DB_IMAGE_REGISTERED = 0, + LARDON3D_PROJECT_DB_IMAGE_ALREADY_PRESENT +} Lardon3DProjectDbImageRegisterStatus; + Lardon3DProjectDbResult lardon3d_project_db_open( const char *path, Lardon3DProjectDb **database, @@ -98,6 +139,9 @@ bool lardon3d_project_db_last_error( Lardon3DProjectDb *database, char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY] ); +Lardon3DProjectDbResult lardon3d_project_db_legacy_catalog_pending( + Lardon3DProjectDb *database, bool *pending +); unsigned int lardon3d_project_db_schema_version(Lardon3DProjectDb *database); Lardon3DProjectDbResult lardon3d_project_db_set_project( @@ -123,8 +167,41 @@ Lardon3DProjectDbResult lardon3d_project_db_record_image_import_task( uint32_t task_kind_version, const Lardon3DProjectDbCheckpoint *checkpoint, const char *source_path, + uint64_t scanset_id, int64_t updated_at ); +Lardon3DProjectDbResult lardon3d_project_db_create_scanset( + Lardon3DProjectDb *database, const char *name, + Lardon3DProjectDbScanSet *scanset +); +Lardon3DProjectDbResult lardon3d_project_db_load_scanset( + Lardon3DProjectDb *database, uint64_t scanset_id, + Lardon3DProjectDbScanSet *scanset +); +Lardon3DProjectDbResult lardon3d_project_db_list_scansets( + Lardon3DProjectDb *database, uint64_t after_scanset_id, + Lardon3DProjectDbScanSet *scansets, size_t capacity, size_t *count +); +Lardon3DProjectDbResult lardon3d_project_db_register_image( + Lardon3DProjectDb *database, uint64_t scanset_id, + const unsigned char sha256[LARDON3D_PROJECT_DB_SHA256_SIZE], + const char *asset_path, uint64_t size_bytes, const char *original_name, + const char *source_path, uint64_t producer_task_id, int64_t imported_at, + Lardon3DProjectDbImageRegisterStatus *status, + Lardon3DProjectDbImage *image +); +Lardon3DProjectDbResult lardon3d_project_db_load_image( + Lardon3DProjectDb *database, uint64_t image_id, + Lardon3DProjectDbImage *image, Lardon3DProjectDbImageAsset *asset +); +Lardon3DProjectDbResult lardon3d_project_db_list_images( + Lardon3DProjectDb *database, uint64_t scanset_id, uint64_t after_image_id, + Lardon3DProjectDbImage *images, Lardon3DProjectDbImageAsset *assets, + size_t capacity, size_t *count +); +Lardon3DProjectDbResult lardon3d_project_db_count_images( + Lardon3DProjectDb *database, uint64_t scanset_id, uint64_t *count +); Lardon3DProjectDbResult lardon3d_project_db_load_image_import( Lardon3DProjectDb *database, uint64_t task_id, diff --git a/meson.build b/meson.build index de6426d..e0fec0e 100644 --- a/meson.build +++ b/meson.build @@ -21,6 +21,7 @@ add_project_arguments( ncursesw = dependency('ncursesw', required: true) threads = dependency('threads') sqlite3 = dependency('sqlite3', required: true) +openssl = dependency('openssl', required: true) executable( 'lardon3d', @@ -33,6 +34,7 @@ executable( 'src/import.c', 'src/import_task.c', 'src/image_catalog.c', + 'src/image_catalog_persistent.c', 'src/image_view.c', 'src/project.c', 'src/project_db.c', @@ -46,7 +48,7 @@ executable( 'src/hardware_profile.c', ], include_directories: include_directories('include'), - dependencies: [ncursesw, threads, sqlite3], + dependencies: [ncursesw, threads, sqlite3, openssl], ) import_test = executable( @@ -55,8 +57,14 @@ import_test = executable( 'tests/test_import.c', 'src/app_state.c', 'src/import.c', + 'src/image_catalog_persistent.c', + 'src/project_db.c', + 'src/task.c', + 'src/resource_governor.c', + 'src/resource_snapshot.c', ], include_directories: include_directories('include'), + dependencies: [threads, sqlite3, openssl], ) test('import', import_test) @@ -78,6 +86,7 @@ import_task_test = executable( 'src/resource_governor.c', 'src/resource_snapshot.c', 'src/image_catalog.c', + 'src/image_catalog_persistent.c', 'src/image_view.c', ], c_args: [ @@ -86,7 +95,7 @@ import_task_test = executable( '-DLARDON3D_CHECKPOINT_TESTING', ], include_directories: include_directories('include'), - dependencies: [threads, sqlite3], + dependencies: [threads, sqlite3, openssl], ) test('import-task', import_task_test, timeout: 30) @@ -97,12 +106,36 @@ image_catalog_test = executable( 'tests/test_image_catalog.c', 'src/app_state.c', 'src/image_catalog.c', + 'src/image_catalog_persistent.c', + 'src/project_db.c', + 'src/task.c', + 'src/resource_governor.c', + 'src/resource_snapshot.c', ], include_directories: include_directories('include'), + dependencies: [threads, sqlite3, openssl], ) test('image-catalog', image_catalog_test, timeout: 30) +persistent_image_catalog_test = executable( + 'test-persistent-image-catalog', + sources: [ + 'tests/test_persistent_image_catalog.c', + 'src/app_state.c', + 'src/image_catalog_persistent.c', + 'src/project_db.c', + 'src/task.c', + 'src/resource_governor.c', + 'src/resource_snapshot.c', + ], + c_args: ['-DLARDON3D_PROJECT_DB_TESTING'], + include_directories: include_directories('include'), + dependencies: [threads, sqlite3, openssl], +) + +test('persistent-image-catalog', persistent_image_catalog_test, timeout: 60) + image_view_test = executable( 'test-image-view', sources: [ @@ -157,7 +190,7 @@ project_db_test = executable( ], c_args: ['-DLARDON3D_PROJECT_DB_TESTING'], include_directories: include_directories('include'), - dependencies: [threads, sqlite3], + dependencies: [threads, sqlite3, openssl], ) test('project-db', project_db_test, timeout: 30) @@ -177,6 +210,7 @@ project_test = executable( 'src/import.c', 'src/import_task.c', 'src/image_catalog.c', + 'src/image_catalog_persistent.c', 'src/image_view.c', 'src/resource_governor.c', 'src/resource_snapshot.c', @@ -186,7 +220,7 @@ project_test = executable( '-DLARDON3D_CHECKPOINT_TESTING', ], include_directories: include_directories('include'), - dependencies: [threads, sqlite3], + dependencies: [threads, sqlite3, openssl], ) test('project', project_test, timeout: 30) diff --git a/src/image_catalog_persistent.c b/src/image_catalog_persistent.c new file mode 100644 index 0000000..a1c1b42 --- /dev/null +++ b/src/image_catalog_persistent.c @@ -0,0 +1,219 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +enum { COPY_BUFFER_SIZE = 64 * 1024 }; + +static bool +join_path(char output[PATH_MAX], const char *parent, const char *child) +{ + int written = snprintf(output, PATH_MAX, "%s/%s", parent, child); + return written > 0 && (size_t)written < PATH_MAX; +} + +static bool +ensure_directory(const char *path) +{ + struct stat information; + if (lstat(path, &information) == 0) + return S_ISDIR(information.st_mode) && !S_ISLNK(information.st_mode); + if (errno != ENOENT) return false; + if (mkdir(path, 0755) == 0) return true; + return errno == EEXIST && lstat(path, &information) == 0 + && S_ISDIR(information.st_mode) && !S_ISLNK(information.st_mode); +} + +static bool +write_all(int descriptor, const unsigned char *data, size_t size) +{ + size_t offset = 0; + while (offset < size) { + ssize_t written = write(descriptor, data + offset, size - offset); + if (written < 0 && errno == EINTR) continue; + if (written <= 0) return false; + offset += (size_t)written; + } + return true; +} + +static bool +hash_stream(int input, int output, + unsigned char hash[LARDON3D_PROJECT_DB_SHA256_SIZE], uint64_t *size) +{ + EVP_MD_CTX *context = EVP_MD_CTX_new(); + if (!context) return false; + bool success = EVP_DigestInit_ex(context, EVP_sha256(), NULL) == 1; + unsigned char buffer[COPY_BUFFER_SIZE]; + uint64_t total = 0; + while (success) { + ssize_t count = read(input, buffer, sizeof(buffer)); + if (count < 0 && errno == EINTR) continue; + if (count < 0) { success = false; break; } + if (count == 0) break; + size_t bytes = (size_t)count; + if (total > (uint64_t)INT64_MAX - bytes + || EVP_DigestUpdate(context, buffer, bytes) != 1 + || (output >= 0 && !write_all(output, buffer, bytes))) { + success = false; break; + } + total += bytes; + } + unsigned int hash_size = 0; + if (!success || EVP_DigestFinal_ex(context, hash, &hash_size) != 1 + || hash_size != LARDON3D_PROJECT_DB_SHA256_SIZE) success = false; + EVP_MD_CTX_free(context); + if (success) *size = total; + return success; +} + +static void +hash_hex(const unsigned char hash[LARDON3D_PROJECT_DB_SHA256_SIZE], + char text[65]) +{ + static const char digits[] = "0123456789abcdef"; + for (size_t index = 0; index < LARDON3D_PROJECT_DB_SHA256_SIZE; ++index) { + text[index * 2] = digits[hash[index] >> 4]; + text[index * 2 + 1] = digits[hash[index] & 15U]; + } + text[64] = '\0'; +} + +static bool +same_published_asset(const char *path, + const unsigned char expected[LARDON3D_PROJECT_DB_SHA256_SIZE], + uint64_t expected_size) +{ + int descriptor = open(path, O_RDONLY | O_NOFOLLOW); + if (descriptor < 0) return false; + struct stat information; + unsigned char actual[LARDON3D_PROJECT_DB_SHA256_SIZE]; + uint64_t size = 0; + bool success = fstat(descriptor, &information) == 0 + && S_ISREG(information.st_mode) + && hash_stream(descriptor, -1, actual, &size) + && size == expected_size + && memcmp(actual, expected, sizeof(actual)) == 0; + (void)close(descriptor); + return success; +} + +static bool +sync_directory(const char *path) +{ + int descriptor = open(path, O_RDONLY | O_DIRECTORY); + if (descriptor < 0) return false; + bool success = fsync(descriptor) == 0; + if (close(descriptor) != 0) success = false; + return success; +} + +bool +lardon3d_image_catalog_create_scanset(Lardon3DAppState *state, + const char *name, Lardon3DProjectDbScanSet *scanset) +{ + return state && state->project_loaded && state->project_db + && lardon3d_project_db_create_scanset(state->project_db, name, scanset) + == LARDON3D_PROJECT_DB_OK; +} + +Lardon3DImageCatalogImportResult +lardon3d_image_catalog_import_file(Lardon3DAppState *state, + uint64_t scanset_id, const char *source_path, uint64_t producer_task_id, + Lardon3DProjectDbImage *image, Lardon3DProjectDbImageAsset *asset) +{ + if (!state || !state->project_loaded || !state->project_db + || scanset_id == 0 || !source_path || !source_path[0] || !image || !asset) + return LARDON3D_IMAGE_CATALOG_INVALID_ARGUMENT; + const char *basename = strrchr(source_path, '/'); + basename = basename ? basename + 1 : source_path; + if (!basename[0] || strchr(basename, '\\') || strchr(basename, '\t') + || strchr(basename, '\r') || strchr(basename, '\n') + || strnlen(basename, LARDON3D_PROJECT_DB_IMAGE_NAME_CAPACITY) + >= LARDON3D_PROJECT_DB_IMAGE_NAME_CAPACITY) + return LARDON3D_IMAGE_CATALOG_INVALID_ARGUMENT; + Lardon3DProjectDbScanSet scanset; + if (lardon3d_project_db_load_scanset(state->project_db, scanset_id, &scanset) + != LARDON3D_PROJECT_DB_OK) return LARDON3D_IMAGE_CATALOG_INVALID_ARGUMENT; + int input = open(source_path, O_RDONLY | O_NOFOLLOW); + if (input < 0) return LARDON3D_IMAGE_CATALOG_SOURCE_ERROR; + struct stat source_information; + if (fstat(input, &source_information) != 0 + || !S_ISREG(source_information.st_mode) || source_information.st_size < 0) { + (void)close(input); return LARDON3D_IMAGE_CATALOG_SOURCE_ERROR; + } + char assets[PATH_MAX], images[PATH_MAX], temporary[PATH_MAX]; + if (!join_path(assets, state->project_path, "assets") + || !ensure_directory(assets) || !sync_directory(state->project_path) + || !join_path(images, assets, "images") || !ensure_directory(images) + || !sync_directory(assets) + || snprintf(temporary, sizeof(temporary), "%s/.asset.tmp.XXXXXX", images) <= 0) { + (void)close(input); return LARDON3D_IMAGE_CATALOG_PUBLICATION_ERROR; + } + int output = mkstemp(temporary); + if (output < 0) { (void)close(input); return LARDON3D_IMAGE_CATALOG_PUBLICATION_ERROR; } + unsigned char hash[LARDON3D_PROJECT_DB_SHA256_SIZE]; + uint64_t size = 0; + bool copied = hash_stream(input, output, hash, &size); + if (close(input) != 0) copied = false; + if (copied && fsync(output) != 0) copied = false; + if (close(output) != 0) copied = false; + if (!copied) { + (void)unlink(temporary); return LARDON3D_IMAGE_CATALOG_SOURCE_ERROR; + } + char hex[65], prefix[3], prefix_path[PATH_MAX], final[PATH_MAX]; + hash_hex(hash, hex); prefix[0] = hex[0]; prefix[1] = hex[1]; prefix[2] = '\0'; + if (!join_path(prefix_path, images, prefix) || !ensure_directory(prefix_path) + || !sync_directory(images) + || !join_path(final, prefix_path, hex)) { + (void)unlink(temporary); return LARDON3D_IMAGE_CATALOG_PUBLICATION_ERROR; + } + bool published = false; + if (link(temporary, final) == 0) { + bool temporary_removed = unlink(temporary) == 0; + bool directory_synced = sync_directory(prefix_path); + published = temporary_removed && directory_synced; + } else if (errno == EEXIST && same_published_asset(final, hash, size)) { + published = unlink(temporary) == 0; + } + if (!published) { + (void)unlink(temporary); return LARDON3D_IMAGE_CATALOG_PUBLICATION_ERROR; + } + char relative[LARDON3D_PROJECT_DB_PATH_CAPACITY]; + int relative_size = snprintf(relative, sizeof(relative), + "assets/images/%s/%s", prefix, hex); + if (relative_size <= 0 || (size_t)relative_size >= sizeof(relative)) + return LARDON3D_IMAGE_CATALOG_PUBLICATION_ERROR; + Lardon3DProjectDbImageRegisterStatus status; + int64_t imported_at = (int64_t)time(NULL); + if (imported_at < 0 || lardon3d_project_db_register_image(state->project_db, + scanset_id, hash, relative, size, basename, source_path, + producer_task_id, imported_at, &status, image) + != LARDON3D_PROJECT_DB_OK) return LARDON3D_IMAGE_CATALOG_DB_ERROR; + if (lardon3d_project_db_load_image(state->project_db, image->image_id, + image, asset) != LARDON3D_PROJECT_DB_OK) + return LARDON3D_IMAGE_CATALOG_DB_ERROR; + return status == LARDON3D_PROJECT_DB_IMAGE_ALREADY_PRESENT + ? LARDON3D_IMAGE_CATALOG_ALREADY_PRESENT + : LARDON3D_IMAGE_CATALOG_IMPORTED; +} + +Lardon3DProjectDbResult +lardon3d_image_catalog_list(Lardon3DAppState *state, uint64_t scanset_id, + uint64_t after_image_id, Lardon3DProjectDbImage *images, + Lardon3DProjectDbImageAsset *assets, size_t capacity, size_t *count) +{ + if (!state || !state->project_loaded || !state->project_db) + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + return lardon3d_project_db_list_images(state->project_db, scanset_id, + after_image_id, images, assets, capacity, count); +} diff --git a/src/import.c b/src/import.c index 82b4959..8a8f626 100644 --- a/src/import.c +++ b/src/import.c @@ -14,6 +14,7 @@ #include #include +#include enum { MANIFEST_LINE_CAPACITY = PATH_MAX + 512, @@ -1162,3 +1163,106 @@ lardon3d_import_directory( NULL ) == LARDON3D_IMPORT_SUCCEEDED; } + +Lardon3DImportOutcome +lardon3d_import_directory_batch_to_scanset( + Lardon3DAppState *state, + uint64_t scanset_id, + uint64_t producer_task_id, + const char *source_directory, + size_t batch_size, + Lardon3DImportResult *result, + const Lardon3DImportControl *control, + bool *complete +) +{ + if (!state || !state->project_loaded || !state->project_db || !result + || !complete || scanset_id == 0 || producer_task_id == 0 + || batch_size == 0) return LARDON3D_IMPORT_FAILED; + *result = (Lardon3DImportResult){0}; + *complete = false; + char trimmed[PATH_MAX], source[PATH_MAX]; + if (!trim_source_path(state, source_directory, trimmed) + || !resolve_source_path(state, trimmed, source)) return LARDON3D_IMPORT_FAILED; + struct stat directory_info; + if (lstat(source, &directory_info) != 0 || !S_ISDIR(directory_info.st_mode) + || S_ISLNK(directory_info.st_mode)) { + set_status(state, "Erreur : dossier source absent ou invalide."); + return LARDON3D_IMPORT_FAILED; + } + DIR *directory = opendir(source); + if (!directory) return LARDON3D_IMPORT_FAILED; + bool success = true, cancelled = false, remaining = false; + size_t registered = 0; + ManifestWriter legacy_manifest = {0}; + bool legacy_manifest_active = false; + char legacy_images[PATH_MAX], legacy_originals[PATH_MAX]; + bool legacy_paths = ensure_originals_directory(state, legacy_images, + legacy_originals); + for (;;) { + if (import_is_cancelled(control)) { cancelled = true; break; } + errno = 0; + struct dirent *entry = readdir(directory); + if (!entry) { if (errno) success = false; break; } + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) + continue; + char path[PATH_MAX]; struct stat information; + if (!join_path(path, source, entry->d_name)) { success = false; break; } + if (lstat(path, &information) != 0 || !S_ISREG(information.st_mode) + || S_ISLNK(information.st_mode) || !has_supported_extension(entry->d_name)) { + ++result->ignored; continue; + } + ++result->admissible_found; + if (registered >= batch_size) { remaining = true; continue; } + Lardon3DProjectDbImage image; + Lardon3DProjectDbImageAsset asset; + Lardon3DImageCatalogImportResult imported = + lardon3d_image_catalog_import_file(state, scanset_id, path, + producer_task_id, &image, &asset); + if (imported == LARDON3D_IMAGE_CATALOG_ALREADY_PRESENT) { + ++result->already_present; + ++result->processed; + } else if (imported == LARDON3D_IMAGE_CATALOG_IMPORTED) { + ++registered; + ++result->newly_manifested; + ++result->copied; + ++result->processed; + if (legacy_paths && (!legacy_manifest_active + ? manifest_begin(state, legacy_images, &legacy_manifest) + : true)) { + legacy_manifest_active = true; + int listed = manifest_contains(&legacy_manifest, entry->d_name); + char asset_path[PATH_MAX], legacy_path[PATH_MAX]; + if (listed == 0 + && join_path(asset_path, state->project_path, asset.path) + && join_path(legacy_path, legacy_originals, entry->d_name) + && link(asset_path, legacy_path) == 0 + && !manifest_append(&legacy_manifest, entry->d_name, + (off_t)asset.size_bytes, path)) { + (void)unlink(legacy_path); + manifest_abort(&legacy_manifest); + legacy_manifest_active = false; + legacy_paths = false; + } + } + } else { + success = false; break; + } + } + if (closedir(directory) != 0) success = false; + if (legacy_manifest_active && !manifest_commit(state, &legacy_manifest)) { + legacy_manifest_active = false; + } + if (cancelled) { + set_status(state, "Import annulé à une frontière sûre."); + return LARDON3D_IMPORT_CANCELLED; + } + if (!success) { + set_status(state, "Erreur pendant un lot d'import."); + return LARDON3D_IMPORT_FAILED; + } + *complete = !remaining; + set_status(state, *complete ? "Import terminé." : "Lot d'import publié."); + publish_progress(control, result, result->processed, state->status_message); + return LARDON3D_IMPORT_SUCCEEDED; +} diff --git a/src/import_task.c b/src/import_task.c index f4a297a..799812b 100644 --- a/src/import_task.c +++ b/src/import_task.c @@ -10,6 +10,7 @@ #include #include +#include #include #include @@ -23,6 +24,7 @@ enum { typedef struct { char project_path[PATH_MAX]; char source_path[PATH_MAX]; + uint64_t scanset_id; Lardon3DProjectDb *project_db; Lardon3DResourceGovernor *governor; } Lardon3DImageImportContext; @@ -124,9 +126,10 @@ run_image_import(Lardon3DTask *task, void *userdata) bool complete = false; struct timespec begin, end; (void)clock_gettime(CLOCK_MONOTONIC, &begin); - Lardon3DImportOutcome outcome = lardon3d_import_directory_batch( - &state, context->source_path, contract.batch_size, &result, - &control, &complete); + Lardon3DImportOutcome outcome = lardon3d_import_directory_batch_to_scanset( + &state, context->scanset_id, lardon3d_task_id(task), + context->source_path, contract.batch_size, &result, &control, + &complete); (void)clock_gettime(CLOCK_MONOTONIC, &end); if (outcome == LARDON3D_IMPORT_CANCELLED) return false; if (outcome != LARDON3D_IMPORT_SUCCEEDED) { @@ -143,7 +146,7 @@ run_image_import(Lardon3DTask *task, void *userdata) result.newly_manifested, elapsed_ns(begin, end), 0); if (lardon3d_project_checkpoint_image_import_task( - &state, task, context->source_path) + &state, task, context->source_path, context->scanset_id) != LARDON3D_PROJECT_TASK_CHECKPOINT_OK) { return lardon3d_task_fail(task, "Checkpoint import impossible."); } @@ -162,9 +165,10 @@ run_image_import(Lardon3DTask *task, void *userdata) static Lardon3DImageImportContext * create_context(const char *project_path, const char *source_path, - Lardon3DProjectDb *database, Lardon3DResourceGovernor *governor) + uint64_t scanset_id, Lardon3DProjectDb *database, + Lardon3DResourceGovernor *governor) { - if (!project_path || !source_path || !database || !governor) return NULL; + if (!project_path || !source_path || scanset_id == 0 || !database || !governor) return NULL; Lardon3DImageImportContext *context = calloc(1, sizeof(*context)); if (!context) return NULL; int project_written = snprintf(context->project_path, @@ -177,6 +181,7 @@ create_context(const char *project_path, const char *source_path, } context->project_db = database; context->governor = governor; + context->scanset_id = scanset_id; return context; } @@ -196,7 +201,8 @@ lardon3d_image_import_reconstruct( char source[PATH_MAX]; if (!canonical_source(parameters.source_path, source)) return false; Lardon3DImageImportContext *context = create_context(runtime->project_path, - source, runtime->project_db, runtime->resource_governor); + source, parameters.scanset_id, runtime->project_db, + runtime->resource_governor); if (!context) return false; *binding = (Lardon3DTaskKindBinding) { .callback = run_image_import, @@ -211,20 +217,24 @@ lardon3d_image_import_reconstruct( Lardon3DTask * lardon3d_project_create_image_import_task( Lardon3DAppState *state, + uint64_t scanset_id, const char *source_directory, uint64_t *task_id ) { if (task_id) *task_id = 0; if (!state || !state->project_loaded || !state->project_db - || !state->resource_governor || !task_id) return NULL; + || !state->resource_governor || !task_id || scanset_id == 0) return NULL; + Lardon3DProjectDbScanSet scanset; + if (lardon3d_project_db_load_scanset(state->project_db, scanset_id, &scanset) + != LARDON3D_PROJECT_DB_OK) return NULL; char source[PATH_MAX]; if (!canonical_source(source_directory, source)) return NULL; uint64_t id = 0; if (lardon3d_project_db_allocate_task_id(state->project_db, &id) != LARDON3D_PROJECT_DB_OK) return NULL; Lardon3DImageImportContext *context = create_context(state->project_path, - source, state->project_db, state->resource_governor); + source, scanset_id, state->project_db, state->resource_governor); if (!context) return NULL; const Lardon3DResourceEstimate estimate = { .memory_fixed_bytes = IMAGE_IMPORT_FIXED_MEMORY, @@ -241,7 +251,8 @@ lardon3d_project_create_image_import_task( destroy_context); if (!task || !lardon3d_task_assign_id(task, id) || !lardon3d_task_set_finished_callback(task, import_finished, context) - || lardon3d_project_checkpoint_image_import_task(state, task, source) + || lardon3d_project_checkpoint_image_import_task(state, task, source, + scanset_id) != LARDON3D_PROJECT_TASK_CHECKPOINT_OK) { lardon3d_task_destroy(task); return NULL; @@ -252,11 +263,11 @@ lardon3d_project_create_image_import_task( bool lardon3d_project_enqueue_image_import(Lardon3DAppState *state, - const char *source_directory, uint64_t *task_id) + uint64_t scanset_id, const char *source_directory, uint64_t *task_id) { if (!state || !state->task_queue) return false; Lardon3DTask *task = lardon3d_project_create_image_import_task( - state, source_directory, task_id); + state, scanset_id, source_directory, task_id); if (!task) return false; if (!lardon3d_task_queue_add(state->task_queue, task, NULL)) { lardon3d_task_destroy(task); @@ -273,8 +284,11 @@ lardon3d_import_task_start(Lardon3DImportTask *task, Lardon3DAppState *state, { if (!task || task->task_id != 0 || !state) return false; task->state = state; - return lardon3d_project_enqueue_image_import(state, source_directory, - &task->task_id); + Lardon3DProjectDbScanSet scanset; + if (!lardon3d_image_catalog_create_scanset(state, "Import d'images", + &scanset)) return false; + return lardon3d_project_enqueue_image_import(state, scanset.scanset_id, + source_directory, &task->task_id); } void lardon3d_import_task_request_cancel(Lardon3DImportTask *task) diff --git a/src/project.c b/src/project.c index 94e102a..800ebc8 100644 --- a/src/project.c +++ b/src/project.c @@ -880,7 +880,8 @@ static Lardon3DProjectTaskCheckpointResult checkpoint_task_internal( Lardon3DAppState *state, const Lardon3DTask *task, - const char *image_import_source + const char *image_import_source, + uint64_t image_import_scanset_id ) { if (!state || !state->project_loaded || !state->project_db) { @@ -926,7 +927,8 @@ checkpoint_task_internal( Lardon3DProjectDbResult recorded = image_import_source ? lardon3d_project_db_record_image_import_task( state->project_db, &snapshot, task_kind, task_kind_version, - &checkpoint, image_import_source, now.tv_sec) + &checkpoint, image_import_source, image_import_scanset_id, + now.tv_sec) : lardon3d_project_db_record_task( state->project_db, &snapshot, task_kind, task_kind_version, &checkpoint, now.tv_sec); @@ -947,20 +949,21 @@ lardon3d_project_checkpoint_task( const Lardon3DTask *task ) { - return checkpoint_task_internal(state, task, NULL); + return checkpoint_task_internal(state, task, NULL, 0); } Lardon3DProjectTaskCheckpointResult lardon3d_project_checkpoint_image_import_task( Lardon3DAppState *state, const Lardon3DTask *task, - const char *source_path + const char *source_path, + uint64_t scanset_id ) { - if (!source_path || !source_path[0]) { + if (!source_path || !source_path[0] || scanset_id == 0) { return LARDON3D_PROJECT_TASK_CHECKPOINT_INVALID_TASK; } - return checkpoint_task_internal(state, task, source_path); + return checkpoint_task_internal(state, task, source_path, scanset_id); } static bool diff --git a/src/project_db.c b/src/project_db.c index 75535f5..f4d88af 100644 --- a/src/project_db.c +++ b/src/project_db.c @@ -17,10 +17,11 @@ struct Lardon3DProjectDb { char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY]; }; -static const char schema_v3[] = +static const char schema_v4[] = "CREATE TABLE metadata(key TEXT PRIMARY KEY,value INTEGER NOT NULL);" - "INSERT INTO metadata(key,value) VALUES('schema_version',3);" + "INSERT INTO metadata(key,value) VALUES('schema_version',4);" "INSERT INTO metadata(key,value) VALUES('next_task_id',1);" + "INSERT INTO metadata(key,value) VALUES('legacy_image_catalog_pending',0);" "CREATE TABLE project(singleton INTEGER PRIMARY KEY CHECK(singleton=1)," "stable_id TEXT NOT NULL UNIQUE,name TEXT NOT NULL,created_at INTEGER NOT NULL,updated_at INTEGER NOT NULL);" "CREATE TABLE tasks(task_id INTEGER PRIMARY KEY CHECK(task_id>0),name TEXT NOT NULL," @@ -40,8 +41,21 @@ static const char schema_v3[] = "producer_task_id INTEGER REFERENCES tasks(task_id),created_at INTEGER NOT NULL,updated_at INTEGER NOT NULL);" "CREATE INDEX artifacts_state_idx ON artifacts(state,artifact_id);" "CREATE INDEX artifacts_producer_idx ON artifacts(producer_task_id);" + "CREATE TABLE scansets(scanset_id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(scanset_id>0)," + "name TEXT NOT NULL CHECK(length(name)>0 AND length(name)<256),created_at INTEGER NOT NULL CHECK(created_at>=0)," + "updated_at INTEGER NOT NULL CHECK(updated_at>=created_at));" + "CREATE TABLE image_assets(asset_id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(asset_id>0),sha256 BLOB NOT NULL UNIQUE CHECK(length(sha256)=32)," + "path TEXT NOT NULL UNIQUE CHECK(length(path)>0 AND length(path)<4096),size_bytes INTEGER NOT NULL CHECK(size_bytes>=0)," + "state INTEGER NOT NULL CHECK(state=1),created_at INTEGER NOT NULL CHECK(created_at>=0));" + "CREATE TABLE images(image_id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(image_id>0),scanset_id INTEGER NOT NULL REFERENCES scansets(scanset_id)," + "asset_id INTEGER NOT NULL REFERENCES image_assets(asset_id),original_name TEXT NOT NULL CHECK(length(original_name)>0 AND length(original_name)<256)," + "source_path TEXT NOT NULL CHECK(length(source_path)>0 AND length(source_path)<4096)," + "producer_task_id INTEGER REFERENCES tasks(task_id),imported_at INTEGER NOT NULL CHECK(imported_at>=0)," + "UNIQUE(scanset_id,asset_id));" + "CREATE INDEX images_scanset_idx ON images(scanset_id,image_id);" + "CREATE INDEX images_producer_idx ON images(producer_task_id,image_id);" "CREATE TABLE image_import_tasks(task_id INTEGER PRIMARY KEY REFERENCES tasks(task_id) ON DELETE CASCADE," - "source_path TEXT NOT NULL);"; + "source_path TEXT NOT NULL,scanset_id INTEGER NOT NULL REFERENCES scansets(scanset_id));"; static void copy_error(char destination[LARDON3D_PROJECT_DB_ERROR_CAPACITY], const char *text) @@ -130,12 +144,13 @@ migrate(Lardon3DProjectDb *database, unsigned int from_version) if (from_version == LARDON3D_PROJECT_DB_SCHEMA_VERSION) { return LARDON3D_PROJECT_DB_OK; } - if (from_version != 0 && from_version != 1 && from_version != 2) { + if (from_version != 0 && from_version != 1 && from_version != 2 + && from_version != 3) { return LARDON3D_PROJECT_DB_CORRUPT; } Lardon3DProjectDbResult result = execute(database, "BEGIN IMMEDIATE", "begin migration"); if (result == LARDON3D_PROJECT_DB_OK && from_version == 0) { - result = execute(database, schema_v3, "create schema v3"); + result = execute(database, schema_v4, "create schema v4"); } if (result == LARDON3D_PROJECT_DB_OK && from_version == 1) { result = execute(database, @@ -156,7 +171,8 @@ migrate(Lardon3DProjectDb *database, unsigned int from_version) "finish schema v2 migration"); } } - if (result == LARDON3D_PROJECT_DB_OK && from_version != 0) { + if (result == LARDON3D_PROJECT_DB_OK && from_version != 0 + && from_version < 3) { result = execute(database, "CREATE TABLE image_import_tasks(task_id INTEGER PRIMARY KEY REFERENCES tasks(task_id) ON DELETE CASCADE,source_path TEXT NOT NULL);" "INSERT INTO metadata(key,value) VALUES('next_task_id',(SELECT CASE WHEN COALESCE(MAX(task_id),0)>=9223372036854775807 THEN 0 ELSE COALESCE(MAX(task_id),0)+1 END FROM tasks))", @@ -175,6 +191,32 @@ migrate(Lardon3DProjectDb *database, unsigned int from_version) "finish schema v3 migration"); } } + if (result == LARDON3D_PROJECT_DB_OK && from_version != 0) { + result = execute(database, + "CREATE TABLE scansets(scanset_id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(scanset_id>0),name TEXT NOT NULL CHECK(length(name)>0 AND length(name)<256),created_at INTEGER NOT NULL CHECK(created_at>=0),updated_at INTEGER NOT NULL CHECK(updated_at>=created_at));" + "CREATE TABLE image_assets(asset_id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(asset_id>0),sha256 BLOB NOT NULL UNIQUE CHECK(length(sha256)=32),path TEXT NOT NULL UNIQUE CHECK(length(path)>0 AND length(path)<4096),size_bytes INTEGER NOT NULL CHECK(size_bytes>=0),state INTEGER NOT NULL CHECK(state=1),created_at INTEGER NOT NULL CHECK(created_at>=0));" + "CREATE TABLE images(image_id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(image_id>0),scanset_id INTEGER NOT NULL REFERENCES scansets(scanset_id),asset_id INTEGER NOT NULL REFERENCES image_assets(asset_id),original_name TEXT NOT NULL CHECK(length(original_name)>0 AND length(original_name)<256),source_path TEXT NOT NULL CHECK(length(source_path)>0 AND length(source_path)<4096),producer_task_id INTEGER REFERENCES tasks(task_id),imported_at INTEGER NOT NULL CHECK(imported_at>=0),UNIQUE(scanset_id,asset_id));" + "CREATE INDEX images_scanset_idx ON images(scanset_id,image_id);" + "CREATE INDEX images_producer_idx ON images(producer_task_id,image_id);" + "ALTER TABLE image_import_tasks ADD COLUMN scanset_id INTEGER REFERENCES scansets(scanset_id);" + "INSERT INTO scansets(name,created_at,updated_at) SELECT 'Imports antérieurs à ScanSet v1',0,0 WHERE EXISTS(SELECT 1 FROM image_import_tasks);" + "UPDATE image_import_tasks SET scanset_id=(SELECT scanset_id FROM scansets WHERE name='Imports antérieurs à ScanSet v1' ORDER BY scanset_id LIMIT 1) WHERE scanset_id IS NULL;" + "INSERT INTO metadata(key,value) VALUES('legacy_image_catalog_pending',CASE WHEN EXISTS(SELECT 1 FROM image_import_tasks) THEN 1 ELSE 0 END)", + "migrate schema v3 to v4"); +#ifdef LARDON3D_PROJECT_DB_TESTING + const char *forced_failure = getenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V4"); + if (result == LARDON3D_PROJECT_DB_OK && forced_failure + && strcmp(forced_failure, "1") == 0) { + result = execute(database, "INSERT INTO missing_test_table VALUES(1)", + "forced migration failure"); + } +#endif + if (result == LARDON3D_PROJECT_DB_OK) { + result = execute(database, + "UPDATE metadata SET value=4 WHERE key='schema_version' AND value=3", + "finish schema v4 migration"); + } + } if (result == LARDON3D_PROJECT_DB_OK) result = execute(database, "COMMIT", "commit migration"); if (result != LARDON3D_PROJECT_DB_OK) (void)execute(database, "ROLLBACK", "rollback migration"); return result; @@ -237,8 +279,8 @@ lardon3d_project_db_open(const char *path, Lardon3DProjectDb **output, char erro if (result == LARDON3D_PROJECT_DB_OK) result = read_schema_version(database, &version); if (result == LARDON3D_PROJECT_DB_OK) result = migrate(database, version); if (result == LARDON3D_PROJECT_DB_OK) { - const char *required[] = {"project", "tasks", "checkpoints", "artifacts", "image_import_tasks"}; - for (size_t index = 0; index < 5 && result == LARDON3D_PROJECT_DB_OK; ++index) { + const char *required[] = {"project", "tasks", "checkpoints", "artifacts", "image_import_tasks", "scansets", "image_assets", "images"}; + for (size_t index = 0; index < 8 && result == LARDON3D_PROJECT_DB_OK; ++index) { if (!table_exists(database->connection, required[index])) { copy_error(database->error, "Schéma v1 incomplet."); result = LARDON3D_PROJECT_DB_CORRUPT; @@ -274,6 +316,32 @@ lardon3d_project_db_last_error(Lardon3DProjectDb *database, char error[LARDON3D_ } unsigned int lardon3d_project_db_schema_version(Lardon3DProjectDb *database) { return database ? LARDON3D_PROJECT_DB_SCHEMA_VERSION : 0U; } +Lardon3DProjectDbResult +lardon3d_project_db_legacy_catalog_pending(Lardon3DProjectDb *database, + bool *pending) +{ + if (pending) *pending = false; + if (!database || !pending) return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + (void)pthread_mutex_lock(&database->mutex); + sqlite3_stmt *statement = NULL; + Lardon3DProjectDbResult result = prepare(database, + "SELECT value FROM metadata WHERE key='legacy_image_catalog_pending'", + &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + int code = sqlite3_step(statement); + sqlite3_int64 value = sqlite3_column_int64(statement, 0); + if (code != SQLITE_ROW || (value != 0 && value != 1) + || sqlite3_step(statement) != SQLITE_DONE) { + result = LARDON3D_PROJECT_DB_CORRUPT; + } else { + *pending = value == 1; + } + (void)sqlite3_finalize(statement); + } + (void)pthread_mutex_unlock(&database->mutex); + return result; +} + static Lardon3DProjectDbResult step_done(Lardon3DProjectDb *database, sqlite3_stmt *statement, const char *context) { @@ -374,7 +442,7 @@ static Lardon3DProjectDbResult record_task_internal(Lardon3DProjectDb *database, const Lardon3DTaskDurableSnapshot *snapshot, const char *task_kind, uint32_t task_kind_version, const Lardon3DProjectDbCheckpoint *checkpoint, const char *source_path, - int64_t updated_at) + uint64_t scanset_id, int64_t updated_at) { bool typed = task_kind != NULL; if (!database || !valid_durable_task(snapshot, updated_at) @@ -383,6 +451,7 @@ record_task_internal(Lardon3DProjectDb *database, const Lardon3DTaskDurableSnaps || (!typed && task_kind_version != 0) || (source_path && !bounded_text(source_path, LARDON3D_PROJECT_DB_PATH_CAPACITY, false)) + || (source_path && !valid_task_id(scanset_id)) || (checkpoint && !valid_checkpoint(checkpoint))) return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; #ifdef LARDON3D_PROJECT_DB_TESTING const char *forced_busy = getenv("LARDON3D_TEST_PROJECT_DB_BUSY_CHECKPOINT"); @@ -440,14 +509,15 @@ record_task_internal(Lardon3DProjectDb *database, const Lardon3DTaskDurableSnaps } if (result == LARDON3D_PROJECT_DB_OK && source_path) { result = prepare(database, - "INSERT INTO image_import_tasks(task_id,source_path) VALUES(?1,?2) " + "INSERT INTO image_import_tasks(task_id,source_path,scanset_id) VALUES(?1,?2,?3) " "ON CONFLICT(task_id) DO UPDATE SET source_path=excluded.source_path " - "WHERE image_import_tasks.source_path=excluded.source_path", + "WHERE image_import_tasks.source_path=excluded.source_path AND image_import_tasks.scanset_id=excluded.scanset_id", &statement); if (result == LARDON3D_PROJECT_DB_OK) { (void)sqlite3_bind_int64(statement, 1, (sqlite3_int64)snapshot->id); (void)sqlite3_bind_text(statement, 2, source_path, -1, SQLITE_TRANSIENT); + (void)sqlite3_bind_int64(statement, 3, (sqlite3_int64)scanset_id); result = step_done(database, statement, "upsert image import"); if (result == LARDON3D_PROJECT_DB_OK && sqlite3_changes(database->connection) != 1) { @@ -482,7 +552,7 @@ lardon3d_project_db_record_task( ) { return record_task_internal(database, snapshot, task_kind, - task_kind_version, checkpoint, NULL, updated_at); + task_kind_version, checkpoint, NULL, 0, updated_at); } Lardon3DProjectDbResult @@ -493,12 +563,13 @@ lardon3d_project_db_record_image_import_task( uint32_t task_kind_version, const Lardon3DProjectDbCheckpoint *checkpoint, const char *source_path, + uint64_t scanset_id, int64_t updated_at ) { if (!source_path) return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; return record_task_internal(database, snapshot, task_kind, - task_kind_version, checkpoint, source_path, updated_at); + task_kind_version, checkpoint, source_path, scanset_id, updated_at); } static bool @@ -650,7 +721,7 @@ lardon3d_project_db_load_image_import( (void)pthread_mutex_lock(&database->mutex); sqlite3_stmt *statement = NULL; Lardon3DProjectDbResult result = prepare(database, - "SELECT source_path FROM image_import_tasks WHERE task_id=?1", + "SELECT source_path,scanset_id FROM image_import_tasks WHERE task_id=?1", &statement); if (result == LARDON3D_PROJECT_DB_OK) { (void)sqlite3_bind_int64(statement, 1, (sqlite3_int64)task_id); @@ -660,7 +731,13 @@ lardon3d_project_db_load_image_import( parameters->source_path, sizeof(parameters->source_path))) { result = LARDON3D_PROJECT_DB_CORRUPT; } else { + sqlite3_int64 scanset = sqlite3_column_int64(statement, 1); + if (scanset <= 0) { + result = LARDON3D_PROJECT_DB_CORRUPT; + } else { parameters->task_id = task_id; + parameters->scanset_id = (uint64_t)scanset; + } } (void)sqlite3_finalize(statement); } @@ -668,6 +745,366 @@ lardon3d_project_db_load_image_import( return result; } +static bool +valid_catalog_id(uint64_t value) +{ + return value > 0 && value <= INT64_MAX; +} + +static bool +valid_original_name(const char *name) +{ + return bounded_text(name, LARDON3D_PROJECT_DB_IMAGE_NAME_CAPACITY, false) + && !strchr(name, '/') && !strchr(name, '\\') + && !strchr(name, '\t') && !strchr(name, '\r') && !strchr(name, '\n'); +} + +static bool +valid_relative_asset_path(const char *path) +{ + return bounded_text(path, LARDON3D_PROJECT_DB_PATH_CAPACITY, false) + && path[0] != '/' && !strchr(path, '\\') + && strncmp(path, "../", 3) != 0 && !strstr(path, "/../") + && strcmp(path, "..") != 0; +} + +static bool +canonical_asset_path( + const unsigned char hash[LARDON3D_PROJECT_DB_SHA256_SIZE], + const char *path +) +{ + static const char digits[] = "0123456789abcdef"; + char hex[65]; + for (size_t index = 0; index < LARDON3D_PROJECT_DB_SHA256_SIZE; ++index) { + hex[index * 2] = digits[hash[index] >> 4]; + hex[index * 2 + 1] = digits[hash[index] & 15U]; + } + hex[64] = '\0'; + char expected[LARDON3D_PROJECT_DB_PATH_CAPACITY]; + int written = snprintf(expected, sizeof(expected), + "assets/images/%c%c/%s", hex[0], hex[1], hex); + return written > 0 && (size_t)written < sizeof(expected) + && strcmp(path, expected) == 0; +} + +Lardon3DProjectDbResult +lardon3d_project_db_create_scanset(Lardon3DProjectDb *database, + const char *name, Lardon3DProjectDbScanSet *scanset) +{ + if (!database || !scanset + || !bounded_text(name, LARDON3D_PROJECT_DB_SCANSET_NAME_CAPACITY, false)) { + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + } + int64_t now = (int64_t)time(NULL); + if (now < 0) return LARDON3D_PROJECT_DB_IO_ERROR; + memset(scanset, 0, sizeof(*scanset)); + (void)pthread_mutex_lock(&database->mutex); + sqlite3_stmt *statement = NULL; + Lardon3DProjectDbResult result = prepare(database, + "INSERT INTO scansets(name,created_at,updated_at) VALUES(?1,?2,?2)", + &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + (void)sqlite3_bind_text(statement, 1, name, -1, SQLITE_TRANSIENT); + (void)sqlite3_bind_int64(statement, 2, now); + result = step_done(database, statement, "create scanset"); + } + sqlite3_int64 id = sqlite3_last_insert_rowid(database->connection); + if (result == LARDON3D_PROJECT_DB_OK && id <= 0) { + result = LARDON3D_PROJECT_DB_CONSTRAINT; + } + if (result == LARDON3D_PROJECT_DB_OK) { + scanset->scanset_id = (uint64_t)id; + scanset->created_at = now; + scanset->updated_at = now; + (void)snprintf(scanset->name, sizeof(scanset->name), "%s", name); + } + (void)pthread_mutex_unlock(&database->mutex); + return result; +} + +static bool +read_scanset(sqlite3_stmt *statement, Lardon3DProjectDbScanSet *scanset) +{ + sqlite3_int64 id = sqlite3_column_int64(statement, 0); + sqlite3_int64 created = sqlite3_column_int64(statement, 2); + sqlite3_int64 updated = sqlite3_column_int64(statement, 3); + if (id <= 0 || created < 0 || updated < created + || !copy_column(statement, 1, scanset->name, sizeof(scanset->name))) { + return false; + } + scanset->scanset_id = (uint64_t)id; + scanset->created_at = created; + scanset->updated_at = updated; + return true; +} + +Lardon3DProjectDbResult +lardon3d_project_db_load_scanset(Lardon3DProjectDb *database, + uint64_t scanset_id, Lardon3DProjectDbScanSet *scanset) +{ + if (!database || !valid_catalog_id(scanset_id) || !scanset) + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + memset(scanset, 0, sizeof(*scanset)); + (void)pthread_mutex_lock(&database->mutex); + sqlite3_stmt *statement = NULL; + Lardon3DProjectDbResult result = prepare(database, + "SELECT scanset_id,name,created_at,updated_at FROM scansets WHERE scanset_id=?1", + &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + (void)sqlite3_bind_int64(statement, 1, (sqlite3_int64)scanset_id); + int code = sqlite3_step(statement); + if (code == SQLITE_DONE) result = LARDON3D_PROJECT_DB_NOT_FOUND; + else if (code != SQLITE_ROW || !read_scanset(statement, scanset)) + result = LARDON3D_PROJECT_DB_CORRUPT; + (void)sqlite3_finalize(statement); + } + (void)pthread_mutex_unlock(&database->mutex); + return result; +} + +Lardon3DProjectDbResult +lardon3d_project_db_list_scansets(Lardon3DProjectDb *database, + uint64_t after_scanset_id, Lardon3DProjectDbScanSet *scansets, + size_t capacity, size_t *count) +{ + if (count) *count = 0; + if (!database || !scansets || !count || after_scanset_id > INT64_MAX + || capacity == 0 || capacity > LARDON3D_PROJECT_DB_CATALOG_PAGE_MAX) + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + (void)pthread_mutex_lock(&database->mutex); + sqlite3_stmt *statement = NULL; + Lardon3DProjectDbResult result = prepare(database, + "SELECT scanset_id,name,created_at,updated_at FROM scansets WHERE scanset_id>?1 ORDER BY scanset_id LIMIT ?2", + &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + (void)sqlite3_bind_int64(statement, 1, (sqlite3_int64)after_scanset_id); + (void)sqlite3_bind_int64(statement, 2, (sqlite3_int64)capacity); + int code = SQLITE_DONE; + while (*count < capacity && (code = sqlite3_step(statement)) == SQLITE_ROW) { + if (!read_scanset(statement, &scansets[*count])) { + result = LARDON3D_PROJECT_DB_CORRUPT; break; + } + ++*count; + } + if (result == LARDON3D_PROJECT_DB_OK && *count < capacity + && code != SQLITE_DONE) result = sqlite_result(database, code, "list scansets"); + (void)sqlite3_finalize(statement); + } + (void)pthread_mutex_unlock(&database->mutex); + return result; +} + +static bool +read_image(sqlite3_stmt *statement, Lardon3DProjectDbImage *image, + Lardon3DProjectDbImageAsset *asset) +{ + sqlite3_int64 image_id = sqlite3_column_int64(statement, 0); + sqlite3_int64 scanset_id = sqlite3_column_int64(statement, 1); + sqlite3_int64 asset_id = sqlite3_column_int64(statement, 2); + sqlite3_int64 size = sqlite3_column_int64(statement, 7); + int hash_size = sqlite3_column_bytes(statement, 5); + const void *hash = sqlite3_column_blob(statement, 5); + if (image_id <= 0 || scanset_id <= 0 || asset_id <= 0 || size < 0 + || hash_size != LARDON3D_PROJECT_DB_SHA256_SIZE || !hash + || !copy_column(statement, 3, image->original_name, sizeof(image->original_name)) + || !copy_column(statement, 4, image->source_path, sizeof(image->source_path)) + || !copy_column(statement, 6, asset->path, sizeof(asset->path)) + || !valid_original_name(image->original_name) + || !valid_relative_asset_path(asset->path)) return false; + image->image_id = (uint64_t)image_id; + image->scanset_id = (uint64_t)scanset_id; + image->asset_id = (uint64_t)asset_id; + asset->asset_id = (uint64_t)asset_id; + memcpy(asset->sha256, hash, LARDON3D_PROJECT_DB_SHA256_SIZE); + if (!canonical_asset_path(asset->sha256, asset->path)) return false; + asset->size_bytes = (uint64_t)size; + asset->state = (Lardon3DProjectDbImageAssetState)sqlite3_column_int(statement, 8); + asset->created_at = sqlite3_column_int64(statement, 9); + image->has_producer_task = sqlite3_column_type(statement, 10) != SQLITE_NULL; + sqlite3_int64 producer = sqlite3_column_int64(statement, 10); + if (image->has_producer_task && producer <= 0) return false; + image->producer_task_id = image->has_producer_task ? (uint64_t)producer : 0; + image->imported_at = sqlite3_column_int64(statement, 11); + return asset->state == LARDON3D_DB_IMAGE_ASSET_READY + && asset->created_at >= 0 && image->imported_at >= 0; +} + +static const char image_select[] = + "SELECT i.image_id,i.scanset_id,i.asset_id,i.original_name,i.source_path,a.sha256,a.path,a.size_bytes,a.state,a.created_at,i.producer_task_id,i.imported_at FROM images i JOIN image_assets a ON a.asset_id=i.asset_id "; + +Lardon3DProjectDbResult +lardon3d_project_db_register_image(Lardon3DProjectDb *database, + uint64_t scanset_id, const unsigned char sha256[LARDON3D_PROJECT_DB_SHA256_SIZE], + const char *asset_path, uint64_t size_bytes, const char *original_name, + const char *source_path, uint64_t producer_task_id, int64_t imported_at, + Lardon3DProjectDbImageRegisterStatus *status, Lardon3DProjectDbImage *image) +{ + if (!database || !valid_catalog_id(scanset_id) || !sha256 + || !valid_relative_asset_path(asset_path) + || !canonical_asset_path(sha256, asset_path) || size_bytes > INT64_MAX + || !valid_original_name(original_name) + || !bounded_text(source_path, LARDON3D_PROJECT_DB_PATH_CAPACITY, false) + || (producer_task_id != 0 && !valid_task_id(producer_task_id)) + || imported_at < 0 || !status || !image) + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; +#ifdef LARDON3D_PROJECT_DB_TESTING + const char *failure = getenv("LARDON3D_TEST_PROJECT_DB_FAIL_IMAGE_REGISTER"); + if (failure && strcmp(failure, "1") == 0) return LARDON3D_PROJECT_DB_BUSY; +#endif + memset(image, 0, sizeof(*image)); + *status = LARDON3D_PROJECT_DB_IMAGE_REGISTERED; + (void)pthread_mutex_lock(&database->mutex); + Lardon3DProjectDbResult result = execute(database, "BEGIN IMMEDIATE", "begin image register"); + sqlite3_stmt *statement = NULL; + if (result == LARDON3D_PROJECT_DB_OK) result = prepare(database, + "INSERT INTO image_assets(sha256,path,size_bytes,state,created_at) VALUES(?1,?2,?3,1,?4) ON CONFLICT(sha256) DO NOTHING", &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + (void)sqlite3_bind_blob(statement, 1, sha256, LARDON3D_PROJECT_DB_SHA256_SIZE, SQLITE_TRANSIENT); + (void)sqlite3_bind_text(statement, 2, asset_path, -1, SQLITE_TRANSIENT); + (void)sqlite3_bind_int64(statement, 3, (sqlite3_int64)size_bytes); + (void)sqlite3_bind_int64(statement, 4, imported_at); + result = step_done(database, statement, "insert image asset"); statement = NULL; + } + sqlite3_int64 asset_id = 0; + if (result == LARDON3D_PROJECT_DB_OK) result = prepare(database, + "SELECT asset_id,path,size_bytes,state FROM image_assets WHERE sha256=?1", &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + (void)sqlite3_bind_blob(statement, 1, sha256, LARDON3D_PROJECT_DB_SHA256_SIZE, SQLITE_TRANSIENT); + int code = sqlite3_step(statement); + char stored_path[LARDON3D_PROJECT_DB_PATH_CAPACITY]; + if (code != SQLITE_ROW || !copy_column(statement, 1, stored_path, sizeof(stored_path)) + || strcmp(stored_path, asset_path) != 0 + || sqlite3_column_int64(statement, 2) != (sqlite3_int64)size_bytes + || sqlite3_column_int(statement, 3) != LARDON3D_DB_IMAGE_ASSET_READY) { + result = LARDON3D_PROJECT_DB_CORRUPT; + } else asset_id = sqlite3_column_int64(statement, 0); + (void)sqlite3_finalize(statement); statement = NULL; + } + if (result == LARDON3D_PROJECT_DB_OK) result = prepare(database, + "INSERT INTO images(scanset_id,asset_id,original_name,source_path,producer_task_id,imported_at) VALUES(?1,?2,?3,?4,?5,?6) ON CONFLICT(scanset_id,asset_id) DO NOTHING", &statement); + int inserted = 0; + if (result == LARDON3D_PROJECT_DB_OK) { + (void)sqlite3_bind_int64(statement, 1, (sqlite3_int64)scanset_id); + (void)sqlite3_bind_int64(statement, 2, asset_id); + (void)sqlite3_bind_text(statement, 3, original_name, -1, SQLITE_TRANSIENT); + (void)sqlite3_bind_text(statement, 4, source_path, -1, SQLITE_TRANSIENT); + if (producer_task_id) (void)sqlite3_bind_int64(statement, 5, (sqlite3_int64)producer_task_id); + else (void)sqlite3_bind_null(statement, 5); + (void)sqlite3_bind_int64(statement, 6, imported_at); + result = step_done(database, statement, "insert logical image"); statement = NULL; + inserted = sqlite3_changes(database->connection); + } + sqlite3_int64 image_id = 0; + if (result == LARDON3D_PROJECT_DB_OK) result = prepare(database, + "SELECT image_id FROM images WHERE scanset_id=?1 AND asset_id=?2", &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + (void)sqlite3_bind_int64(statement, 1, (sqlite3_int64)scanset_id); + (void)sqlite3_bind_int64(statement, 2, asset_id); + int code = sqlite3_step(statement); + if (code != SQLITE_ROW || (image_id = sqlite3_column_int64(statement, 0)) <= 0) + result = LARDON3D_PROJECT_DB_CORRUPT; + (void)sqlite3_finalize(statement); statement = NULL; + } + if (result == LARDON3D_PROJECT_DB_OK) result = execute(database, "COMMIT", "commit image register"); + if (result != LARDON3D_PROJECT_DB_OK) (void)execute(database, "ROLLBACK", "rollback image register"); + (void)pthread_mutex_unlock(&database->mutex); + if (result != LARDON3D_PROJECT_DB_OK) return result; + *status = inserted == 1 ? LARDON3D_PROJECT_DB_IMAGE_REGISTERED + : LARDON3D_PROJECT_DB_IMAGE_ALREADY_PRESENT; + return lardon3d_project_db_load_image(database, (uint64_t)image_id, image, + &(Lardon3DProjectDbImageAsset){0}); +} + +Lardon3DProjectDbResult +lardon3d_project_db_load_image(Lardon3DProjectDb *database, uint64_t image_id, + Lardon3DProjectDbImage *image, Lardon3DProjectDbImageAsset *asset) +{ + if (!database || !valid_catalog_id(image_id) || !image || !asset) + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + memset(image, 0, sizeof(*image)); memset(asset, 0, sizeof(*asset)); + (void)pthread_mutex_lock(&database->mutex); + char sql[768]; (void)snprintf(sql, sizeof(sql), "%s WHERE i.image_id=?1", image_select); + sqlite3_stmt *statement = NULL; + Lardon3DProjectDbResult result = prepare(database, sql, &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + (void)sqlite3_bind_int64(statement, 1, (sqlite3_int64)image_id); + int code = sqlite3_step(statement); + if (code == SQLITE_DONE) result = LARDON3D_PROJECT_DB_NOT_FOUND; + else if (code != SQLITE_ROW || !read_image(statement, image, asset)) + result = LARDON3D_PROJECT_DB_CORRUPT; + (void)sqlite3_finalize(statement); + } + (void)pthread_mutex_unlock(&database->mutex); + return result; +} + +Lardon3DProjectDbResult +lardon3d_project_db_list_images(Lardon3DProjectDb *database, + uint64_t scanset_id, uint64_t after_image_id, Lardon3DProjectDbImage *images, + Lardon3DProjectDbImageAsset *assets, size_t capacity, size_t *count) +{ + if (count) *count = 0; + if (!database || !valid_catalog_id(scanset_id) || after_image_id > INT64_MAX + || !images || !assets || !count || capacity == 0 + || capacity > LARDON3D_PROJECT_DB_CATALOG_PAGE_MAX) + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + Lardon3DProjectDbScanSet scanset; + Lardon3DProjectDbResult exists = lardon3d_project_db_load_scanset( + database, scanset_id, &scanset); + if (exists != LARDON3D_PROJECT_DB_OK) return exists; + (void)pthread_mutex_lock(&database->mutex); + char sql[896]; (void)snprintf(sql, sizeof(sql), + "%s WHERE i.scanset_id=?1 AND i.image_id>?2 ORDER BY i.image_id LIMIT ?3", image_select); + sqlite3_stmt *statement = NULL; + Lardon3DProjectDbResult result = prepare(database, sql, &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + (void)sqlite3_bind_int64(statement, 1, (sqlite3_int64)scanset_id); + (void)sqlite3_bind_int64(statement, 2, (sqlite3_int64)after_image_id); + (void)sqlite3_bind_int64(statement, 3, (sqlite3_int64)capacity); + int code = SQLITE_DONE; + while (*count < capacity && (code = sqlite3_step(statement)) == SQLITE_ROW) { + memset(&images[*count], 0, sizeof(images[*count])); + memset(&assets[*count], 0, sizeof(assets[*count])); + if (!read_image(statement, &images[*count], &assets[*count])) { + result = LARDON3D_PROJECT_DB_CORRUPT; break; + } + ++*count; + } + if (result == LARDON3D_PROJECT_DB_OK && *count < capacity && code != SQLITE_DONE) + result = sqlite_result(database, code, "list images"); + (void)sqlite3_finalize(statement); + } + (void)pthread_mutex_unlock(&database->mutex); + return result; +} + +Lardon3DProjectDbResult +lardon3d_project_db_count_images(Lardon3DProjectDb *database, + uint64_t scanset_id, uint64_t *count) +{ + if (count) *count = 0; + if (!database || !valid_catalog_id(scanset_id) || !count) + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + Lardon3DProjectDbScanSet scanset; + Lardon3DProjectDbResult exists = lardon3d_project_db_load_scanset( + database, scanset_id, &scanset); + if (exists != LARDON3D_PROJECT_DB_OK) return exists; + (void)pthread_mutex_lock(&database->mutex); + sqlite3_stmt *statement = NULL; + Lardon3DProjectDbResult result = prepare(database, + "SELECT count(*) FROM images WHERE scanset_id=?1", &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + (void)sqlite3_bind_int64(statement, 1, (sqlite3_int64)scanset_id); + int code = sqlite3_step(statement); sqlite3_int64 value = sqlite3_column_int64(statement, 0); + if (code != SQLITE_ROW || value < 0) result = LARDON3D_PROJECT_DB_CORRUPT; + else *count = (uint64_t)value; + (void)sqlite3_finalize(statement); + } + (void)pthread_mutex_unlock(&database->mutex); + return result; +} + Lardon3DProjectDbResult lardon3d_project_db_create_artifact(Lardon3DProjectDb *database, const Lardon3DProjectDbArtifact *artifact) { @@ -732,4 +1169,39 @@ lardon3d_project_db_test_orphan_checkpoint(Lardon3DProjectDb *database) (void)pthread_mutex_unlock(&database->mutex); return result; } + +Lardon3DProjectDbResult +lardon3d_project_db_test_delete_catalog_identity(Lardon3DProjectDb *database, + uint64_t scanset_id, uint64_t image_id, uint64_t asset_id) +{ + if (!database || !valid_catalog_id(scanset_id) + || !valid_catalog_id(image_id) || !valid_catalog_id(asset_id)) + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + (void)pthread_mutex_lock(&database->mutex); + Lardon3DProjectDbResult result = execute(database, "BEGIN IMMEDIATE", + "begin catalog identity deletion test"); + const char *sql[] = { + "DELETE FROM images WHERE image_id=?1", + "DELETE FROM image_assets WHERE asset_id=?1", + "DELETE FROM scansets WHERE scanset_id=?1", + }; + const uint64_t ids[] = {image_id, asset_id, scanset_id}; + for (size_t index = 0; index < 3 && result == LARDON3D_PROJECT_DB_OK; ++index) { + sqlite3_stmt *statement = NULL; + result = prepare(database, sql[index], &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + (void)sqlite3_bind_int64(statement, 1, (sqlite3_int64)ids[index]); + result = step_done(database, statement, "delete catalog identity test"); + if (result == LARDON3D_PROJECT_DB_OK + && sqlite3_changes(database->connection) != 1) + result = LARDON3D_PROJECT_DB_NOT_FOUND; + } + } + if (result == LARDON3D_PROJECT_DB_OK) + result = execute(database, "COMMIT", "commit catalog identity deletion test"); + if (result != LARDON3D_PROJECT_DB_OK) + (void)execute(database, "ROLLBACK", "rollback catalog identity deletion test"); + (void)pthread_mutex_unlock(&database->mutex); + return result; +} #endif diff --git a/tests/test_import_task.c b/tests/test_import_task.c index 0dff608..7fc7c17 100644 --- a/tests/test_import_task.c +++ b/tests/test_import_task.c @@ -105,11 +105,14 @@ run_test(void) Lardon3DAppState state; lardon3d_app_state_init(&state); CHECK(setup_runtime(&state, 4)); CHECK(lardon3d_project_create(&state, "Persistent Import")); + Lardon3DProjectDbScanSet scanset; + CHECK(lardon3d_image_catalog_create_scanset(&state, "Campagne A", &scanset)); CHECK(setenv("LARDON3D_TEST_IMPORT_PAUSE_AFTER_BATCH", "1", 1) == 0); CHECK(setenv("LARDON3D_TEST_IMPORT_SKIP_FINISHED_CHECKPOINT", "1", 1) == 0); uint64_t task_id = 0; - CHECK(lardon3d_project_enqueue_image_import(&state, source, &task_id)); + CHECK(lardon3d_project_enqueue_image_import(&state, scanset.scanset_id, + source, &task_id)); CHECK(task_id > 0 && task_id <= INT64_MAX); Lardon3DTaskSnapshot runtime; CHECK(wait_for_state(state.task_queue, task_id, TASK_PAUSED, &runtime)); @@ -134,7 +137,7 @@ run_test(void) CHECK(lardon3d_project_db_record_image_import_task(state.project_db, &persisted_snapshot, LARDON3D_IMAGE_IMPORT_TASK_KIND, LARDON3D_IMAGE_IMPORT_TASK_KIND_VERSION, &published_not_durable, - source, 1) == LARDON3D_PROJECT_DB_OK); + source, scanset.scanset_id, 1) == LARDON3D_PROJECT_DB_OK); lardon3d_project_close(&state); CHECK(unsetenv("LARDON3D_TEST_IMPORT_PAUSE_AFTER_BATCH") == 0); CHECK(unsetenv("LARDON3D_TEST_IMPORT_SKIP_FINISHED_CHECKPOINT") == 0); @@ -154,14 +157,19 @@ run_test(void) CHECK(lardon3d_project_db_load_image_import(state.project_db, task_id, &persisted_parameters) == LARDON3D_PROJECT_DB_OK); CHECK(strcmp(persisted_parameters.source_path, source) == 0); + CHECK(persisted_parameters.scanset_id == scanset.scanset_id); CHECK(wait_for_state(state.task_queue, task_id, TASK_COMPLETED, &runtime)); CHECK(runtime.progress == 100); - char error[256]; - Lardon3DImageCatalog *catalog = lardon3d_image_catalog_load( - &state, error, sizeof(error)); - CHECK(catalog && lardon3d_image_catalog_count(catalog) == 80); - lardon3d_image_catalog_destroy(catalog); + uint64_t image_count = 0; + CHECK(lardon3d_project_db_count_images(state.project_db, + scanset.scanset_id, &image_count) == LARDON3D_PROJECT_DB_OK + && image_count == 80); + char catalog_error[256]; + Lardon3DImageCatalog *legacy_catalog = lardon3d_image_catalog_load(&state, + catalog_error, sizeof(catalog_error)); + CHECK(legacy_catalog && lardon3d_image_catalog_count(legacy_catalog) == 80); + lardon3d_image_catalog_destroy(legacy_catalog); lardon3d_task_queue_destroy(state.task_queue); state.task_queue = NULL; lardon3d_project_close(&state); @@ -208,15 +216,20 @@ test_selective_capacity_one(void) lardon3d_app_state_init(&state); CHECK(setup_runtime(&state, 4)); CHECK(lardon3d_project_create(&state, "Selective Recovery")); + Lardon3DProjectDbScanSet scanset; + CHECK(lardon3d_image_catalog_create_scanset(&state, "Sélectif", &scanset)); CHECK(setenv("LARDON3D_TEST_IMPORT_PAUSE_AFTER_BATCH", "1", 1) == 0); CHECK(setenv("LARDON3D_TEST_IMPORT_SKIP_FINISHED_CHECKPOINT", "1", 1) == 0); uint64_t missing_id = 0, valid_id = 0, second_valid_id = 0; - CHECK(lardon3d_project_enqueue_image_import(&state, missing_source, + CHECK(lardon3d_project_enqueue_image_import(&state, scanset.scanset_id, + missing_source, &missing_id)); - CHECK(lardon3d_project_enqueue_image_import(&state, valid_source, + CHECK(lardon3d_project_enqueue_image_import(&state, scanset.scanset_id, + valid_source, &valid_id)); - CHECK(lardon3d_project_enqueue_image_import(&state, second_valid_source, + CHECK(lardon3d_project_enqueue_image_import(&state, scanset.scanset_id, + second_valid_source, &second_valid_id)); Lardon3DTaskSnapshot snapshot; CHECK(wait_for_state(state.task_queue, missing_id, TASK_PAUSED, &snapshot)); diff --git a/tests/test_persistent_image_catalog.c b/tests/test_persistent_image_catalog.c new file mode 100644 index 0000000..06cf8d1 --- /dev/null +++ b/tests/test_persistent_image_catalog.c @@ -0,0 +1,278 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define CHECK(condition) do { if (!(condition)) { \ + (void)fprintf(stderr, "Échec ligne %d : %s\n", __LINE__, #condition); return false; \ +} } while (0) + +static bool +join_path(char output[PATH_MAX], const char *parent, const char *child) +{ + int written = snprintf(output, PATH_MAX, "%s/%s", parent, child); + return written > 0 && (size_t)written < PATH_MAX; +} + +static bool +write_file(const char *path, const char *content) +{ + int descriptor = open(path, O_WRONLY | O_CREAT | O_EXCL, 0600); + if (descriptor < 0) return false; + size_t size = strlen(content), offset = 0; + while (offset < size) { + ssize_t written = write(descriptor, content + offset, size - offset); + if (written < 0 && errno == EINTR) continue; + if (written <= 0) { (void)close(descriptor); return false; } + offset += (size_t)written; + } + return close(descriptor) == 0; +} + +static bool +canonical_asset_path(const char *path) +{ + static const char prefix[] = "assets/images/"; + if (strncmp(path, prefix, sizeof(prefix) - 1) != 0) return false; + const char *short_hash = path + sizeof(prefix) - 1; + if (strlen(short_hash) != 2 + 1 + 64 || short_hash[2] != '/') return false; + for (size_t index = 0; index < 2; ++index) { + if (short_hash[index] != short_hash[index + 3]) return false; + } + for (size_t index = 0; index < 64; ++index) { + char character = short_hash[index + 3]; + if (!((character >= '0' && character <= '9') + || (character >= 'a' && character <= 'f'))) return false; + } + return true; +} + +static bool +remove_tree(const char *path) +{ + struct stat information; + if (lstat(path, &information) != 0) return errno == ENOENT; + if (!S_ISDIR(information.st_mode)) return unlink(path) == 0; + DIR *directory = opendir(path); + if (!directory) return false; + bool success = true; + for (struct dirent *entry = readdir(directory); entry; entry = readdir(directory)) { + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) + continue; + char child[PATH_MAX]; + if (!join_path(child, path, entry->d_name) || !remove_tree(child)) success = false; + } + if (closedir(directory) != 0 || rmdir(path) != 0) success = false; + return success; +} + +static size_t +count_regular_files(const char *path) +{ + DIR *directory = opendir(path); + if (!directory) return 0; + size_t count = 0; + for (struct dirent *entry = readdir(directory); entry; entry = readdir(directory)) { + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) + continue; + char child[PATH_MAX]; struct stat information; + if (!join_path(child, path, entry->d_name) || lstat(child, &information) != 0) + continue; + if (S_ISDIR(information.st_mode)) count += count_regular_files(child); + else if (S_ISREG(information.st_mode)) ++count; + } + (void)closedir(directory); + return count; +} + +typedef struct { + Lardon3DAppState *state; + uint64_t scanset_id; + const char *path; + Lardon3DImageCatalogImportResult result; + uint64_t image_id; +} ImportThread; + +static void * +import_thread(void *userdata) +{ + ImportThread *thread = userdata; + Lardon3DProjectDbImage image; + Lardon3DProjectDbImageAsset asset; + thread->result = lardon3d_image_catalog_import_file(thread->state, + thread->scanset_id, thread->path, 0, &image, &asset); + thread->image_id = image.image_id; + return NULL; +} + +static bool +run_test(void) +{ + char root[] = "/tmp/lardon3d-catalog-v1-XXXXXX"; + CHECK(mkdtemp(root)); + char database_path[PATH_MAX], source_a[PATH_MAX], source_b[PATH_MAX]; + CHECK(join_path(database_path, root, "project.db")); + CHECK(join_path(source_a, root, "photo001.jpg")); + CHECK(join_path(source_b, root, "other-name.jpg")); + CHECK(write_file(source_a, "same-image-content")); + CHECK(write_file(source_b, "same-image-content")); + Lardon3DProjectDb *database = NULL; + char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY]; + CHECK(lardon3d_project_db_open(database_path, &database, error) + == LARDON3D_PROJECT_DB_OK); + Lardon3DAppState state; + lardon3d_app_state_init(&state); + state.project_loaded = true; + state.project_db = database; + CHECK(snprintf(state.project_path, sizeof(state.project_path), "%s", root) > 0); + + Lardon3DProjectDbScanSet a, b; + CHECK(lardon3d_image_catalog_create_scanset(&state, "Campagne générale", &a)); + CHECK(lardon3d_image_catalog_create_scanset(&state, "Pièce démontée", &b)); + CHECK(a.scanset_id != b.scanset_id); + Lardon3DProjectDbScanSet scanset_page[2]; size_t scanset_count = 0; + CHECK(lardon3d_project_db_list_scansets(database, 0, scanset_page, 1, + &scanset_count) == LARDON3D_PROJECT_DB_OK && scanset_count == 1 + && scanset_page[0].scanset_id == a.scanset_id); + CHECK(lardon3d_project_db_list_scansets(database, a.scanset_id, + scanset_page, 2, &scanset_count) == LARDON3D_PROJECT_DB_OK + && scanset_count == 1 && scanset_page[0].scanset_id == b.scanset_id); + uint64_t count = 99; + CHECK(lardon3d_project_db_count_images(database, a.scanset_id, &count) + == LARDON3D_PROJECT_DB_OK && count == 0); + unsigned char zero_hash[LARDON3D_PROJECT_DB_SHA256_SIZE] = {0}; + Lardon3DProjectDbImageRegisterStatus invalid_status; + Lardon3DProjectDbImage invalid_image; + CHECK(lardon3d_project_db_register_image(database, a.scanset_id, zero_hash, + "../../outside", 1, "x.jpg", "/source/x.jpg", 0, 1, + &invalid_status, &invalid_image) == LARDON3D_PROJECT_DB_INVALID_ARGUMENT); + CHECK(lardon3d_project_db_count_images(database, UINT64_C(999999), &count) + == LARDON3D_PROJECT_DB_NOT_FOUND); + + Lardon3DProjectDbImage image_a, duplicate_a, image_b; + Lardon3DProjectDbImageAsset asset_a, duplicate_asset, asset_b; + CHECK(lardon3d_image_catalog_import_file(&state, a.scanset_id, source_a, 0, + &image_a, &asset_a) == LARDON3D_IMAGE_CATALOG_IMPORTED); + CHECK(canonical_asset_path(asset_a.path)); + CHECK(lardon3d_image_catalog_import_file(&state, a.scanset_id, source_b, 0, + &duplicate_a, &duplicate_asset) == LARDON3D_IMAGE_CATALOG_ALREADY_PRESENT); + CHECK(image_a.image_id == duplicate_a.image_id + && asset_a.asset_id == duplicate_asset.asset_id); + CHECK(lardon3d_image_catalog_import_file(&state, b.scanset_id, source_b, 0, + &image_b, &asset_b) == LARDON3D_IMAGE_CATALOG_IMPORTED); + CHECK(image_b.image_id != image_a.image_id && asset_b.asset_id == asset_a.asset_id); + CHECK(strcmp(asset_b.path, asset_a.path) == 0); + + char different_dir[PATH_MAX], same_name[PATH_MAX]; + CHECK(join_path(different_dir, root, "different") && mkdir(different_dir, 0700) == 0); + CHECK(join_path(same_name, different_dir, "photo001.jpg")); + CHECK(write_file(same_name, "different-content")); + Lardon3DProjectDbImage different_image; + Lardon3DProjectDbImageAsset different_asset; + CHECK(lardon3d_image_catalog_import_file(&state, a.scanset_id, same_name, 0, + &different_image, &different_asset) == LARDON3D_IMAGE_CATALOG_IMPORTED); + CHECK(different_image.image_id != image_a.image_id + && different_asset.asset_id != asset_a.asset_id + && canonical_asset_path(different_asset.path) + && strcmp(different_asset.path, asset_a.path) != 0); + + char orphan_source[PATH_MAX], asset_root[PATH_MAX]; + CHECK(join_path(orphan_source, root, "orphan.jpg")); + CHECK(join_path(asset_root, root, "assets/images")); + CHECK(write_file(orphan_source, "published-before-db-failure")); + size_t files_before = count_regular_files(asset_root); + uint64_t logical_before = 0; + CHECK(lardon3d_project_db_count_images(database, a.scanset_id, + &logical_before) == LARDON3D_PROJECT_DB_OK); + CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_IMAGE_REGISTER", "1", 1) == 0); + CHECK(lardon3d_image_catalog_import_file(&state, a.scanset_id, + orphan_source, 0, &different_image, &different_asset) + == LARDON3D_IMAGE_CATALOG_DB_ERROR); + CHECK(unsetenv("LARDON3D_TEST_PROJECT_DB_FAIL_IMAGE_REGISTER") == 0); + CHECK(count_regular_files(asset_root) == files_before + 1); + uint64_t logical_after = 0; + CHECK(lardon3d_project_db_count_images(database, a.scanset_id, + &logical_after) == LARDON3D_PROJECT_DB_OK + && logical_after == logical_before); + + char concurrent_path[PATH_MAX]; + CHECK(join_path(concurrent_path, root, "concurrent.jpg")); + CHECK(write_file(concurrent_path, "concurrent-content")); + ImportThread contexts[2] = { + {.state = &state, .scanset_id = a.scanset_id, .path = concurrent_path}, + {.state = &state, .scanset_id = a.scanset_id, .path = concurrent_path}, + }; + pthread_t threads[2]; + CHECK(pthread_create(&threads[0], NULL, import_thread, &contexts[0]) == 0); + CHECK(pthread_create(&threads[1], NULL, import_thread, &contexts[1]) == 0); + CHECK(pthread_join(threads[0], NULL) == 0 && pthread_join(threads[1], NULL) == 0); + CHECK(((contexts[0].result == LARDON3D_IMAGE_CATALOG_IMPORTED + && contexts[1].result == LARDON3D_IMAGE_CATALOG_ALREADY_PRESENT) + || (contexts[1].result == LARDON3D_IMAGE_CATALOG_IMPORTED + && contexts[0].result == LARDON3D_IMAGE_CATALOG_ALREADY_PRESENT)) + && contexts[0].image_id == contexts[1].image_id); + + char bulk_directory[PATH_MAX]; + CHECK(join_path(bulk_directory, root, "bulk") && mkdir(bulk_directory, 0700) == 0); + for (uint64_t index = 1; index <= 2048; ++index) { + char name[64], source[PATH_MAX], content[64]; + CHECK(snprintf(name, sizeof(name), "bulk-%llu.jpg", + (unsigned long long)index) > 0); + CHECK(join_path(source, bulk_directory, name)); + CHECK(snprintf(content, sizeof(content), "unique-content-%llu", + (unsigned long long)index) > 0); + CHECK(write_file(source, content)); + Lardon3DProjectDbImage bulk_image; + Lardon3DProjectDbImageAsset bulk_asset; + CHECK(lardon3d_image_catalog_import_file(&state, a.scanset_id, source, + 0, &bulk_image, &bulk_asset) == LARDON3D_IMAGE_CATALOG_IMPORTED); + } + CHECK(lardon3d_project_db_count_images(database, a.scanset_id, &count) + == LARDON3D_PROJECT_DB_OK && count == 2051); + Lardon3DProjectDbImage page[8]; Lardon3DProjectDbImageAsset assets[8]; + size_t page_count = 0; uint64_t cursor = 0, visited = 0; + while (lardon3d_image_catalog_list(&state, a.scanset_id, cursor, page, + assets, 8, &page_count) == LARDON3D_PROJECT_DB_OK && page_count) { + for (size_t index = 0; index < page_count; ++index) { + CHECK(page[index].image_id > cursor); + cursor = page[index].image_id; ++visited; + } + } + CHECK(visited == count); + size_t one_count = 0; + CHECK(lardon3d_image_catalog_list(&state, a.scanset_id, 0, page, assets, + 1, &one_count) == LARDON3D_PROJECT_DB_OK && one_count == 1); + Lardon3DProjectDbImage maximum_page[LARDON3D_PROJECT_DB_CATALOG_PAGE_MAX]; + Lardon3DProjectDbImageAsset maximum_assets[LARDON3D_PROJECT_DB_CATALOG_PAGE_MAX]; + size_t maximum_count = 0; + CHECK(lardon3d_image_catalog_list(&state, a.scanset_id, 0, maximum_page, + maximum_assets, LARDON3D_PROJECT_DB_CATALOG_PAGE_MAX, &maximum_count) + == LARDON3D_PROJECT_DB_OK + && maximum_count == LARDON3D_PROJECT_DB_CATALOG_PAGE_MAX); + CHECK(lardon3d_image_catalog_list(&state, a.scanset_id, 0, page, assets, + LARDON3D_PROJECT_DB_CATALOG_PAGE_MAX + 1, &page_count) + == LARDON3D_PROJECT_DB_INVALID_ARGUMENT); + CHECK(lardon3d_image_catalog_list(&state, UINT64_C(999999), 0, page, + assets, 8, &page_count) == LARDON3D_PROJECT_DB_NOT_FOUND); + CHECK(unlink(source_a) == 0 && unlink(source_b) == 0); + Lardon3DProjectDbImage loaded; Lardon3DProjectDbImageAsset loaded_asset; + CHECK(lardon3d_project_db_load_image(database, image_a.image_id, &loaded, + &loaded_asset) == LARDON3D_PROJECT_DB_OK + && loaded.asset_id == asset_a.asset_id); + + lardon3d_project_db_close(database); + state.project_db = NULL; + CHECK(remove_tree(root)); + return true; +} + +int main(void) { return run_test() ? EXIT_SUCCESS : EXIT_FAILURE; } diff --git a/tests/test_project_db.c b/tests/test_project_db.c index ac022ea..6e4e121 100644 --- a/tests/test_project_db.c +++ b/tests/test_project_db.c @@ -16,6 +16,24 @@ } } while (0) Lardon3DProjectDbResult lardon3d_project_db_test_orphan_checkpoint(Lardon3DProjectDb *database); +Lardon3DProjectDbResult lardon3d_project_db_test_delete_catalog_identity( + Lardon3DProjectDb *database, uint64_t scanset_id, uint64_t image_id, + uint64_t asset_id); + +static void +asset_path_for_hash(const unsigned char hash[LARDON3D_PROJECT_DB_SHA256_SIZE], + char path[LARDON3D_PROJECT_DB_PATH_CAPACITY]) +{ + static const char digits[] = "0123456789abcdef"; + char hex[65]; + for (size_t index = 0; index < sizeof(hex) / 2; ++index) { + hex[index * 2] = digits[hash[index] >> 4]; + hex[index * 2 + 1] = digits[hash[index] & 15U]; + } + hex[64] = '\0'; + (void)snprintf(path, LARDON3D_PROJECT_DB_PATH_CAPACITY, + "assets/images/%c%c/%s", hex[0], hex[1], hex); +} static Lardon3DTaskDurableSnapshot task_snapshot(uint64_t id, Lardon3DTaskState saved) @@ -67,7 +85,7 @@ create_future_database(const char *path) sqlite3 *connection = NULL; if (sqlite3_open(path, &connection) != SQLITE_OK) return false; bool ok = sqlite3_exec(connection, "CREATE TABLE metadata(key TEXT PRIMARY KEY,value INTEGER NOT NULL);" - "INSERT INTO metadata VALUES('schema_version',4);", NULL, NULL, NULL) == SQLITE_OK; + "INSERT INTO metadata VALUES('schema_version',5);", NULL, NULL, NULL) == SQLITE_OK; return sqlite3_close(connection) == SQLITE_OK && ok; } @@ -110,6 +128,21 @@ create_v2_database(const char *path) return sqlite3_close(connection) == SQLITE_OK && ok; } +static bool +create_v3_database(const char *path) +{ + if (!create_v2_database(path)) return false; + sqlite3 *connection = NULL; + if (sqlite3_open(path, &connection) != SQLITE_OK) return false; + bool ok = sqlite3_exec(connection, + "CREATE TABLE image_import_tasks(task_id INTEGER PRIMARY KEY REFERENCES tasks(task_id) ON DELETE CASCADE,source_path TEXT NOT NULL);" + "INSERT INTO image_import_tasks VALUES(9,'/legacy/source');" + "INSERT INTO metadata(key,value) VALUES('next_task_id',10);" + "UPDATE metadata SET value=3 WHERE key='schema_version'", + NULL, NULL, NULL) == SQLITE_OK; + return sqlite3_close(connection) == SQLITE_OK && ok; +} + static bool query_integer(const char *path, const char *sql, sqlite3_int64 expected) { @@ -135,7 +168,7 @@ run_test(void) CHECK(mkdtemp(directory)); char database_path[512], artifact_path[512], future_path[512], corrupt_path[512]; char legacy_path[512], failed_migration_path[512], v2_path[512]; - char failed_v3_migration_path[512]; + char failed_v3_migration_path[512], v3_path[512], failed_v4_path[512]; CHECK(snprintf(database_path, sizeof(database_path), "%s/project.db", directory) > 0); CHECK(snprintf(artifact_path, sizeof(artifact_path), "%s/artifact.bin", directory) > 0); CHECK(snprintf(future_path, sizeof(future_path), "%s/future.db", directory) > 0); @@ -145,11 +178,47 @@ run_test(void) CHECK(snprintf(v2_path, sizeof(v2_path), "%s/v2.db", directory) > 0); CHECK(snprintf(failed_v3_migration_path, sizeof(failed_v3_migration_path), "%s/failed-v3-migration.db", directory) > 0); + CHECK(snprintf(v3_path, sizeof(v3_path), "%s/v3.db", directory) > 0); + CHECK(snprintf(failed_v4_path, sizeof(failed_v4_path), + "%s/failed-v4-migration.db", directory) > 0); char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY]; Lardon3DProjectDb *database = NULL; CHECK(lardon3d_project_db_open(database_path, &database, error) == LARDON3D_PROJECT_DB_OK); - CHECK(database && lardon3d_project_db_schema_version(database) == 3); + CHECK(database && lardon3d_project_db_schema_version(database) == 4); + bool legacy_pending = true; + CHECK(lardon3d_project_db_legacy_catalog_pending(database, &legacy_pending) + == LARDON3D_PROJECT_DB_OK && !legacy_pending); + + Lardon3DProjectDbScanSet deleted_scanset; + CHECK(lardon3d_project_db_create_scanset(database, "Deleted identity", + &deleted_scanset) == LARDON3D_PROJECT_DB_OK); + unsigned char first_hash[LARDON3D_PROJECT_DB_SHA256_SIZE] = {1}; + char first_asset_path[LARDON3D_PROJECT_DB_PATH_CAPACITY]; + asset_path_for_hash(first_hash, first_asset_path); + Lardon3DProjectDbImageRegisterStatus identity_status; + Lardon3DProjectDbImage deleted_image; + CHECK(lardon3d_project_db_register_image(database, + deleted_scanset.scanset_id, first_hash, first_asset_path, 1, + "deleted.jpg", "/source/deleted.jpg", 0, 1, &identity_status, + &deleted_image) == LARDON3D_PROJECT_DB_OK); + CHECK(lardon3d_project_db_test_delete_catalog_identity(database, + deleted_scanset.scanset_id, deleted_image.image_id, + deleted_image.asset_id) == LARDON3D_PROJECT_DB_OK); + Lardon3DProjectDbScanSet replacement_scanset; + CHECK(lardon3d_project_db_create_scanset(database, "Replacement identity", + &replacement_scanset) == LARDON3D_PROJECT_DB_OK + && replacement_scanset.scanset_id > deleted_scanset.scanset_id); + unsigned char second_hash[LARDON3D_PROJECT_DB_SHA256_SIZE] = {2}; + char second_asset_path[LARDON3D_PROJECT_DB_PATH_CAPACITY]; + asset_path_for_hash(second_hash, second_asset_path); + Lardon3DProjectDbImage replacement_image; + CHECK(lardon3d_project_db_register_image(database, + replacement_scanset.scanset_id, second_hash, second_asset_path, 1, + "replacement.jpg", "/source/replacement.jpg", 0, 2, + &identity_status, &replacement_image) == LARDON3D_PROJECT_DB_OK + && replacement_image.image_id > deleted_image.image_id + && replacement_image.asset_id > deleted_image.asset_id); Lardon3DProjectDbProject project = {.created_at = 100, .updated_at = 100}; (void)snprintf(project.stable_id, sizeof(project.stable_id), "project-0001"); @@ -174,6 +243,9 @@ run_test(void) CHECK(lardon3d_project_db_record_task(database, &running, "test.work", 1, &checkpoint, 200) == LARDON3D_PROJECT_DB_OK); running.progress = 30; running.sequence_count = 3; CHECK(lardon3d_project_db_record_task(database, &running, "test.work", 1, &checkpoint, 201) == LARDON3D_PROJECT_DB_OK); + Lardon3DProjectDbScanSet import_scanset; + CHECK(lardon3d_project_db_create_scanset(database, "Legacy import", + &import_scanset) == LARDON3D_PROJECT_DB_OK); Lardon3DProjectDbTask task; CHECK(lardon3d_project_db_load_task(database, 1, &task) == LARDON3D_PROJECT_DB_OK); CHECK(task.recovery_state == TASK_PENDING && task.progress == 30 && task.sequence_count == 3); @@ -191,7 +263,8 @@ run_test(void) CHECK(lardon3d_project_db_record_task(database, &running, "test.work", 0, &checkpoint, 202) == LARDON3D_PROJECT_DB_INVALID_ARGUMENT); CHECK(lardon3d_project_db_record_image_import_task(database, &running, - "import.images", 1, &checkpoint, "/tmp/source-a", 202) + "import.images", 1, &checkpoint, "/tmp/source-a", + import_scanset.scanset_id, 202) == LARDON3D_PROJECT_DB_CONSTRAINT); Lardon3DTaskDurableSnapshot completed = task_snapshot(2, TASK_COMPLETED); @@ -206,15 +279,18 @@ run_test(void) Lardon3DTaskDurableSnapshot image_import = task_snapshot(5, TASK_PENDING); CHECK(lardon3d_project_db_record_image_import_task(database, &image_import, - "import.images", 1, &checkpoint, "/tmp/source-a", 202) + "import.images", 1, &checkpoint, "/tmp/source-a", + import_scanset.scanset_id, 202) == LARDON3D_PROJECT_DB_OK); Lardon3DProjectDbImageImport import_parameters; CHECK(lardon3d_project_db_load_image_import(database, 5, &import_parameters) == LARDON3D_PROJECT_DB_OK); CHECK(import_parameters.task_id == 5 - && strcmp(import_parameters.source_path, "/tmp/source-a") == 0); + && strcmp(import_parameters.source_path, "/tmp/source-a") == 0 + && import_parameters.scanset_id == import_scanset.scanset_id); CHECK(lardon3d_project_db_record_image_import_task(database, &image_import, - "import.images", 1, &checkpoint, "/tmp/source-b", 203) + "import.images", 1, &checkpoint, "/tmp/source-b", + import_scanset.scanset_id, 203) == LARDON3D_PROJECT_DB_CONSTRAINT); Lardon3DTaskDurableSnapshot rollback_task = task_snapshot(3, TASK_PENDING); @@ -265,7 +341,7 @@ run_test(void) char too_long[LARDON3D_PROJECT_DB_PATH_CAPACITY + 1]; memset(too_long, 'x', sizeof(too_long)); too_long[sizeof(too_long) - 1] = '\0'; CHECK(lardon3d_project_db_open(too_long, &database, error) == LARDON3D_PROJECT_DB_INVALID_ARGUMENT); lardon3d_project_db_close(contexts[0].database); database = NULL; - CHECK(query_integer(database_path, "SELECT value FROM metadata WHERE key='schema_version'", 3)); + CHECK(query_integer(database_path, "SELECT value FROM metadata WHERE key='schema_version'", 4)); CHECK(query_integer(database_path, "SELECT count(*) FROM tasks WHERE task_id=1", 1)); CHECK(lardon3d_project_db_open(database_path, &database, error) == LARDON3D_PROJECT_DB_OK); CHECK(lardon3d_project_db_load_task(database, 1, &task) == LARDON3D_PROJECT_DB_OK); @@ -280,7 +356,7 @@ run_test(void) CHECK(create_v1_database(legacy_path)); CHECK(lardon3d_project_db_open(legacy_path, &database, error) == LARDON3D_PROJECT_DB_OK); - CHECK(lardon3d_project_db_schema_version(database) == 3); + CHECK(lardon3d_project_db_schema_version(database) == 4); CHECK(lardon3d_project_db_get_project(database, &loaded_project) == LARDON3D_PROJECT_DB_OK && strcmp(loaded_project.stable_id, "legacy-project") == 0); CHECK(lardon3d_project_db_load_task(database, 9, &task) == LARDON3D_PROJECT_DB_OK); @@ -289,7 +365,7 @@ run_test(void) CHECK(lardon3d_project_db_load_artifact(database, "legacy-artifact", &loaded_artifact) == LARDON3D_PROJECT_DB_OK); lardon3d_project_db_close(database); database = NULL; - CHECK(query_integer(legacy_path, "SELECT value FROM metadata WHERE key='schema_version'", 3)); + CHECK(query_integer(legacy_path, "SELECT value FROM metadata WHERE key='schema_version'", 4)); CHECK(create_v1_database(failed_migration_path)); CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V2", "1", 1) == 0); @@ -316,7 +392,7 @@ run_test(void) &loaded_artifact) == LARDON3D_PROJECT_DB_OK); lardon3d_project_db_close(database); database = NULL; CHECK(query_integer(v2_path, - "SELECT value FROM metadata WHERE key='schema_version'", 3)); + "SELECT value FROM metadata WHERE key='schema_version'", 4)); CHECK(create_v2_database(failed_v3_migration_path)); CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V3", "1", 1) == 0); @@ -328,9 +404,37 @@ run_test(void) CHECK(query_integer(failed_v3_migration_path, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='image_import_tasks'", 0)); + CHECK(create_v3_database(v3_path)); + CHECK(lardon3d_project_db_open(v3_path, &database, error) + == LARDON3D_PROJECT_DB_OK); + CHECK(lardon3d_project_db_load_image_import(database, 9, + &import_parameters) == LARDON3D_PROJECT_DB_OK); + CHECK(strcmp(import_parameters.source_path, "/legacy/source") == 0 + && import_parameters.scanset_id > 0); + CHECK(lardon3d_project_db_legacy_catalog_pending(database, &legacy_pending) + == LARDON3D_PROJECT_DB_OK && legacy_pending); + CHECK(lardon3d_project_db_load_task(database, 9, &task) + == LARDON3D_PROJECT_DB_OK && task.has_task_kind); + CHECK(lardon3d_project_db_load_artifact(database, "legacy-artifact", + &loaded_artifact) == LARDON3D_PROJECT_DB_OK); + lardon3d_project_db_close(database); database = NULL; + CHECK(query_integer(v3_path, + "SELECT value FROM metadata WHERE key='schema_version'", 4)); + + CHECK(create_v3_database(failed_v4_path)); + CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V4", "1", 1) == 0); + CHECK(lardon3d_project_db_open(failed_v4_path, &database, error) + == LARDON3D_PROJECT_DB_IO_ERROR); + CHECK(unsetenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V4") == 0); + CHECK(query_integer(failed_v4_path, + "SELECT value FROM metadata WHERE key='schema_version'", 3)); + CHECK(query_integer(failed_v4_path, + "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='scansets'", 0)); + CHECK(unlink(artifact_path) == 0); CHECK(unlink(database_path) == 0); CHECK(unlink(future_path) == 0); CHECK(unlink(corrupt_path) == 0); CHECK(unlink(legacy_path) == 0); CHECK(unlink(failed_migration_path) == 0); CHECK(unlink(v2_path) == 0); CHECK(unlink(failed_v3_migration_path) == 0); + CHECK(unlink(v3_path) == 0); CHECK(unlink(failed_v4_path) == 0); CHECK(rmdir(directory) == 0); return true; }