diff --git a/README.md b/README.md index e8e1e91..5e6a1c5 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Lardon3D ne vise pas simplement "dossier de photos → objet 3D", mais "ensemble - **Image View** : vues triées et filtrées pour la TUI - **Task** : moteur de tâches avec pause/reprise, annulation et séquences - **Task Checkpoint v1** : snapshot durable, fichier atomique et reprise sûre -- **Project Database v5** : tâches, ScanSets, images, Feature Sets/assets et migrations SQLite +- **Project Database v6** : tâches, catalogue, Feature Store et Visual Index segmenté - **Task Kind Registry** : identité métier durable et reconstruction runtime explicite - **Recovery projet** : reprise automatique sélective et bornée des imports récupérables - **Task Queue** : file FIFO avec sélection adaptative et backpressure @@ -39,6 +39,7 @@ Lardon3D ne vise pas simplement "dossier de photos → objet 3D", mais "ensemble - Intégration scheduler ↔ governor avec séquences adaptatives - Documentation architecture +- **Visual Index v1** : LSH ORB persistant, incrémental et validé, avec recherche top-K bornée ### Briques prévues (PLANNED) @@ -104,6 +105,7 @@ Acquisitions - [Persistance](docs/architecture/persistence.md) - [Base de données projet](docs/architecture/project_database.md) - [Feature Store](docs/architecture/feature_store.md) +- [Visual Index](docs/architecture/visual_index.md) - [Viewer](docs/architecture/viewer.md) - [Revue des fondations](docs/architecture/foundation_review.md) diff --git a/docs/architecture/feature_store.md b/docs/architecture/feature_store.md index 14bd4dd..7107abb 100644 --- a/docs/architecture/feature_store.md +++ b/docs/architecture/feature_store.md @@ -88,7 +88,7 @@ publication laisse un fichier orphelin et aucune ligne logique partielle. Un nouvel essai qui revalide l'asset et réussit le `fsync` promeut explicitement sa durabilité vers `DURABLE` dans la transaction DB. -Le schéma v5 sépare `feature_assets`, `feature_sets` et +Le sous-schéma introduit en v5 sépare `feature_assets`, `feature_sets` et `feature_extract_tasks`. L'unicité logique porte sur image, kind, version et fingerprint. Une tâche est persistée avant enqueue, passe par la queue et le Governor (CPU, IO, 576 Mio conservateurs, lot 1), puis checkpointée initialement @@ -106,12 +106,13 @@ ou les octets exige d'auditer et, si nécessaire, d'incrémenter cette version. ## Statut -**IMPLEMENTED** — ORB réel, format v1, assets content-addressed, DB v5, +**IMPLEMENTED** — ORB réel, format v1, assets content-addressed, DB v6, publication atomique, reader borné, task kind production et reprise automatique. **NOT_YET_WIRED** — commande de réconciliation/scrub des orphelins, contrôle fin du backend parallèle OpenCV, orientation EXIF, lancement automatique de l'extraction après import et planification multi-image/DAG. -**PLANNED** — Visual Index, paires candidates, matching, vérification -géométrique, tracks et SfM. +**IMPLEMENTED** — Visual Index v1 consomme ce reader sans changer le format. + +**PLANNED** — paires candidates, matching, vérification géométrique, tracks et SfM. diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 3b568ed..a59bf12 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -68,6 +68,13 @@ binaires content-addressed lisibles par plages bornées. **Statut :** IMPLEMENTED +### Visual Index +Index de retrieval ORB LSH persistant en segments immuables. Il indexe les +Feature Sets par lots bornés et retourne des candidats inter-ScanSets sans +matching géométrique. + +**Statut :** IMPLEMENTED + ### Task Moteur de tâches avec états, progression, pause/reprise coopérative, annulation, checkpoints et estimations de ressources. @@ -128,4 +135,5 @@ frontière de séquence connue. - Absence de pools de workers multiples (CPU/GPU/IO). - La TUI legacy ne sélectionne pas encore explicitement ses ScanSets. - La réconciliation globale des assets/checkpoints orphelins n'est pas implémentée. +- La compaction des segments Visual Index n'est pas implémentée. - Viewer et publication live non implémentés. diff --git a/docs/architecture/persistence.md b/docs/architecture/persistence.md index d59cb88..80b87d1 100644 --- a/docs/architecture/persistence.md +++ b/docs/architecture/persistence.md @@ -108,7 +108,7 @@ par le `time_t` local avant conversion. Le format reste donc lisible entre plateformes uniquement pour les valeurs communes à leurs domaines `size_t` et `time_t`. -## Project Database v5 +## Project Database v6 SQLite contient l'état logique interrogable et les références aux fichiers ; les checkpoints et artefacts volumineux restent externes. L'enregistrement du @@ -147,8 +147,9 @@ code. **IMPLEMENTED** — modèle durable, codec v1, lecture validée, publication atomique et restauration sûre d'une tâche isolée. -**IMPLEMENTED** — Project Database v5 pour identité, tâches typées, ScanSets, -images logiques, Feature Sets/assets SHA-256, checkpoints et artefacts génériques. +**IMPLEMENTED** — Project Database v6 pour identité, tâches typées, ScanSets, +images logiques, Feature Sets/assets SHA-256, Visual Index segmenté, +checkpoints et artefacts génériques. **IMPLEMENTED** — registry statique bornée et reconstruction explicite avec ownership du userdata. @@ -208,5 +209,7 @@ SQLite, scrub des assets et réconciliation globale des orphelins. **IMPLEMENTED** — Feature Store externe versionné, immutable, borné et relié transactionnellement à ses métadonnées SQLite après publication. -**PLANNED** — Visual Index, migrations v6+ et reprise avec -dépendances. +**IMPLEMENTED** — Visual Index externe segmenté, memberships transactionnels +et tâche `visual_index.update` récupérable. + +**PLANNED** — migrations v7+ et reprise avec dépendances. diff --git a/docs/architecture/project_database.md b/docs/architecture/project_database.md index e1d5ea8..347eddd 100644 --- a/docs/architecture/project_database.md +++ b/docs/architecture/project_database.md @@ -168,9 +168,9 @@ référence vers le fichier checkpoint. Le fichier checkpoint validé reste la source complète pour `lardon3d_task_restore()` ; la DB seule ne reconstruit jamais une tâche. Un écart ou un fichier invalide interdit la reprise. -## Schéma v5 implémenté +## Schéma v6 implémenté -- `metadata(key PRIMARY KEY, value)` contient `schema_version=5` et +- `metadata(key PRIMARY KEY, value)` contient `schema_version=6` et `next_task_id`, prochain ID durable allouable. - `project(singleton=1, stable_id UNIQUE, name, created_at, updated_at)` décrit l'unique identité logique de la DB. @@ -195,8 +195,22 @@ jamais une tâche. Un écart ou un fichier invalide interdit la reprise. - `image_import_tasks(task_id PRIMARY KEY REFERENCES tasks ON DELETE CASCADE, source_path, scanset_id REFERENCES scansets)` conserve les paramètres métier immuables de `import.images`. +- `visual_indexes` conserve l'identité `AUTOINCREMENT`, la configuration + Feature homogène, les paramètres LSH et leurs fingerprints. +- `visual_index_segments` conserve identité `AUTOINCREMENT`, génération, + SHA-256, chemin, taille, compteurs, durabilité et tâche productrice. +- `visual_index_memberships` a pour clé primaire + `(visual_index_id,feature_set_id)` et référence le segment immutable. +- `visual_index_update_tasks` conserve `visual_index_id` et le curseur durable. -`AUTOINCREMENT` est volontairement limité à ces trois identités publiées. Il +Les indexes v6 sont +`visual_index_segments(visual_index_id,generation)` et +`visual_index_memberships(visual_index_segment_id,feature_set_id)`. Les FKs +ciblent `visual_indexes`, `feature_sets`, `visual_index_segments` et `tasks`. +Les postings ne sont jamais stockés dans SQLite. + +`AUTOINCREMENT` couvre les identités publiées catalogue, Feature Store et +Visual Index. Il empêche la réutilisation d'un ID issu d'une transaction validée même si sa ligne maximale est supprimée plus tard. Le coût de `sqlite_sequence` est accepté pour garantir qu'un futur Feature Store, match ou track ne voie jamais son identifiant @@ -209,15 +223,15 @@ SHA-256 et le chemin asset sont déjà indexés par leurs contraintes `UNIQUE`. ## Ouverture et migrations -Une DB vide reçoit directement le schéma v5 dans une transaction +Une DB vide reçoit directement le schéma v6 dans une transaction `BEGIN IMMEDIATE`. Une DB v1 reçoit transactionnellement les colonnes nullable `task_kind` et `task_kind_version`, puis les migrations v2→v3. Les anciennes lignes restent `NULL/NULL`, sans type inventé et sans perte des projets, tâches, checkpoints ou artefacts. Une interruption ou erreur provoque un rollback complet. Les DB v1, -v2, v3 et v4 sont migrées séquentiellement vers v5. Une version future est refusée et une DB contenant +v2, v3, v4 et v5 sont migrées séquentiellement vers v6. Une version future est refusée et une DB contenant des tables sans métadonnée de version est considérée corrompue. La fonction -interne de migration ne connaît que `0 → 4`, `1 → 2 → 3 → 4`, -`2 → 3 → 4` et `3 → 4`. +interne de migration applique uniquement la chaîne séquentielle connue jusqu'à +v6 ; une valeur hors de 1..6 est refusée. Migration v1→v2 exacte, exécutée entre `BEGIN IMMEDIATE` et `COMMIT` : @@ -293,7 +307,7 @@ UPDATE metadata SET value=4 WHERE key='schema_version' AND value=3; ``` -Configuration v5 : `foreign_keys=ON`, `journal_mode=DELETE`, +Configuration v6 : `foreign_keys=ON`, `journal_mode=DELETE`, `synchronous=FULL`, `busy_timeout=5000`. Le mode DELETE convient au propriétaire unique actuel, évite les fichiers WAL/SHM durables et conserve la synchronisation forte. Le timeout borne l'attente d'un verrou externe à cinq secondes. @@ -354,7 +368,7 @@ ouvert. ## Statut -**IMPLEMENTED** — SQLite système, schéma v5 et migrations v1→v2→v3→v4→v5, identité +**IMPLEMENTED** — SQLite système, schéma v6 et migrations v1→v2→v3→v4→v5→v6, identité projet, transactions tâche+checkpoint, pagination de reprise et artefacts génériques. @@ -382,7 +396,9 @@ cataloguées », pas « images migrées ». **NOT_YET_WIRED** — autosave à toutes les transitions, retry UI des sources indisponibles, migration de la TUI legacy et réconciliation des fichiers -orphelins et Visual Index. Le Feature Store v1 est implémenté. +orphelins et compaction Visual Index. Feature Store et Visual Index v1 sont implémentés. +Visual Index v1 borne un index à 256 segments de 16 memberships, soit 4096 Feature Sets; +la couverture de 50 000 Feature Sets nécessitera la compaction ou une évolution v2. -**PLANNED** — migrations v5+, dépendances d'artefacts, graphe géométrique et +**PLANNED** — migrations v7+, dépendances d'artefacts, graphe géométrique et reconstruction incrémentale. diff --git a/docs/architecture/reconstruction_pipeline.md b/docs/architecture/reconstruction_pipeline.md index fdb0257..091c17c 100644 --- a/docs/architecture/reconstruction_pipeline.md +++ b/docs/architecture/reconstruction_pipeline.md @@ -63,7 +63,8 @@ content-addressed. Les états Feature/Matching/Reconstruction restent planifiés | **Pipeline conceptuel** | Extraction de features globales → construction de l'index → requête par similarité → retour des K plus proches voisins. | | **Proximité temporelle comme signal secondaire** | Lorsque les images portent un horodatage EXIF, la proximité temporelle sert de signal complémentaire au contenu visuel, mais ne remplace jamais l'analyse visuelle. | -**Statut :** PLANNED — aucun index visuel ni générateur de paires n'existe. +**Statut :** IMPLEMENTED v1 — LSH binaire déterministe, segments immuables, +updates incrémentales et query top-K bornée. Le générateur de paires reste planifié. --- @@ -215,9 +216,9 @@ Image Catalog (B) ──► Feature Store (C) ## Statut du pipeline -Import, Image Catalog, Feature Extraction et Feature Store sont -**IMPLEMENTED**. Visual Index, Candidate Pair, Matching, Tracks et SfM sont -**PLANNED**. +Import, Image Catalog, Feature Extraction et Feature Store sont **IMPLEMENTED**. +Visual Index est implémenté mais reste en validation partielle. Candidate Pair, +Matching, Tracks et SfM sont **PLANNED**. Ce document décrit la vision architecturale cible du pipeline de reconstruction. Les modules listés ici ne sont pas tous implémentés. diff --git a/docs/architecture/resource_governor.md b/docs/architecture/resource_governor.md index 55dcfa3..c3d1653 100644 --- a/docs/architecture/resource_governor.md +++ b/docs/architecture/resource_governor.md @@ -78,6 +78,9 @@ Le Resource Governor est l'unique propriétaire des budgets (RAM, GPU, CPU, IO). chemin actuel sans prétendre mesurer les allocations internes d'OpenCV. `record_batch` couvre la validation source, le décodage, ORB, la publication et la finalisation DB ; `peak_memory_bytes == 0` signifie « mesure inconnue ». +- `visual_index.update` réserve un thread CPU, un slot I/O, 8 Mio fixes et + 2 Mio par Feature Set, par lots de 1 à 16. Le GPU vaut zéro. `record_batch` + compte uniquement les memberships commités et conserve la mémoire inconnue à zéro. ## Limites actuelles diff --git a/docs/architecture/runtime.md b/docs/architecture/runtime.md index f14add6..d60d74f 100644 --- a/docs/architecture/runtime.md +++ b/docs/architecture/runtime.md @@ -115,6 +115,10 @@ conserve le task ID et laisse le worker obtenir une nouvelle réservation. Kinds inconnus, tâches legacy, checkpoints invalides et sources absentes ne bloquent pas l'ouverture. +**IMPLEMENTED** — `visual_index.update` reprend à la dernière membership +commitée. Un segment temporaire n'est jamais visible et un rejeu exclut les +Feature Sets déjà membres. + ## Invariants - ncurses appartient exclusivement au thread principal diff --git a/docs/architecture/task_kind_registry.md b/docs/architecture/task_kind_registry.md index 47f1a2d..33bf174 100644 --- a/docs/architecture/task_kind_registry.md +++ b/docs/architecture/task_kind_registry.md @@ -26,7 +26,7 @@ fin de l'exécution. Le constructeur métier n'est jamais appelé sous mutex DB. ## Persistance et legacy -Le checkpoint générique reste en version 1. Project Database v5 conserve le +Le checkpoint générique reste en version 1. Project Database v6 conserve le kind/version ; les lignes migrées depuis v1 restent `NULL/NULL` et sont classées `LEGACY_UNTYPED`. Un kind inconnu ou une version non supportée reste inspectable mais inexécutable. Aucun type n'est inventé et aucun code n'est sélectionné par @@ -49,5 +49,7 @@ pour restaurer hors mutex DB et transférer chaque tâche acceptée à la queue. **IMPLEMENTED** — `features.extract` version 1 reconstruit une extraction ORB depuis `image_id` et ses paramètres bornés. -**PLANNED** — kinds des tâches Visual -Index et reconstruction lorsque ces traitements existeront réellement. +**IMPLEMENTED** — `visual_index.update`, version 1, recharge +`visual_index_id + after_feature_set_id` et reconstruit un contexte neuf. + +**PLANNED** — kinds de matching et reconstruction. diff --git a/docs/architecture/task_system.md b/docs/architecture/task_system.md index 2bd24ff..35fb27f 100644 --- a/docs/architecture/task_system.md +++ b/docs/architecture/task_system.md @@ -88,6 +88,9 @@ l'ouverture du projet. **NOT_YET_WIRED** — autosave générique et dépendances entre tâches. +**IMPLEMENTED** — `visual_index.update` traite jusqu'à seize Feature Sets par +séquence, checkpoint après commit de segment et repasse par le Governor. + Le chemin de production de l'import ne possède plus de thread ni de drapeau d'annulation privés. Son wrapper TUI ne fait qu'enqueue/cancel/observer la tâche générique. Chaque callback traite un lot borné, checkpoint hors mutex de @@ -106,7 +109,7 @@ localement : son userdata est détruit, sans callback terminal ni écriture durable d'une fausse annulation. Une annulation explicitement demandée conserve le contrat de notification terminale. -La Project Database v5 peut enregistrer transactionnellement un résumé +La Project Database v6 peut enregistrer transactionnellement un résumé `Lardon3DTaskDurableSnapshot` et la référence de son checkpoint. Elle ne stocke ni estimation sérialisée complète, ni callback, ni réservation, et ne remplace pas la validation du fichier checkpoint avant `task_restore()`. diff --git a/docs/architecture/visual_index.md b/docs/architecture/visual_index.md new file mode 100644 index 0000000..5d0c634 --- /dev/null +++ b/docs/architecture/visual_index.md @@ -0,0 +1,185 @@ +# Visual Index v1 + +## Problème et frontière + +Le Visual Index transforme une collection homogène de `FeatureSet` READY en +candidats de recherche. Il consomme exclusivement `feature_set_id` et les +descripteurs ORB lus par le Feature Reader. Il ne fait ni matching final, ni +ratio test, ni vérification géométrique. + +## Choix algorithmique + +La v1 utilise un LSH binaire déterministe à six tables. Chaque table extrait +24 positions distinctes des 256 bits ORB. La position v1 est +`(41*table + 11*bit) mod 256`; 11 étant premier avec 256, les 24 positions +d'une table sont distinctes. Une clé est `(table_id, key24)`. Des descripteurs proches en +Hamming ont une probabilité élevée de collision dans au moins une table, sans +conversion flottante. + +Alternatives évaluées : + +- le hash exact est très compact et déterministe, mais son rappel s'effondre + dès qu'un descriptor varie d'un bit ; +- le multi-index hashing avec sous-chaînes et multiprobes offre des garanties + Hamming intéressantes, mais le nombre de postings/probes nécessaire au + rappel utile d'ORB est trop élevé pour une v1 bornée ; +- FLANN-LSH masque son format, ses allocations et sa stabilité de + sérialisation, ce qui nuit à la reprise et à l'audit ; +- BoW/IVF donne un bon retrieval image, mais impose vocabulaire, entraînement, + identité et politique de mise à jour avant de pouvoir être incrémental ; +- HNSW et FAISS ajoutent une dépendance et un état mutable complexes sans + avantage décisif à quelques milliers d'images. + +Ce LSH n'est pas un matcher. Il privilégie une base déterministe, segmentable +et contrôlable. Une évolution de la sélection de bits exige une nouvelle +`visual_index_version`. + +## Identité et configuration + +Le kind est `orb-lsh`, version 1. Un index contient exclusivement des Feature +Sets de même `descriptor_type`, dimension, `extractor_kind`, version et +`parameter_fingerprint`. Sa configuration v1 contient : + +- `table_count=6` ; +- `key_bits=24` ; +- `max_features_per_set` entre 1 et 1024, défaut 512 ; +- `max_bucket_postings` entre 1 et 4096, défaut 256 ; +- `max_segments=256` ; +- `max_feature_sets_per_segment=16`. + +Le fingerprint de paramètres est SHA-256 des 32 octets canoniques +`L3DVICF1`, version et cinq entiers little-endian. Aucun padding, JSON, locale +ou endianness hôte n'intervient. `visual_index_id` est une identité SQLite +`AUTOINCREMENT`, jamais réutilisée après publication. + +## Échantillonnage + +Au plus `max_features_per_set` features sont indexées. La sélection v1 retient +le préfixe de `feature_index` croissant. Les postings conservent l'indice +original. Un Feature Set vide est +membre valide sans posting. Le build ne garde qu'un Feature Set et une tranche +Feature Reader de 256 descripteurs en mémoire. + +## Segments et Project Database v6 + +Un index logique possède des segments immuables READY. Chaque update publie un +segment de un à seize nouveaux Feature Sets, puis ajoute atomiquement segments +et memberships. `UNIQUE(visual_index_id,feature_set_id)` assure l'idempotence. +Une recherche copie au début la liste bornée des segments READY, relâche le +mutex DB, puis lit ce snapshot. Un segment commité au milieu sera visible à la +requête suivante. + +SQLite conserve les tables `visual_indexes`, `visual_index_segments`, +`visual_index_memberships` et `visual_index_update_tasks`. Les gros postings +restent hors DB. La configuration et chaque membership sont immuables. La +compaction est `NOT_YET_WIRED`; au-delà de 256 segments une update est refusée avec +`LARDON3D_VISUAL_INDEX_LIMIT`. Avec seize membres par segment, la capacité v1 est donc +exactement 4096 Feature Sets par index. Le refus ne publie ni segment ni membership et +l'index existant reste requêtable. + +Le DDL v6 exact est `schema_visual_v6` dans `src/project_db.c`. Il impose +`AUTOINCREMENT` aux index/segments, les uniques +`(visual_index_id,generation)`, `(visual_index_id,sha256)` et +`(visual_index_id,feature_set_id)`, ainsi que les FKs vers index, Feature Set, +segment et tâche. Les CHECKS bornent tables 1..32, bits 8..32, sampling +1..1024, bucket 1..4096, membres segment 1..16 et durabilité 0..1. La migration +entière reste sous `BEGIN IMMEDIATE` et possède une injection de rollback v6. + +## Segment File v1 + +Le fichier est little-endian et ne sérialise aucune structure C. Layout : + +| Offset | Taille | Champ | +|---:|---:|---| +| 0 | 8 | magic `L3DVIDX\0` | +| 8 | 4 | format version 1 | +| 12 | 4 | header size 128 | +| 16 | 4 | table count | +| 20 | 4 | key bits | +| 24 | 8 | posting count | +| 32 | 8 | member count | +| 40 | 8 | postings offset, 128 | +| 48 | 8 | total size | +| 56 | 32 | index parameter fingerprint | +| 88 | 32 | feature parameter fingerprint | +| 120 | 8 | réservés, zéro | + +Chaque posting fait 24 octets : `table_id:u32`, `key24:u32`, +`feature_set_id:u64`, `feature_index:u32`, réservé zéro `u32`. L'ordre est +`table_id`, clé, Feature Set, feature index. Le fichier exact est SHA-256 et +vit sous `assets/visual-index/<2 hex>/`. + +Publication : temporaire local, écriture, `fsync`, hash, `link` sans +écrasement, validation d'une adoption concurrente, `fsync` du répertoire, puis +transaction DB. Un échec après publication peut laisser un orphelin mais jamais +un segment READY partiel. La durabilité distingue `DURABLE` et +`PUBLISHED_NOT_DURABLE`. + +Le reader vérifie le SHA avant le parsing. Un fichier au SHA et aux métadonnées cohérents +mais portant une version future produit `UNSUPPORTED_VERSION`; les comptes et produits +d'offset invalides produisent `CORRUPT` avant allocation, conversion ou lecture de posting. + +## Recherche, score et bornes + +L'API est centrée sur `(visual_index_id, query_feature_set_id)`. Elle accepte +`ANY_SCANSET`, `SAME_SCANSET` ou `OTHER_SCANSETS`, l'exclusion du même asset, +un minimum de preuves et `top_k` entre 1 et 256. Elle ne retourne jamais le +Feature Set ni l'image de requête. + +Une preuve est un `feature_index` de requête distinct ayant au moins une +collision avec le candidat. Plusieurs tables, postings ou descriptors du +candidat ne multiplient pas cette preuve. Le score final vaut +`evidence_count / sampled_query_feature_count` dans `[0,1]`. Le volume du +candidat ne peut donc pas augmenter le score sans preuve distincte. L'ordre est +score décroissant, preuves décroissantes, `image_id`, puis `feature_set_id`. + +La burstiness est bornée par une contribution maximum par feature de requête +et candidat. Une première passe additionne la fréquence d'un bucket sur tous +les segments du snapshot. Au-delà de `max_bucket_postings`, il est ignoré : un motif +très commun ne peut ni allouer une liste géante ni dominer le score. Le reader +lit au plus 256 postings par appel. L'accumulateur contient au plus 4096 +candidats ; les nouveaux candidats sont ignorés après saturation, de manière +déterministe par l'ordre des postings. Aucun cache global n'existe et un seul +segment est ouvert à la fois. + +Deux updates concurrentes peuvent sélectionner le même lot et construire le même asset. +La transaction SQLite et les contraintes uniques ne laissent publier qu'un segment et +un membership par Feature Set; l'autre update échoue/rejoue en no-op. Une query prend son +snapshot de métadonnées avant les lectures et ouvre/ferme un seul segment à la fois, y +compris avec 250 à 256 segments : le nombre de descripteurs de fichier reste borné. + +## Tâche, reprise et ressources + +`visual_index.update`, version 1, persiste `visual_index_id` et un curseur +`after_feature_set_id`. Une séquence traite au plus seize Feature Sets non +indexés, publie et commit un segment, puis checkpoint. Pause et annulation sont +coopératives entre lectures et avant publication ; un segment déjà READY reste +valide. La reprise recommence au dernier curseur commité et l'unicité des +memberships rend le rejeu idempotent. + +L'estimation réserve un thread CPU, un slot I/O, GPU zéro, 8 Mio fixes et 2 Mio +par Feature Set, lot 1..16. `record_batch` reçoit le nombre de Feature Sets +réellement commités, la durée réelle et `peak_memory_bytes=0` (inconnue). + +## Complexité et limites + +Pour `D` descriptors échantillonnés, construction et disque sont `O(6D)`. +Une requête effectue `O(6Q log P + H)` par segment (`Q<=1024`, `H` hits bornés), +pas `O(images²)`. La mémoire build est bornée par huit métadonnées, 256 +descripteurs et les postings d'un segment; la mémoire query par 4096 candidats, +256 postings et 256 résultats. À 3700 images et 512 features, environ 11,4 +millions de postings sont produits. Un test structurel persiste 50 000 Feature Sets puis +confirme la pagination par 16 et le refus propre après 4096 memberships. Un index unique +ne couvre donc pas encore 50 000 images : le risque principal est le nombre de segments +et les seeks. Une compaction/base+delta ou une évolution v2 sera nécessaire, sans changer +les identités durables; elle est `NOT_YET_WIRED`. + +Les fixtures de validation incluent un Feature Set vide, un crop réel, une rotation de +8 degrés, deux campagnes, un asset source partagé et une attaque de motif répétitif. Ces +tests valident le classement de candidats LSH, jamais une compatibilité géométrique. + +## Frontière future + +Le Candidate Pair Generator pourra filtrer sur score et `evidence_count`, puis +transmettre `feature_set_id + feature_index` au futur matcher. Le score Visual +Index ne constitue jamais une preuve géométrique. diff --git a/docs/concepts/matching_and_tracks.md b/docs/concepts/matching_and_tracks.md index 12c92e0..bc62a1a 100644 --- a/docs/concepts/matching_and_tracks.md +++ b/docs/concepts/matching_and_tracks.md @@ -8,7 +8,8 @@ Le matching transforme les features individuelles en relations inter-images. Les ## Statut -**PLANNED** — Étape critique du pipeline, pas encore implémentée. Nécessite le Visual Index comme prérequis. +**PLANNED** — Étape critique du pipeline, pas encore implémentée. Visual Index +v1 fournit désormais les candidats, leur score et leurs preuves distinctes. ## Place dans le pipeline @@ -28,6 +29,10 @@ Reconstruction Layers (triangulation) Le matching est le pont entre les caractéristiques 2D des images et la structure 3D de la scène. +Le score Visual Index est un signal de retrieval. Il ne constitue ni un match +descriptor-descriptor final, ni une preuve épipolaire. La chaîne reste : +candidate Visual Index → futur matching → vérification géométrique → tracks. + La génération future de paires candidates combinera `image_id`, appartenance au ScanSet, résultats du Visual Index et provenance. Une proximité temporelle pourra servir de signal secondaire ; la proximité dans un dossier et le nom de diff --git a/docs/concepts/scan_sets.md b/docs/concepts/scan_sets.md index 36e7cb6..eff4942 100644 --- a/docs/concepts/scan_sets.md +++ b/docs/concepts/scan_sets.md @@ -116,5 +116,8 @@ legacy. La migration de la TUI vers les pages SQLite reste donc nécessaire. vue mémoire, vérification/scrub des assets et réconciliation globale des orphelins. -**PLANNED** — Feature Store, Visual Index, paires candidates, matching, tracks, -SfM, MVS et relations géométriques entre ScanSets. +**IMPLEMENTED** — Feature Store et Visual Index peuvent relier visuellement +des images de ScanSets différents sans dépendre de leur nom ou ordre. + +**PLANNED** — paires candidates, matching, tracks, SfM, MVS et relations +géométriques entre ScanSets. diff --git a/docs/concepts/visual_index.md b/docs/concepts/visual_index.md index 9b68ae4..c72c29c 100644 --- a/docs/concepts/visual_index.md +++ b/docs/concepts/visual_index.md @@ -8,7 +8,17 @@ L'index visuel transforme chaque image en un vecteur de descripteurs (features) ## Statut -**IDEA** — Concept fondamental pour l'étape de matching. Pas encore implémenté. Actuellement, le pipeline de photogrammétrie classique utilise une approche exhaustive ou semi-exhaustive qui ne passe pas à l'échelle. +**IMPLEMENTED v1** — Lardon3D utilise un LSH binaire déterministe en segments +persistants. Une image ajoutée lors d'une campagne ultérieure peut retrouver +une ancienne image par contenu, même dans un autre ScanSet et loin d'elle dans +le dossier. Le score indique une priorité de comparaison, jamais une validité +géométrique. + +Les tests de retrieval passent par l'import, l'extraction ORB et l'index réel : +crop significatif, rotation modérée et recherche inter-ScanSets. Un asset image +partagé est signalé par `same_image_asset` et peut être explicitement exclu. +Les buckets trop communs sont ignorés afin qu'un motif répétitif banal ne domine +pas des détails distincts. ## Place dans le pipeline @@ -82,7 +92,7 @@ Image → [Feature 1, Feature 2, ..., Feature N] - L'index doit tenir en mémoire pour un scan set complet typique (100-1000 images). - La construction de l'index ne doit pas bloquer la TUI : tâche asynchrone avec progression. -- L'index est reconstruction-only : il n'est pas persisté entre les sessions (reconstruit à la demande). +- L'index et ses memberships sont persistants ; seuls les buffers de requête sont transitoires. - La taille maximale de l'index est bornée par le budget RAM du Resource Governor. - Les descripteurs doivent être calculés de manière déterministe (même image → même index). diff --git a/docs/development/concurrency.md b/docs/development/concurrency.md index 94c09f1..7793d08 100644 --- a/docs/development/concurrency.md +++ b/docs/development/concurrency.md @@ -156,6 +156,11 @@ task_destroy(task); ## Validation +Les readers Visual Index sont sans état partagé mutable. Une query copie la +liste bornée des segments sous le mutex DB, puis effectue hash, lectures et +accumulation après déverrouillage. Un update ne rend le nouveau segment visible +qu'au commit memberships+segment ; une query en cours garde son snapshot. + Pour tout ticket touchant la concurrence, exécuter : ```sh diff --git a/docs/development/testing.md b/docs/development/testing.md index 1d85c9b..0d66338 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -74,6 +74,11 @@ int main(int argc, char **argv) { ## Tests unitaires vs tests d'intégration +`test-visual-index` couvre les descriptors synthétiques, le retrieval ORB réel, +les filtres inter-ScanSets, quatre queries concurrentes, la corruption/absence/ +troncature d'un segment et 4 000 Feature Sets synthétiques. Le scénario de +reprise `visual_index.update` est exercé dans `test-feature-task`. + | Type | Portée | Fichier | |---|---|---| | Unitaire | Un module isolé | `tests/test_.c` | diff --git a/docs/roadmap/roadmap.md b/docs/roadmap/roadmap.md index 7cb1995..de0fdf0 100644 --- a/docs/roadmap/roadmap.md +++ b/docs/roadmap/roadmap.md @@ -29,7 +29,7 @@ Lardon3D suit une feuille de route ordonnée qui privilégie la stabilité et la ### Phase 3 : Persistance - ✅ Fondation versionnée des checkpoints de tâches -- ✅ Project Database v5 (tâches, ScanSets, images, Feature Sets/assets et migrations) +- ✅ Project Database v6 (tâches, catalogue, Feature Store et Visual Index) - ✅ Branchement Project Database au cycle de vie projet et inventaire de reprise - ✅ Registry durable des types métier de tâches - ✅ Premier type métier reconstructible (`import.images`) @@ -38,7 +38,7 @@ Lardon3D suit une feuille de route ordonnée qui privilégie la stabilité et la ### Phase 4 : Pipeline avancé - ✅ Feature Store v1 et extraction ORB persistante -- 📋 Visual Index +- ✅ Visual Index v1 segmenté et persistant - 📋 Candidate Pair Generator - 📋 Matching et vérification géométrique - 📋 Tracks et SfM diff --git a/include/lardon3d/project.h b/include/lardon3d/project.h index 0848f81..de409d9 100644 --- a/include/lardon3d/project.h +++ b/include/lardon3d/project.h @@ -61,6 +61,9 @@ lardon3d_project_checkpoint_image_import_task(Lardon3DAppState *state, const Lar Lardon3DProjectTaskCheckpointResult lardon3d_project_checkpoint_feature_extract_task( Lardon3DAppState *state, const Lardon3DTask *task, const Lardon3DProjectDbFeatureExtractTask *parameters); +Lardon3DProjectTaskCheckpointResult lardon3d_project_checkpoint_visual_index_update_task( + Lardon3DAppState *state, const Lardon3DTask *task, + const Lardon3DProjectDbVisualIndexUpdateTask *parameters); Lardon3DProjectDbResult lardon3d_project_list_recoverable(Lardon3DAppState *state, const Lardon3DTaskKindRegistry *registry, uint64_t after_task_id, diff --git a/include/lardon3d/project_db.h b/include/lardon3d/project_db.h index 0ea185a..4a975e6 100644 --- a/include/lardon3d/project_db.h +++ b/include/lardon3d/project_db.h @@ -9,7 +9,7 @@ #include enum { - LARDON3D_PROJECT_DB_SCHEMA_VERSION = 5, + LARDON3D_PROJECT_DB_SCHEMA_VERSION = 6, LARDON3D_PROJECT_DB_ID_CAPACITY = 65, LARDON3D_PROJECT_DB_KIND_CAPACITY = 65, LARDON3D_PROJECT_DB_PATH_CAPACITY = 4096, @@ -169,6 +169,49 @@ typedef struct { unsigned char parameter_fingerprint[32]; } Lardon3DProjectDbFeatureExtractTask; +typedef enum { + LARDON3D_DB_VISUAL_INDEX_DURABLE = 0, + LARDON3D_DB_VISUAL_INDEX_PUBLISHED_NOT_DURABLE = 1 +} Lardon3DProjectDbVisualIndexDurability; + +typedef struct { + uint64_t visual_index_id; + char index_kind[LARDON3D_PROJECT_DB_KIND_CAPACITY]; + uint32_t index_version; + uint32_t descriptor_type; + uint32_t descriptor_dimension; + char extractor_kind[LARDON3D_PROJECT_DB_KIND_CAPACITY]; + uint32_t extractor_version; + unsigned char feature_parameter_fingerprint[32]; + unsigned char index_parameter_fingerprint[32]; + uint32_t table_count; + uint32_t key_bits; + uint32_t max_features_per_set; + uint32_t max_bucket_postings; + int64_t created_at; +} Lardon3DProjectDbVisualIndex; + +typedef struct { + uint64_t visual_index_segment_id; + uint64_t visual_index_id; + uint64_t generation; + unsigned char sha256[32]; + char path[LARDON3D_PROJECT_DB_PATH_CAPACITY]; + uint64_t size_bytes; + uint64_t posting_count; + uint32_t feature_set_count; + Lardon3DProjectDbVisualIndexDurability durability; + bool has_producer_task; + uint64_t producer_task_id; + int64_t created_at; +} Lardon3DProjectDbVisualIndexSegment; + +typedef struct { + uint64_t task_id; + uint64_t visual_index_id; + uint64_t after_feature_set_id; +} Lardon3DProjectDbVisualIndexUpdateTask; + Lardon3DProjectDbResult lardon3d_project_db_open(const char *path, Lardon3DProjectDb **database, char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY]); void lardon3d_project_db_close(Lardon3DProjectDb *database); @@ -263,5 +306,29 @@ Lardon3DProjectDbResult lardon3d_project_db_list_feature_sets(Lardon3DProjectDb *database, uint64_t after_feature_set_id, Lardon3DProjectDbFeatureSet *feature_sets, size_t capacity, size_t *count); +Lardon3DProjectDbResult lardon3d_project_db_create_visual_index( + Lardon3DProjectDb *database, const Lardon3DProjectDbVisualIndex *configuration, + Lardon3DProjectDbVisualIndex *visual_index); +Lardon3DProjectDbResult lardon3d_project_db_load_visual_index( + Lardon3DProjectDb *database, uint64_t visual_index_id, + Lardon3DProjectDbVisualIndex *visual_index); +Lardon3DProjectDbResult lardon3d_project_db_list_visual_index_segments( + Lardon3DProjectDb *database, uint64_t visual_index_id, uint64_t after_generation, + Lardon3DProjectDbVisualIndexSegment *segments, size_t capacity, size_t *count); +Lardon3DProjectDbResult lardon3d_project_db_list_visual_index_pending( + Lardon3DProjectDb *database, uint64_t visual_index_id, uint64_t after_feature_set_id, + Lardon3DProjectDbFeatureSet *feature_sets, size_t capacity, size_t *count); +Lardon3DProjectDbResult lardon3d_project_db_publish_visual_index_segment( + Lardon3DProjectDb *database, const Lardon3DProjectDbVisualIndexSegment *segment, + const uint64_t *feature_set_ids, size_t feature_set_count, + Lardon3DProjectDbVisualIndexSegment *published); +Lardon3DProjectDbResult lardon3d_project_db_record_visual_index_update_task( + Lardon3DProjectDb *database, const Lardon3DTaskDurableSnapshot *snapshot, + const char *task_kind, uint32_t task_kind_version, + const Lardon3DProjectDbCheckpoint *checkpoint, + const Lardon3DProjectDbVisualIndexUpdateTask *parameters, int64_t updated_at); +Lardon3DProjectDbResult lardon3d_project_db_load_visual_index_update_task( + Lardon3DProjectDb *database, uint64_t task_id, + Lardon3DProjectDbVisualIndexUpdateTask *parameters); #endif diff --git a/include/lardon3d/visual_index.h b/include/lardon3d/visual_index.h new file mode 100644 index 0000000..a657f07 --- /dev/null +++ b/include/lardon3d/visual_index.h @@ -0,0 +1,91 @@ +#ifndef LARDON3D_VISUAL_INDEX_H +#define LARDON3D_VISUAL_INDEX_H + +#include +#include +#include + +#include +#include + +#define LARDON3D_VISUAL_INDEX_KIND "orb-lsh" + +enum { + LARDON3D_VISUAL_INDEX_VERSION = 1, + LARDON3D_VISUAL_INDEX_TABLE_COUNT = 6, + LARDON3D_VISUAL_INDEX_KEY_BITS = 24, + LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX = 16, + LARDON3D_VISUAL_INDEX_SEGMENT_MAX = 256, + LARDON3D_VISUAL_INDEX_TOP_K_MAX = 256, + LARDON3D_VISUAL_INDEX_CANDIDATE_MAX = 4096, + LARDON3D_VISUAL_INDEX_POSTING_READ_MAX = 256, +}; + +typedef enum { + LARDON3D_VISUAL_INDEX_OK = 0, + LARDON3D_VISUAL_INDEX_NO_CHANGE, + LARDON3D_VISUAL_INDEX_INVALID_ARGUMENT, + LARDON3D_VISUAL_INDEX_NOT_FOUND, + LARDON3D_VISUAL_INDEX_INCOMPATIBLE, + LARDON3D_VISUAL_INDEX_UNSUPPORTED_VERSION, + LARDON3D_VISUAL_INDEX_CORRUPT, + LARDON3D_VISUAL_INDEX_LIMIT, + LARDON3D_VISUAL_INDEX_IO_ERROR, + LARDON3D_VISUAL_INDEX_DB_BUSY, + LARDON3D_VISUAL_INDEX_DB_ERROR, + LARDON3D_VISUAL_INDEX_PUBLISHED_NOT_DURABLE +} Lardon3DVisualIndexResult; + +typedef struct { + uint32_t version; + uint32_t max_features_per_set; + uint32_t max_bucket_postings; +} Lardon3DVisualIndexConfiguration; + +typedef enum { + LARDON3D_VISUAL_INDEX_ANY_SCANSET = 0, + LARDON3D_VISUAL_INDEX_SAME_SCANSET, + LARDON3D_VISUAL_INDEX_OTHER_SCANSETS +} Lardon3DVisualIndexScanSetFilter; + +typedef struct { + uint32_t top_k; + uint32_t minimum_evidence_count; + Lardon3DVisualIndexScanSetFilter scanset_filter; + bool exclude_same_asset; +} Lardon3DVisualIndexQueryOptions; + +typedef struct { + uint64_t feature_set_id; + uint64_t image_id; + uint64_t scanset_id; + double score; + uint32_t evidence_count; + bool same_image_asset; +} Lardon3DVisualIndexCandidate; + +bool lardon3d_visual_index_configuration_valid( + const Lardon3DVisualIndexConfiguration *configuration); +void lardon3d_visual_index_configuration_fingerprint( + const Lardon3DVisualIndexConfiguration *configuration, unsigned char fingerprint[32]); +Lardon3DVisualIndexResult lardon3d_visual_index_create( + Lardon3DProjectDb *database, const Lardon3DProjectDbFeatureSet *prototype, + const Lardon3DVisualIndexConfiguration *configuration, uint64_t *visual_index_id); +Lardon3DVisualIndexResult lardon3d_visual_index_update_once( + const char *project_path, Lardon3DProjectDb *database, uint64_t visual_index_id, + uint64_t producer_task_id, uint64_t after_feature_set_id, size_t maximum_feature_sets, + uint64_t *last_feature_set_id, size_t *indexed_count); +#ifdef LARDON3D_VISUAL_INDEX_TESTING +typedef void (*Lardon3DVisualIndexAfterSelectHook)(void *userdata); +Lardon3DVisualIndexResult lardon3d_visual_index_test_update_once( + const char *project_path, Lardon3DProjectDb *database, uint64_t visual_index_id, + uint64_t producer_task_id, uint64_t after_feature_set_id, size_t maximum_feature_sets, + uint64_t *last_feature_set_id, size_t *indexed_count, + Lardon3DVisualIndexAfterSelectHook after_select, void *hook_userdata); +#endif +Lardon3DVisualIndexResult lardon3d_visual_index_query( + const char *project_path, Lardon3DProjectDb *database, uint64_t visual_index_id, + uint64_t query_feature_set_id, const Lardon3DVisualIndexQueryOptions *options, + Lardon3DVisualIndexCandidate *results, size_t capacity, size_t *result_count); + +#endif diff --git a/include/lardon3d/visual_index_task.h b/include/lardon3d/visual_index_task.h new file mode 100644 index 0000000..3a91666 --- /dev/null +++ b/include/lardon3d/visual_index_task.h @@ -0,0 +1,22 @@ +#ifndef LARDON3D_VISUAL_INDEX_TASK_H +#define LARDON3D_VISUAL_INDEX_TASK_H + +#include +#include + +#include +#include + +#define LARDON3D_VISUAL_INDEX_UPDATE_TASK_KIND "visual_index.update" +enum { LARDON3D_VISUAL_INDEX_UPDATE_TASK_KIND_VERSION = 1 }; + +Lardon3DTask *lardon3d_project_create_visual_index_update_task( + Lardon3DAppState *state, uint64_t visual_index_id, uint64_t *task_id); +bool lardon3d_project_enqueue_visual_index_update(Lardon3DAppState *state, + uint64_t visual_index_id, + uint64_t *task_id); +bool lardon3d_visual_index_update_reconstruct(const Lardon3DTaskDurableSnapshot *snapshot, + void *context, + Lardon3DTaskKindBinding *binding); + +#endif diff --git a/meson.build b/meson.build index eea062f..4f22cab 100644 --- a/meson.build +++ b/meson.build @@ -23,7 +23,12 @@ ncursesw = dependency('ncursesw', required: true) threads = dependency('threads') sqlite3 = dependency('sqlite3', required: true) openssl = dependency('openssl', required: true) -opencv = dependency('opencv5', required: true, include_type: 'system', modules: ['opencv_core', 'opencv_imgcodecs', 'opencv_features2d']) +opencv = dependency( + 'opencv5', + required: true, + include_type: 'system', + modules: ['opencv_core', 'opencv_imgcodecs', 'opencv_features2d'], +) executable( 'lardon3d', @@ -40,6 +45,8 @@ executable( 'src/feature_extractor_opencv.cpp', 'src/feature_store.c', 'src/feature_task.c', + 'src/visual_index.c', + 'src/visual_index_task.c', 'src/image_view.c', 'src/project.c', 'src/project_db.c', @@ -89,6 +96,8 @@ import_task_test = executable( 'src/task_kinds.c', 'src/feature_task.c', 'src/feature_store.c', + 'src/visual_index.c', + 'src/visual_index_task.c', 'src/feature_extractor_opencv.cpp', 'src/task_queue.c', 'src/resource_governor.c', @@ -164,6 +173,23 @@ feature_store_test = executable( test('feature-store', feature_store_test, timeout: 60) +visual_index_test = executable( + 'test-visual-index', + sources: [ + 'tests/test_visual_index.c', 'src/app_state.c', + 'src/image_catalog_persistent.c', 'src/feature_store.c', + 'src/feature_extractor_opencv.cpp', + 'src/visual_index.c', 'src/project_db.c', 'src/task.c', + 'src/resource_governor.c', 'src/resource_snapshot.c', + ], + c_args: [ + '-DLARDON3D_PROJECT_DB_TESTING', '-DLARDON3D_VISUAL_INDEX_TESTING', + ], + include_directories: include_directories('include'), + dependencies: [threads, sqlite3, openssl, opencv], +) +test('visual-index', visual_index_test, timeout: 60) + feature_task_test = executable( 'test-feature-task', sources: [ @@ -172,11 +198,15 @@ feature_task_test = executable( 'src/task_kind_registry.c', 'src/task_kinds.c', 'src/task_queue.c', 'src/import.c', 'src/import_task.c', 'src/image_catalog.c', 'src/image_catalog_persistent.c', 'src/image_view.c', - 'src/feature_task.c', 'src/feature_store.c', + 'src/feature_task.c', 'src/feature_store.c', 'src/visual_index.c', + 'src/visual_index_task.c', 'src/feature_extractor_opencv.cpp', 'src/resource_governor.c', 'src/resource_snapshot.c', ], - c_args: ['-DLARDON3D_PROJECT_DB_TESTING', '-DLARDON3D_FEATURE_TASK_TESTING'], + c_args: [ + '-DLARDON3D_PROJECT_DB_TESTING', '-DLARDON3D_FEATURE_TASK_TESTING', + '-DLARDON3D_VISUAL_INDEX_TASK_TESTING', + ], include_directories: include_directories('include'), dependencies: [threads, sqlite3, openssl, opencv], ) @@ -254,6 +284,8 @@ project_test = executable( 'src/task_kinds.c', 'src/feature_task.c', 'src/feature_store.c', + 'src/visual_index.c', + 'src/visual_index_task.c', 'src/feature_extractor_opencv.cpp', 'src/task_queue.c', 'src/import.c', diff --git a/src/project.c b/src/project.c index 1aee256..21ea2d3 100644 --- a/src/project.c +++ b/src/project.c @@ -669,7 +669,8 @@ static bool checkpoint_paths(const Lardon3DAppState *state, uint64_t task_id, static Lardon3DProjectTaskCheckpointResult checkpoint_task_internal(Lardon3DAppState *state, const Lardon3DTask *task, const char *image_import_source, uint64_t image_import_scanset_id, - const Lardon3DProjectDbFeatureExtractTask *feature_parameters) { + const Lardon3DProjectDbFeatureExtractTask *feature_parameters, + const Lardon3DProjectDbVisualIndexUpdateTask *visual_parameters) { if (!state || !state->project_loaded || !state->project_db) { return LARDON3D_PROJECT_TASK_CHECKPOINT_NO_PROJECT; } @@ -716,6 +717,10 @@ checkpoint_task_internal(Lardon3DAppState *state, const Lardon3DTask *task, ? lardon3d_project_db_record_feature_extract_task(state->project_db, &snapshot, task_kind, task_kind_version, &checkpoint, feature_parameters, now.tv_sec) + : visual_parameters + ? lardon3d_project_db_record_visual_index_update_task( + state->project_db, &snapshot, task_kind, task_kind_version, &checkpoint, + visual_parameters, now.tv_sec) : lardon3d_project_db_record_task(state->project_db, &snapshot, task_kind, task_kind_version, &checkpoint, now.tv_sec); if (recorded == LARDON3D_PROJECT_DB_BUSY) { @@ -731,7 +736,7 @@ checkpoint_task_internal(Lardon3DAppState *state, const Lardon3DTask *task, Lardon3DProjectTaskCheckpointResult lardon3d_project_checkpoint_task(Lardon3DAppState *state, const Lardon3DTask *task) { - return checkpoint_task_internal(state, task, NULL, 0, NULL); + return checkpoint_task_internal(state, task, NULL, 0, NULL, NULL); } Lardon3DProjectTaskCheckpointResult @@ -740,7 +745,7 @@ lardon3d_project_checkpoint_image_import_task(Lardon3DAppState *state, const Lar if (!source_path || !source_path[0] || scanset_id == 0) { return LARDON3D_PROJECT_TASK_CHECKPOINT_INVALID_TASK; } - return checkpoint_task_internal(state, task, source_path, scanset_id, NULL); + return checkpoint_task_internal(state, task, source_path, scanset_id, NULL, NULL); } Lardon3DProjectTaskCheckpointResult lardon3d_project_checkpoint_feature_extract_task( @@ -749,7 +754,16 @@ Lardon3DProjectTaskCheckpointResult lardon3d_project_checkpoint_feature_extract_ if (!parameters) { return LARDON3D_PROJECT_TASK_CHECKPOINT_INVALID_TASK; } - return checkpoint_task_internal(state, task, NULL, 0, parameters); + return checkpoint_task_internal(state, task, NULL, 0, parameters, NULL); +} + +Lardon3DProjectTaskCheckpointResult lardon3d_project_checkpoint_visual_index_update_task( + Lardon3DAppState *state, const Lardon3DTask *task, + const Lardon3DProjectDbVisualIndexUpdateTask *parameters) { + if (!parameters) { + return LARDON3D_PROJECT_TASK_CHECKPOINT_INVALID_TASK; + } + return checkpoint_task_internal(state, task, NULL, 0, NULL, parameters); } static bool coherent_recovery(const Lardon3DProjectDbTask *database_task, diff --git a/src/project_db.c b/src/project_db.c index b3610da..f09d9fa 100644 --- a/src/project_db.c +++ b/src/project_db.c @@ -102,6 +102,43 @@ static const char schema_feature_v5[] = "CHECK(fast_threshold BETWEEN 1 AND 255),parameter_fingerprint BLOB NOT NULL " "CHECK(length(parameter_fingerprint)=32));"; +static const char schema_visual_v6[] = + "CREATE TABLE visual_indexes(visual_index_id INTEGER PRIMARY KEY AUTOINCREMENT " + "CHECK(visual_index_id>0),index_kind TEXT NOT NULL CHECK(length(index_kind)>0 AND " + "length(index_kind)<65),index_version INTEGER NOT NULL CHECK(index_version>0)," + "descriptor_type INTEGER NOT NULL CHECK(descriptor_type IN(1,2)),descriptor_dimension " + "INTEGER NOT NULL CHECK(descriptor_dimension BETWEEN 1 AND 4096),extractor_kind TEXT NOT " + "NULL CHECK(length(extractor_kind)>0 AND length(extractor_kind)<65),extractor_version " + "INTEGER NOT NULL CHECK(extractor_version>0),feature_parameter_fingerprint BLOB NOT NULL " + "CHECK(length(feature_parameter_fingerprint)=32),index_parameter_fingerprint BLOB NOT NULL " + "CHECK(length(index_parameter_fingerprint)=32),table_count INTEGER NOT NULL CHECK(table_count " + "BETWEEN 1 AND 32),key_bits INTEGER NOT NULL CHECK(key_bits BETWEEN 8 AND 32)," + "max_features_per_set INTEGER NOT NULL CHECK(max_features_per_set BETWEEN 1 AND 1024)," + "max_bucket_postings INTEGER NOT NULL CHECK(max_bucket_postings BETWEEN 1 AND 4096)," + "created_at INTEGER NOT NULL CHECK(created_at>=0),UNIQUE(index_kind,index_version," + "descriptor_type,descriptor_dimension,extractor_kind,extractor_version," + "feature_parameter_fingerprint,index_parameter_fingerprint));" + "CREATE TABLE visual_index_segments(visual_index_segment_id INTEGER PRIMARY KEY " + "AUTOINCREMENT CHECK(visual_index_segment_id>0),visual_index_id INTEGER NOT NULL REFERENCES " + "visual_indexes(visual_index_id),generation INTEGER NOT NULL CHECK(generation>0),sha256 BLOB " + "NOT NULL CHECK(length(sha256)=32),path TEXT NOT NULL UNIQUE CHECK(length(path)>0 AND " + "length(path)<4096),size_bytes INTEGER NOT NULL CHECK(size_bytes>=128),posting_count INTEGER " + "NOT NULL CHECK(posting_count>=0),feature_set_count INTEGER NOT NULL CHECK(feature_set_count " + "BETWEEN 1 AND 16),durability INTEGER NOT NULL CHECK(durability BETWEEN 0 AND 1)," + "producer_task_id INTEGER REFERENCES tasks(task_id),created_at INTEGER NOT NULL " + "CHECK(created_at>=0),UNIQUE(visual_index_id,generation),UNIQUE(visual_index_id,sha256));" + "CREATE INDEX visual_index_segments_index_idx ON visual_index_segments(visual_index_id," + "generation);" + "CREATE TABLE visual_index_memberships(visual_index_id INTEGER NOT NULL REFERENCES " + "visual_indexes(visual_index_id),feature_set_id INTEGER NOT NULL REFERENCES " + "feature_sets(feature_set_id),visual_index_segment_id INTEGER NOT NULL REFERENCES " + "visual_index_segments(visual_index_segment_id),PRIMARY KEY(visual_index_id,feature_set_id));" + "CREATE INDEX visual_index_memberships_segment_idx ON visual_index_memberships(" + "visual_index_segment_id,feature_set_id);" + "CREATE TABLE visual_index_update_tasks(task_id INTEGER PRIMARY KEY REFERENCES tasks(task_id) " + "ON DELETE CASCADE,visual_index_id INTEGER NOT NULL REFERENCES visual_indexes(visual_index_id)," + "after_feature_set_id INTEGER NOT NULL CHECK(after_feature_set_id>=0));"; + static void copy_error(char destination[LARDON3D_PROJECT_DB_ERROR_CAPACITY], const char *text) { if (destination) { (void)snprintf(destination, LARDON3D_PROJECT_DB_ERROR_CAPACITY, "%s", text ? text : ""); @@ -190,7 +227,7 @@ static Lardon3DProjectDbResult migrate(Lardon3DProjectDb *database, unsigned int return LARDON3D_PROJECT_DB_OK; } if (from_version != 0 && from_version != 1 && from_version != 2 && from_version != 3 && - from_version != 4) { + from_version != 4 && from_version != 5) { return LARDON3D_PROJECT_DB_CORRUPT; } Lardon3DProjectDbResult result = execute(database, "BEGIN IMMEDIATE", "begin migration"); @@ -199,6 +236,14 @@ static Lardon3DProjectDbResult migrate(Lardon3DProjectDb *database, unsigned int if (result == LARDON3D_PROJECT_DB_OK) { result = execute(database, schema_feature_v5, "create feature schema v5"); } + if (result == LARDON3D_PROJECT_DB_OK) { + result = execute(database, schema_visual_v6, "create visual index schema v6"); + } + if (result == LARDON3D_PROJECT_DB_OK) { + result = execute(database, + "UPDATE metadata SET value=6 WHERE key='schema_version' AND value=5", + "finish new schema v6"); + } } if (result == LARDON3D_PROJECT_DB_OK && from_version == 1) { result = execute(database, @@ -285,7 +330,7 @@ static Lardon3DProjectDbResult migrate(Lardon3DProjectDb *database, unsigned int "finish schema v4 migration"); } } - if (result == LARDON3D_PROJECT_DB_OK && from_version != 0) { + if (result == LARDON3D_PROJECT_DB_OK && from_version != 0 && from_version < 5) { result = execute( database, "CREATE TABLE feature_assets(feature_asset_id INTEGER PRIMARY KEY AUTOINCREMENT " @@ -333,6 +378,21 @@ static Lardon3DProjectDbResult migrate(Lardon3DProjectDb *database, unsigned int "finish schema v5 migration"); } } + if (result == LARDON3D_PROJECT_DB_OK && from_version != 0 && from_version < 6) { + result = execute(database, schema_visual_v6, "migrate schema v5 to v6"); +#ifdef LARDON3D_PROJECT_DB_TESTING + const char *forced_failure = getenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V6"); + if (result == LARDON3D_PROJECT_DB_OK && forced_failure && strcmp(forced_failure, "1") == 0) { + result = execute(database, "INSERT INTO missing_test_table VALUES(1)", + "forced migration v6 failure"); + } +#endif + if (result == LARDON3D_PROJECT_DB_OK) { + result = execute(database, + "UPDATE metadata SET value=6 WHERE key='schema_version' AND value=5", + "finish schema v6 migration"); + } + } if (result == LARDON3D_PROJECT_DB_OK) { result = execute(database, "COMMIT", "commit migration"); } @@ -626,6 +686,7 @@ record_task_internal(Lardon3DProjectDb *database, const Lardon3DTaskDurableSnaps const char *task_kind, uint32_t task_kind_version, const Lardon3DProjectDbCheckpoint *checkpoint, const char *source_path, uint64_t scanset_id, const Lardon3DProjectDbFeatureExtractTask *feature, + const Lardon3DProjectDbVisualIndexUpdateTask *visual, int64_t updated_at) { bool typed = task_kind != NULL; if (!database || !valid_durable_task(snapshot, updated_at) || @@ -640,6 +701,9 @@ record_task_internal(Lardon3DProjectDb *database, const Lardon3DTaskDurableSnaps feature->max_features == 0 || feature->max_features > 8192 || feature->pyramid_levels == 0 || feature->pyramid_levels > 16 || feature->fast_threshold == 0 || feature->fast_threshold > 255)) || + (visual && + (!valid_task_id(visual->task_id) || visual->task_id != snapshot->id || + !valid_task_id(visual->visual_index_id) || visual->after_feature_set_id > INT64_MAX)) || (checkpoint && !valid_checkpoint(checkpoint))) { return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; } @@ -766,6 +830,24 @@ record_task_internal(Lardon3DProjectDb *database, const Lardon3DTaskDurableSnaps } } } + if (result == LARDON3D_PROJECT_DB_OK && visual) { + result = prepare(database, + "INSERT INTO visual_index_update_tasks(task_id,visual_index_id," + "after_feature_set_id) VALUES(?1,?2,?3) ON CONFLICT(task_id) DO UPDATE SET " + "after_feature_set_id=excluded.after_feature_set_id WHERE " + "visual_index_update_tasks.visual_index_id=excluded.visual_index_id", + &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + sqlite3_bind_int64(statement, 1, (sqlite3_int64)visual->task_id); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)visual->visual_index_id); + sqlite3_bind_int64(statement, 3, (sqlite3_int64)visual->after_feature_set_id); + result = step_done(database, statement, "upsert visual index update"); + if (result == LARDON3D_PROJECT_DB_OK && sqlite3_changes(database->connection) != 1) { + copy_error(database->error, "Index cible de tâche immuable."); + result = LARDON3D_PROJECT_DB_CONSTRAINT; + } + } + } if (result == LARDON3D_PROJECT_DB_OK && checkpoint) { result = prepare(database, @@ -798,7 +880,7 @@ Lardon3DProjectDbResult lardon3d_project_db_record_task( Lardon3DProjectDb *database, const Lardon3DTaskDurableSnapshot *snapshot, const char *task_kind, uint32_t task_kind_version, const Lardon3DProjectDbCheckpoint *checkpoint, int64_t updated_at) { return record_task_internal(database, snapshot, task_kind, task_kind_version, checkpoint, NULL, 0, - NULL, updated_at); + NULL, NULL, updated_at); } Lardon3DProjectDbResult lardon3d_project_db_record_image_import_task( @@ -809,7 +891,7 @@ Lardon3DProjectDbResult lardon3d_project_db_record_image_import_task( return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; } return record_task_internal(database, snapshot, task_kind, task_kind_version, checkpoint, - source_path, scanset_id, NULL, updated_at); + source_path, scanset_id, NULL, NULL, updated_at); } Lardon3DProjectDbResult lardon3d_project_db_record_feature_extract_task( @@ -820,7 +902,7 @@ Lardon3DProjectDbResult lardon3d_project_db_record_feature_extract_task( return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; } return record_task_internal(database, snapshot, task_kind, task_kind_version, checkpoint, NULL, 0, - parameters, updated_at); + parameters, NULL, updated_at); } static bool read_task(sqlite3_stmt *statement, Lardon3DProjectDbTask *task) { @@ -1939,6 +2021,397 @@ lardon3d_project_db_load_feature_extract_task(Lardon3DProjectDb *db, uint64_t ta return result; } +static bool valid_visual_index(const Lardon3DProjectDbVisualIndex *index) { + return index && lardon3d_task_kind_is_valid(index->index_kind) && index->index_version > 0 && + index->descriptor_type >= 1 && index->descriptor_type <= 2 && + index->descriptor_dimension > 0 && index->descriptor_dimension <= 4096 && + lardon3d_task_kind_is_valid(index->extractor_kind) && index->extractor_version > 0 && + index->table_count > 0 && index->table_count <= 32 && index->key_bits >= 8 && + index->key_bits <= 32 && index->max_features_per_set > 0 && + index->max_features_per_set <= 1024 && index->max_bucket_postings > 0 && + index->max_bucket_postings <= 4096 && index->created_at >= 0; +} + +static bool read_visual_index(sqlite3_stmt *statement, Lardon3DProjectDbVisualIndex *index) { + memset(index, 0, sizeof(*index)); + sqlite3_int64 id = sqlite3_column_int64(statement, 0); + const void *feature_fp = sqlite3_column_blob(statement, 7); + const void *index_fp = sqlite3_column_blob(statement, 8); + if (id <= 0 || !copy_column(statement, 1, index->index_kind, sizeof(index->index_kind)) || + !copy_column(statement, 5, index->extractor_kind, sizeof(index->extractor_kind)) || + sqlite3_column_bytes(statement, 7) != 32 || sqlite3_column_bytes(statement, 8) != 32 || + !feature_fp || !index_fp) { + return false; + } + index->visual_index_id = (uint64_t)id; + index->index_version = (uint32_t)sqlite3_column_int64(statement, 2); + index->descriptor_type = (uint32_t)sqlite3_column_int64(statement, 3); + index->descriptor_dimension = (uint32_t)sqlite3_column_int64(statement, 4); + index->extractor_version = (uint32_t)sqlite3_column_int64(statement, 6); + memcpy(index->feature_parameter_fingerprint, feature_fp, 32); + memcpy(index->index_parameter_fingerprint, index_fp, 32); + index->table_count = (uint32_t)sqlite3_column_int64(statement, 9); + index->key_bits = (uint32_t)sqlite3_column_int64(statement, 10); + index->max_features_per_set = (uint32_t)sqlite3_column_int64(statement, 11); + index->max_bucket_postings = (uint32_t)sqlite3_column_int64(statement, 12); + index->created_at = sqlite3_column_int64(statement, 13); + return valid_visual_index(index); +} + +static const char visual_index_select[] = + "SELECT visual_index_id,index_kind,index_version,descriptor_type,descriptor_dimension," + "extractor_kind,extractor_version,feature_parameter_fingerprint," + "index_parameter_fingerprint,table_count,key_bits,max_features_per_set," + "max_bucket_postings,created_at FROM visual_indexes"; + +Lardon3DProjectDbResult lardon3d_project_db_create_visual_index( + Lardon3DProjectDb *db, const Lardon3DProjectDbVisualIndex *configuration, + Lardon3DProjectDbVisualIndex *out) { + if (!db || !valid_visual_index(configuration) || !out) { + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + } + (void)pthread_mutex_lock(&db->mutex); + sqlite3_stmt *statement = NULL; + Lardon3DProjectDbResult result = prepare( + db, + "INSERT INTO visual_indexes(index_kind,index_version,descriptor_type,descriptor_dimension," + "extractor_kind,extractor_version,feature_parameter_fingerprint," + "index_parameter_fingerprint,table_count,key_bits,max_features_per_set," + "max_bucket_postings,created_at) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13) " + "ON CONFLICT(index_kind,index_version,descriptor_type,descriptor_dimension,extractor_kind," + "extractor_version,feature_parameter_fingerprint,index_parameter_fingerprint) DO NOTHING", + &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + sqlite3_bind_text(statement, 1, configuration->index_kind, -1, SQLITE_TRANSIENT); + sqlite3_bind_int64(statement, 2, configuration->index_version); + sqlite3_bind_int64(statement, 3, configuration->descriptor_type); + sqlite3_bind_int64(statement, 4, configuration->descriptor_dimension); + sqlite3_bind_text(statement, 5, configuration->extractor_kind, -1, SQLITE_TRANSIENT); + sqlite3_bind_int64(statement, 6, configuration->extractor_version); + sqlite3_bind_blob(statement, 7, configuration->feature_parameter_fingerprint, 32, + SQLITE_TRANSIENT); + sqlite3_bind_blob(statement, 8, configuration->index_parameter_fingerprint, 32, + SQLITE_TRANSIENT); + sqlite3_bind_int64(statement, 9, configuration->table_count); + sqlite3_bind_int64(statement, 10, configuration->key_bits); + sqlite3_bind_int64(statement, 11, configuration->max_features_per_set); + sqlite3_bind_int64(statement, 12, configuration->max_bucket_postings); + sqlite3_bind_int64(statement, 13, configuration->created_at); + result = step_done(db, statement, "create visual index"); + } + if (result == LARDON3D_PROJECT_DB_OK) { + char sql[1024]; + snprintf(sql, sizeof(sql), + "%s WHERE index_kind=?1 AND index_version=?2 AND descriptor_type=?3 AND " + "descriptor_dimension=?4 AND extractor_kind=?5 AND extractor_version=?6 AND " + "feature_parameter_fingerprint=?7 AND index_parameter_fingerprint=?8", + visual_index_select); + result = prepare(db, sql, &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + sqlite3_bind_text(statement, 1, configuration->index_kind, -1, SQLITE_TRANSIENT); + sqlite3_bind_int64(statement, 2, configuration->index_version); + sqlite3_bind_int64(statement, 3, configuration->descriptor_type); + sqlite3_bind_int64(statement, 4, configuration->descriptor_dimension); + sqlite3_bind_text(statement, 5, configuration->extractor_kind, -1, SQLITE_TRANSIENT); + sqlite3_bind_int64(statement, 6, configuration->extractor_version); + sqlite3_bind_blob(statement, 7, configuration->feature_parameter_fingerprint, 32, + SQLITE_TRANSIENT); + sqlite3_bind_blob(statement, 8, configuration->index_parameter_fingerprint, 32, + SQLITE_TRANSIENT); + int code = sqlite3_step(statement); + if (code != SQLITE_ROW || !read_visual_index(statement, out)) { + result = LARDON3D_PROJECT_DB_CORRUPT; + } + sqlite3_finalize(statement); + } + } + (void)pthread_mutex_unlock(&db->mutex); + return result; +} + +Lardon3DProjectDbResult lardon3d_project_db_load_visual_index( + Lardon3DProjectDb *db, uint64_t id, Lardon3DProjectDbVisualIndex *out) { + if (!db || !valid_catalog_id(id) || !out) { + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + } + (void)pthread_mutex_lock(&db->mutex); + char sql[768]; + snprintf(sql, sizeof(sql), "%s WHERE visual_index_id=?1", visual_index_select); + sqlite3_stmt *statement = NULL; + Lardon3DProjectDbResult result = prepare(db, sql, &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + sqlite3_bind_int64(statement, 1, (sqlite3_int64)id); + int code = sqlite3_step(statement); + result = code == SQLITE_DONE ? LARDON3D_PROJECT_DB_NOT_FOUND + : code == SQLITE_ROW && read_visual_index(statement, out) + ? LARDON3D_PROJECT_DB_OK + : LARDON3D_PROJECT_DB_CORRUPT; + sqlite3_finalize(statement); + } + (void)pthread_mutex_unlock(&db->mutex); + return result; +} + +static bool canonical_visual_path(const unsigned char hash[32], const char *path) { + static const char digits[] = "0123456789abcdef"; + char hex[65]; + for (size_t i = 0; i < 32; ++i) { + hex[i * 2] = digits[hash[i] >> 4]; + hex[i * 2 + 1] = digits[hash[i] & 15]; + } + hex[64] = '\0'; + char expected[LARDON3D_PROJECT_DB_PATH_CAPACITY]; + int written = snprintf(expected, sizeof(expected), "assets/visual-index/%c%c/%s", hex[0], + hex[1], hex); + return written > 0 && (size_t)written < sizeof(expected) && strcmp(expected, path) == 0; +} + +static bool read_visual_segment(sqlite3_stmt *statement, + Lardon3DProjectDbVisualIndexSegment *segment) { + memset(segment, 0, sizeof(*segment)); + sqlite3_int64 id = sqlite3_column_int64(statement, 0); + sqlite3_int64 index_id = sqlite3_column_int64(statement, 1); + sqlite3_int64 generation = sqlite3_column_int64(statement, 2); + const void *hash = sqlite3_column_blob(statement, 3); + sqlite3_int64 size = sqlite3_column_int64(statement, 5); + sqlite3_int64 postings = sqlite3_column_int64(statement, 6); + sqlite3_int64 members = sqlite3_column_int64(statement, 7); + sqlite3_int64 durability = sqlite3_column_int64(statement, 8); + if (id <= 0 || index_id <= 0 || generation <= 0 || !hash || + sqlite3_column_bytes(statement, 3) != 32 || size < 128 || postings < 0 || members < 1 || + members > 16 || durability < 0 || durability > 1 || + !copy_column(statement, 4, segment->path, sizeof(segment->path))) { + return false; + } + segment->visual_index_segment_id = (uint64_t)id; + segment->visual_index_id = (uint64_t)index_id; + segment->generation = (uint64_t)generation; + memcpy(segment->sha256, hash, 32); + segment->size_bytes = (uint64_t)size; + segment->posting_count = (uint64_t)postings; + segment->feature_set_count = (uint32_t)members; + segment->durability = (Lardon3DProjectDbVisualIndexDurability)durability; + segment->has_producer_task = sqlite3_column_type(statement, 9) != SQLITE_NULL; + segment->producer_task_id = segment->has_producer_task + ? (uint64_t)sqlite3_column_int64(statement, 9) + : 0; + segment->created_at = sqlite3_column_int64(statement, 10); + return segment->created_at >= 0 && canonical_visual_path(segment->sha256, segment->path); +} + +static const char visual_segment_select[] = + "SELECT visual_index_segment_id,visual_index_id,generation,sha256,path,size_bytes," + "posting_count,feature_set_count,durability,producer_task_id,created_at " + "FROM visual_index_segments"; + +Lardon3DProjectDbResult lardon3d_project_db_list_visual_index_segments( + Lardon3DProjectDb *db, uint64_t index_id, uint64_t after, + Lardon3DProjectDbVisualIndexSegment *segments, size_t capacity, size_t *count) { + if (count) { + *count = 0; + } + if (!db || !valid_catalog_id(index_id) || after > INT64_MAX || !segments || !count || + capacity == 0 || capacity > 256) { + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + } + (void)pthread_mutex_lock(&db->mutex); + char sql[768]; + snprintf(sql, sizeof(sql), "%s WHERE visual_index_id=?1 AND generation>?2 ORDER BY generation " + "LIMIT ?3", visual_segment_select); + sqlite3_stmt *statement = NULL; + Lardon3DProjectDbResult result = prepare(db, sql, &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + sqlite3_bind_int64(statement, 1, (sqlite3_int64)index_id); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)after); + sqlite3_bind_int64(statement, 3, (sqlite3_int64)capacity); + int code = SQLITE_DONE; + while (*count < capacity && (code = sqlite3_step(statement)) == SQLITE_ROW) { + if (!read_visual_segment(statement, &segments[*count])) { + result = LARDON3D_PROJECT_DB_CORRUPT; + break; + } + ++*count; + } + if (result == LARDON3D_PROJECT_DB_OK && code != SQLITE_DONE && *count < capacity) { + result = sqlite_result(db, code, "list visual index segments"); + } + sqlite3_finalize(statement); + } + (void)pthread_mutex_unlock(&db->mutex); + return result; +} + +Lardon3DProjectDbResult lardon3d_project_db_list_visual_index_pending( + Lardon3DProjectDb *db, uint64_t index_id, uint64_t after, + Lardon3DProjectDbFeatureSet *sets, size_t capacity, size_t *count) { + if (count) { + *count = 0; + } + if (!db || !valid_catalog_id(index_id) || after > INT64_MAX || !sets || !count || + capacity == 0 || capacity > 16) { + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + } + (void)pthread_mutex_lock(&db->mutex); + char sql[2048]; + snprintf(sql, sizeof(sql), + "%s JOIN visual_indexes v ON v.visual_index_id=?1 LEFT JOIN " + "visual_index_memberships m ON m.visual_index_id=v.visual_index_id AND " + "m.feature_set_id=f.feature_set_id WHERE f.feature_set_id>?2 AND m.feature_set_id IS " + "NULL AND f.descriptor_type=v.descriptor_type AND f.descriptor_dimension=" + "v.descriptor_dimension AND f.extractor_kind=v.extractor_kind AND " + "f.extractor_version=v.extractor_version AND f.parameter_fingerprint=" + "v.feature_parameter_fingerprint ORDER BY f.feature_set_id LIMIT ?3", + feature_select); + sqlite3_stmt *statement = NULL; + Lardon3DProjectDbResult result = prepare(db, sql, &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + sqlite3_bind_int64(statement, 1, (sqlite3_int64)index_id); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)after); + sqlite3_bind_int64(statement, 3, (sqlite3_int64)capacity); + int code = SQLITE_DONE; + while (*count < capacity && (code = sqlite3_step(statement)) == SQLITE_ROW) { + if (!read_feature_set(statement, &sets[*count])) { + result = LARDON3D_PROJECT_DB_CORRUPT; + break; + } + ++*count; + } + if (result == LARDON3D_PROJECT_DB_OK && code != SQLITE_DONE && *count < capacity) { + result = sqlite_result(db, code, "list pending visual index features"); + } + sqlite3_finalize(statement); + } + (void)pthread_mutex_unlock(&db->mutex); + return result; +} + +Lardon3DProjectDbResult lardon3d_project_db_publish_visual_index_segment( + Lardon3DProjectDb *db, const Lardon3DProjectDbVisualIndexSegment *segment, + const uint64_t *ids, size_t count, Lardon3DProjectDbVisualIndexSegment *published) { + if (!db || !segment || !valid_catalog_id(segment->visual_index_id) || segment->generation == 0 || + !canonical_visual_path(segment->sha256, segment->path) || segment->size_bytes < 128 || + segment->size_bytes > INT64_MAX || segment->posting_count > INT64_MAX || !ids || count == 0 || + count > 16 || segment->feature_set_count != count || segment->durability < 0 || + segment->durability > 1 || (segment->producer_task_id && + !valid_task_id(segment->producer_task_id)) || + segment->created_at < 0 || !published) { + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + } +#ifdef LARDON3D_PROJECT_DB_TESTING + const char *fail_publish = getenv("LARDON3D_TEST_PROJECT_DB_FAIL_VISUAL_SEGMENT"); + if (fail_publish && strcmp(fail_publish, "1") == 0) { + return LARDON3D_PROJECT_DB_BUSY; + } +#endif + (void)pthread_mutex_lock(&db->mutex); + Lardon3DProjectDbResult result = execute(db, "BEGIN IMMEDIATE", "begin visual segment publish"); + sqlite3_stmt *statement = NULL; + if (result == LARDON3D_PROJECT_DB_OK) { + result = prepare(db, "INSERT INTO visual_index_segments(visual_index_id,generation,sha256," + "path,size_bytes,posting_count,feature_set_count,durability," + "producer_task_id,created_at) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)", + &statement); + } + if (result == LARDON3D_PROJECT_DB_OK) { + sqlite3_bind_int64(statement, 1, (sqlite3_int64)segment->visual_index_id); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)segment->generation); + sqlite3_bind_blob(statement, 3, segment->sha256, 32, SQLITE_TRANSIENT); + sqlite3_bind_text(statement, 4, segment->path, -1, SQLITE_TRANSIENT); + sqlite3_bind_int64(statement, 5, (sqlite3_int64)segment->size_bytes); + sqlite3_bind_int64(statement, 6, (sqlite3_int64)segment->posting_count); + sqlite3_bind_int64(statement, 7, (sqlite3_int64)count); + sqlite3_bind_int(statement, 8, (int)segment->durability); + if (segment->producer_task_id) { + sqlite3_bind_int64(statement, 9, (sqlite3_int64)segment->producer_task_id); + } else { + sqlite3_bind_null(statement, 9); + } + sqlite3_bind_int64(statement, 10, segment->created_at); + result = step_done(db, statement, "insert visual segment"); + } + sqlite3_int64 segment_id = sqlite3_last_insert_rowid(db->connection); + for (size_t i = 0; i < count && result == LARDON3D_PROJECT_DB_OK; ++i) { + if (!valid_catalog_id(ids[i])) { + result = LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + break; + } + result = prepare(db, "INSERT INTO visual_index_memberships(visual_index_id,feature_set_id," + "visual_index_segment_id) VALUES(?1,?2,?3)", &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + sqlite3_bind_int64(statement, 1, (sqlite3_int64)segment->visual_index_id); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)ids[i]); + sqlite3_bind_int64(statement, 3, segment_id); + result = step_done(db, statement, "insert visual membership"); + } + } + if (result == LARDON3D_PROJECT_DB_OK) { + result = execute(db, "COMMIT", "commit visual segment publish"); + } + if (result != LARDON3D_PROJECT_DB_OK) { + (void)execute(db, "ROLLBACK", "rollback visual segment publish"); + } + (void)pthread_mutex_unlock(&db->mutex); + if (result != LARDON3D_PROJECT_DB_OK) { + return result; + } + Lardon3DProjectDbVisualIndexSegment page[1]; + size_t page_count = 0; + result = lardon3d_project_db_list_visual_index_segments(db, segment->visual_index_id, + segment->generation - 1, page, 1, + &page_count); + if (result == LARDON3D_PROJECT_DB_OK && page_count == 1 && + page[0].generation == segment->generation) { + *published = page[0]; + return LARDON3D_PROJECT_DB_OK; + } + return result == LARDON3D_PROJECT_DB_OK ? LARDON3D_PROJECT_DB_CORRUPT : result; +} + +Lardon3DProjectDbResult lardon3d_project_db_record_visual_index_update_task( + Lardon3DProjectDb *db, const Lardon3DTaskDurableSnapshot *snapshot, const char *kind, + uint32_t version, const Lardon3DProjectDbCheckpoint *checkpoint, + const Lardon3DProjectDbVisualIndexUpdateTask *parameters, int64_t updated_at) { + if (!snapshot || !parameters || parameters->task_id != snapshot->id || + !valid_catalog_id(parameters->visual_index_id) || + parameters->after_feature_set_id > INT64_MAX) { + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + } + return record_task_internal(db, snapshot, kind, version, checkpoint, NULL, 0, NULL, + parameters, updated_at); +} + +Lardon3DProjectDbResult lardon3d_project_db_load_visual_index_update_task( + Lardon3DProjectDb *db, uint64_t task_id, + Lardon3DProjectDbVisualIndexUpdateTask *parameters) { + if (!db || !valid_task_id(task_id) || !parameters) { + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + } + memset(parameters, 0, sizeof(*parameters)); + (void)pthread_mutex_lock(&db->mutex); + sqlite3_stmt *statement = NULL; + Lardon3DProjectDbResult result = prepare( + db, "SELECT visual_index_id,after_feature_set_id FROM visual_index_update_tasks WHERE " + "task_id=?1", &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + sqlite3_bind_int64(statement, 1, (sqlite3_int64)task_id); + int code = sqlite3_step(statement); + sqlite3_int64 index_id = sqlite3_column_int64(statement, 0); + sqlite3_int64 after = sqlite3_column_int64(statement, 1); + if (code == SQLITE_DONE) { + result = LARDON3D_PROJECT_DB_NOT_FOUND; + } else if (code != SQLITE_ROW || index_id <= 0 || after < 0) { + result = LARDON3D_PROJECT_DB_CORRUPT; + } else { + parameters->task_id = task_id; + parameters->visual_index_id = (uint64_t)index_id; + parameters->after_feature_set_id = (uint64_t)after; + } + sqlite3_finalize(statement); + } + (void)pthread_mutex_unlock(&db->mutex); + return result; +} + #ifdef LARDON3D_PROJECT_DB_TESTING Lardon3DProjectDbResult lardon3d_project_db_test_delete_feature_identity( Lardon3DProjectDb *database, uint64_t feature_set_id, uint64_t feature_asset_id) { diff --git a/src/task_kinds.c b/src/task_kinds.c index d08e2a8..dc58d80 100644 --- a/src/task_kinds.c +++ b/src/task_kinds.c @@ -1,5 +1,6 @@ #include #include +#include #include const Lardon3DTaskKindRegistry *lardon3d_task_kind_registry_production(void) { @@ -13,6 +14,11 @@ const Lardon3DTaskKindRegistry *lardon3d_task_kind_registry_production(void) { .kind = LARDON3D_FEATURE_EXTRACT_TASK_KIND, .kind_version = LARDON3D_FEATURE_EXTRACT_TASK_KIND_VERSION, .reconstruct = lardon3d_feature_extract_reconstruct, + }, + { + .kind = LARDON3D_VISUAL_INDEX_UPDATE_TASK_KIND, + .kind_version = LARDON3D_VISUAL_INDEX_UPDATE_TASK_KIND_VERSION, + .reconstruct = lardon3d_visual_index_update_reconstruct, }}; static const Lardon3DTaskKindRegistry registry = { .descriptors = descriptors, diff --git a/src/visual_index.c b/src/visual_index.c new file mode 100644 index 0000000..dbf2a52 --- /dev/null +++ b/src/visual_index.c @@ -0,0 +1,852 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +enum { + SEGMENT_HEADER_SIZE = 128, + POSTING_SIZE = 24, + DESCRIPTOR_SIZE = 32, + SEGMENT_POSTING_MAX = LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX * 1024 * + LARDON3D_VISUAL_INDEX_TABLE_COUNT, +}; + +typedef struct { + uint32_t table; + uint32_t key; + uint64_t feature_set_id; + uint32_t feature_index; +} Posting; + +typedef struct { + uint64_t feature_set_id; + uint32_t evidence; + uint32_t last_query_feature; +} Accumulator; + +static void put_u32(unsigned char *output, uint32_t value) { + output[0] = (unsigned char)value; + output[1] = (unsigned char)(value >> 8); + output[2] = (unsigned char)(value >> 16); + output[3] = (unsigned char)(value >> 24); +} + +static void put_u64(unsigned char *output, uint64_t value) { + put_u32(output, (uint32_t)value); + put_u32(output + 4, (uint32_t)(value >> 32)); +} + +static uint32_t get_u32(const unsigned char *input) { + return (uint32_t)input[0] | (uint32_t)input[1] << 8 | (uint32_t)input[2] << 16 | + (uint32_t)input[3] << 24; +} + +static uint64_t get_u64(const unsigned char *input) { + return (uint64_t)get_u32(input) | (uint64_t)get_u32(input + 4) << 32; +} + +bool lardon3d_visual_index_configuration_valid( + const Lardon3DVisualIndexConfiguration *configuration) { + return configuration && configuration->version == LARDON3D_VISUAL_INDEX_VERSION && + configuration->max_features_per_set > 0 && + configuration->max_features_per_set <= 1024 && + configuration->max_bucket_postings > 0 && + configuration->max_bucket_postings <= 4096; +} + +void lardon3d_visual_index_configuration_fingerprint( + const Lardon3DVisualIndexConfiguration *configuration, unsigned char fingerprint[32]) { + unsigned char canonical[32] = {0}; + memcpy(canonical, "L3DVICF1", 8); + if (lardon3d_visual_index_configuration_valid(configuration)) { + put_u32(canonical + 8, configuration->version); + put_u32(canonical + 12, LARDON3D_VISUAL_INDEX_TABLE_COUNT); + put_u32(canonical + 16, LARDON3D_VISUAL_INDEX_KEY_BITS); + put_u32(canonical + 20, configuration->max_features_per_set); + put_u32(canonical + 24, configuration->max_bucket_postings); + put_u32(canonical + 28, LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX); + } + EVP_Digest(canonical, sizeof(canonical), fingerprint, NULL, EVP_sha256(), NULL); +} + +static Lardon3DVisualIndexResult db_result(Lardon3DProjectDbResult result) { + switch (result) { + case LARDON3D_PROJECT_DB_OK: + return LARDON3D_VISUAL_INDEX_OK; + case LARDON3D_PROJECT_DB_NOT_FOUND: + return LARDON3D_VISUAL_INDEX_NOT_FOUND; + case LARDON3D_PROJECT_DB_BUSY: + return LARDON3D_VISUAL_INDEX_DB_BUSY; + case LARDON3D_PROJECT_DB_INVALID_ARGUMENT: + return LARDON3D_VISUAL_INDEX_INVALID_ARGUMENT; + case LARDON3D_PROJECT_DB_UNSUPPORTED_SCHEMA: + return LARDON3D_VISUAL_INDEX_UNSUPPORTED_VERSION; + case LARDON3D_PROJECT_DB_CORRUPT: + return LARDON3D_VISUAL_INDEX_CORRUPT; + default: + return LARDON3D_VISUAL_INDEX_DB_ERROR; + } +} + +Lardon3DVisualIndexResult lardon3d_visual_index_create( + Lardon3DProjectDb *database, const Lardon3DProjectDbFeatureSet *prototype, + const Lardon3DVisualIndexConfiguration *configuration, uint64_t *visual_index_id) { + if (visual_index_id) { + *visual_index_id = 0; + } + if (!database || !prototype || !visual_index_id || + !lardon3d_visual_index_configuration_valid(configuration) || + prototype->descriptor_type != LARDON3D_FEATURE_DESCRIPTOR_U8 || + prototype->descriptor_dimension != DESCRIPTOR_SIZE) { + return LARDON3D_VISUAL_INDEX_INVALID_ARGUMENT; + } + struct timespec now; + if (clock_gettime(CLOCK_REALTIME, &now) != 0) { + return LARDON3D_VISUAL_INDEX_IO_ERROR; + } + Lardon3DProjectDbVisualIndex requested = { + .index_version = LARDON3D_VISUAL_INDEX_VERSION, + .descriptor_type = prototype->descriptor_type, + .descriptor_dimension = prototype->descriptor_dimension, + .extractor_version = prototype->extractor_version, + .table_count = LARDON3D_VISUAL_INDEX_TABLE_COUNT, + .key_bits = LARDON3D_VISUAL_INDEX_KEY_BITS, + .max_features_per_set = configuration->max_features_per_set, + .max_bucket_postings = configuration->max_bucket_postings, + .created_at = now.tv_sec, + }; + snprintf(requested.index_kind, sizeof(requested.index_kind), "%s", LARDON3D_VISUAL_INDEX_KIND); + snprintf(requested.extractor_kind, sizeof(requested.extractor_kind), "%s", + prototype->extractor_kind); + memcpy(requested.feature_parameter_fingerprint, prototype->parameter_fingerprint, 32); + lardon3d_visual_index_configuration_fingerprint(configuration, + requested.index_parameter_fingerprint); + Lardon3DProjectDbVisualIndex created; + Lardon3DProjectDbResult result = + lardon3d_project_db_create_visual_index(database, &requested, &created); + if (result == LARDON3D_PROJECT_DB_OK) { + *visual_index_id = created.visual_index_id; + } + return db_result(result); +} + +static uint32_t lsh_key(const unsigned char descriptor[32], uint32_t table) { + uint32_t key = 0; + for (uint32_t bit = 0; bit < LARDON3D_VISUAL_INDEX_KEY_BITS; ++bit) { + uint32_t source = (table * 41U + bit * 11U) & 255U; + uint32_t value = (descriptor[source / 8] >> (source % 8)) & 1U; + key |= value << bit; + } + return key; +} + +static int compare_posting(const void *left, const void *right) { + const Posting *a = left; + const Posting *b = right; + if (a->table != b->table) { + return a->table < b->table ? -1 : 1; + } + if (a->key != b->key) { + return a->key < b->key ? -1 : 1; + } + if (a->feature_set_id != b->feature_set_id) { + return a->feature_set_id < b->feature_set_id ? -1 : 1; + } + return a->feature_index < b->feature_index ? -1 : a->feature_index > b->feature_index; +} + +static Lardon3DVisualIndexResult append_feature_postings( + const char *project_path, const Lardon3DProjectDbFeatureSet *set, uint32_t maximum, + Posting *postings, size_t capacity, size_t *count) { + Lardon3DFeatureReader *reader = NULL; + Lardon3DFeatureFileMetadata metadata; + Lardon3DFeatureStoreResult opened = + lardon3d_feature_reader_open(project_path, set, &reader, &metadata); + if (opened != LARDON3D_FEATURE_STORE_OK) { + return opened == LARDON3D_FEATURE_STORE_UNSUPPORTED_VERSION + ? LARDON3D_VISUAL_INDEX_UNSUPPORTED_VERSION + : opened == LARDON3D_FEATURE_STORE_CORRUPT ? LARDON3D_VISUAL_INDEX_CORRUPT + : LARDON3D_VISUAL_INDEX_IO_ERROR; + } + uint32_t selected = metadata.feature_count < maximum ? metadata.feature_count : maximum; + unsigned char descriptors[LARDON3D_FEATURE_READER_RANGE_MAX * DESCRIPTOR_SIZE]; + for (uint32_t start = 0; start < selected;) { + uint32_t amount = selected - start; + if (amount > LARDON3D_FEATURE_READER_RANGE_MAX) { + amount = LARDON3D_FEATURE_READER_RANGE_MAX; + } + if (lardon3d_feature_reader_descriptors(reader, start, descriptors, amount, + sizeof(descriptors)) != LARDON3D_FEATURE_STORE_OK) { + lardon3d_feature_reader_close(reader); + return LARDON3D_VISUAL_INDEX_IO_ERROR; + } + for (uint32_t i = 0; i < amount; ++i) { + for (uint32_t table = 0; table < LARDON3D_VISUAL_INDEX_TABLE_COUNT; ++table) { + if (*count >= capacity) { + lardon3d_feature_reader_close(reader); + return LARDON3D_VISUAL_INDEX_LIMIT; + } + postings[*count] = (Posting){table, lsh_key(descriptors + i * DESCRIPTOR_SIZE, table), + set->feature_set_id, start + i}; + ++*count; + } + } + start += amount; + } + lardon3d_feature_reader_close(reader); + return LARDON3D_VISUAL_INDEX_OK; +} + +static bool write_exact(int descriptor, const void *data, size_t size) { + const unsigned char *bytes = data; + while (size > 0) { + ssize_t written = write(descriptor, bytes, size); + if (written <= 0) { + return false; + } + bytes += (size_t)written; + size -= (size_t)written; + } + return true; +} + +static bool read_exact(int descriptor, void *data, size_t size, off_t offset) { + unsigned char *bytes = data; + while (size > 0) { + ssize_t amount = pread(descriptor, bytes, size, offset); + if (amount <= 0) { + return false; + } + bytes += (size_t)amount; + size -= (size_t)amount; + offset += amount; + } + return true; +} + +static bool hash_fd(int descriptor, unsigned char output[32]) { + EVP_MD_CTX *context = EVP_MD_CTX_new(); + if (!context || EVP_DigestInit_ex(context, EVP_sha256(), NULL) != 1 || lseek(descriptor, 0, SEEK_SET) < 0) { + EVP_MD_CTX_free(context); + return false; + } + unsigned char buffer[65536]; + ssize_t amount; + while ((amount = read(descriptor, buffer, sizeof(buffer))) > 0) { + if (EVP_DigestUpdate(context, buffer, (size_t)amount) != 1) { + EVP_MD_CTX_free(context); + return false; + } + } + unsigned int size = 0; + bool ok = amount == 0 && EVP_DigestFinal_ex(context, output, &size) == 1 && size == 32; + EVP_MD_CTX_free(context); + return ok; +} + +static bool make_directory(const char *path) { + return mkdir(path, 0755) == 0 || errno == EEXIST; +} + +static bool visual_directories(const char *project_path, const char prefix[3], char *directory, + size_t capacity) { + char assets[PATH_MAX]; + char visual[PATH_MAX]; + int a = snprintf(assets, sizeof(assets), "%s/assets", project_path); + int v = snprintf(visual, sizeof(visual), "%s/visual-index", assets); + int d = snprintf(directory, capacity, "%s/%s", visual, prefix); + return a > 0 && (size_t)a < sizeof(assets) && v > 0 && (size_t)v < sizeof(visual) && d > 0 && + (size_t)d < capacity && make_directory(assets) && make_directory(visual) && + make_directory(directory); +} + +static Lardon3DVisualIndexResult publish_file(const char *project_path, const char *temporary, + const unsigned char hash[32], char path[4096], + bool *durable) { + static const char digits[] = "0123456789abcdef"; + char hex[65]; + for (size_t i = 0; i < 32; ++i) { + hex[i * 2] = digits[hash[i] >> 4]; + hex[i * 2 + 1] = digits[hash[i] & 15]; + } + hex[64] = '\0'; + char prefix[3] = {hex[0], hex[1], '\0'}; + char directory[PATH_MAX]; + if (!visual_directories(project_path, prefix, directory, sizeof(directory))) { + return LARDON3D_VISUAL_INDEX_IO_ERROR; + } + char destination[PATH_MAX]; + int absolute = snprintf(destination, sizeof(destination), "%s/%s", directory, hex); + int relative = snprintf(path, 4096, "assets/visual-index/%s/%s", prefix, hex); + if (absolute <= 0 || (size_t)absolute >= sizeof(destination) || relative <= 0 || relative >= 4096) { + return LARDON3D_VISUAL_INDEX_IO_ERROR; + } + if (link(temporary, destination) != 0) { + if (errno != EEXIST) { + return LARDON3D_VISUAL_INDEX_IO_ERROR; + } + int existing = open(destination, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + unsigned char actual[32]; + struct stat status; + bool valid = existing >= 0 && fstat(existing, &status) == 0 && S_ISREG(status.st_mode) && + hash_fd(existing, actual) && memcmp(actual, hash, 32) == 0; + if (existing >= 0) { + close(existing); + } + if (!valid) { + return LARDON3D_VISUAL_INDEX_CORRUPT; + } + } + int directory_fd = open(directory, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (directory_fd < 0) { + return LARDON3D_VISUAL_INDEX_IO_ERROR; + } + *durable = fsync(directory_fd) == 0; +#ifdef LARDON3D_VISUAL_INDEX_TESTING + const char *fail_sync = getenv("LARDON3D_TEST_VISUAL_INDEX_FAIL_DIR_FSYNC"); + if (fail_sync && strcmp(fail_sync, "1") == 0) { + *durable = false; + } +#endif + close(directory_fd); + return *durable ? LARDON3D_VISUAL_INDEX_OK : LARDON3D_VISUAL_INDEX_PUBLISHED_NOT_DURABLE; +} + +static Lardon3DVisualIndexResult create_segment_file( + const char *project_path, const Lardon3DProjectDbVisualIndex *index, Posting *postings, + size_t posting_count, size_t member_count, unsigned char hash[32], char path[4096], + uint64_t *size, bool *durable) { + qsort(postings, posting_count, sizeof(*postings), compare_posting); + char template_path[PATH_MAX]; + int n = snprintf(template_path, sizeof(template_path), "%s/.lardon3d/visual-index-XXXXXX", + project_path); + if (n <= 0 || (size_t)n >= sizeof(template_path)) { + return LARDON3D_VISUAL_INDEX_IO_ERROR; + } + int descriptor = mkstemp(template_path); + if (descriptor < 0) { + return LARDON3D_VISUAL_INDEX_IO_ERROR; + } + unsigned char header[SEGMENT_HEADER_SIZE] = {0}; + memcpy(header, "L3DVIDX\0", 8); + put_u32(header + 8, 1); + put_u32(header + 12, SEGMENT_HEADER_SIZE); + put_u32(header + 16, index->table_count); + put_u32(header + 20, index->key_bits); + put_u64(header + 24, posting_count); + put_u64(header + 32, member_count); + put_u64(header + 40, SEGMENT_HEADER_SIZE); + *size = SEGMENT_HEADER_SIZE + posting_count * POSTING_SIZE; + put_u64(header + 48, *size); + memcpy(header + 56, index->index_parameter_fingerprint, 32); + memcpy(header + 88, index->feature_parameter_fingerprint, 32); + bool ok = write_exact(descriptor, header, sizeof(header)); + unsigned char record[POSTING_SIZE]; + for (size_t i = 0; i < posting_count && ok; ++i) { + memset(record, 0, sizeof(record)); + put_u32(record, postings[i].table); + put_u32(record + 4, postings[i].key); + put_u64(record + 8, postings[i].feature_set_id); + put_u32(record + 16, postings[i].feature_index); + ok = write_exact(descriptor, record, sizeof(record)); + } + ok = ok && fsync(descriptor) == 0 && hash_fd(descriptor, hash); + Lardon3DVisualIndexResult result = ok ? publish_file(project_path, template_path, hash, path, durable) + : LARDON3D_VISUAL_INDEX_IO_ERROR; + close(descriptor); + unlink(template_path); + return result; +} + +static Lardon3DVisualIndexResult update_once_internal( + const char *project_path, Lardon3DProjectDb *database, uint64_t index_id, + uint64_t producer_task_id, uint64_t after, size_t maximum_feature_sets, uint64_t *last, + size_t *indexed_count, void (*after_select)(void *), void *hook_userdata) { + if (last) { + *last = after; + } + if (indexed_count) { + *indexed_count = 0; + } + if (!project_path || !database || index_id == 0 || maximum_feature_sets == 0 || + maximum_feature_sets > LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX || !last || + !indexed_count) { + return LARDON3D_VISUAL_INDEX_INVALID_ARGUMENT; + } + Lardon3DProjectDbVisualIndex index; + Lardon3DProjectDbResult loaded = lardon3d_project_db_load_visual_index(database, index_id, &index); + if (loaded != LARDON3D_PROJECT_DB_OK) { + return db_result(loaded); + } + Lardon3DProjectDbVisualIndexSegment segments[LARDON3D_VISUAL_INDEX_SEGMENT_MAX]; + size_t segment_count = 0; + loaded = lardon3d_project_db_list_visual_index_segments(database, index_id, 0, segments, + LARDON3D_VISUAL_INDEX_SEGMENT_MAX, + &segment_count); + if (loaded != LARDON3D_PROJECT_DB_OK) { + return db_result(loaded); + } + if (segment_count >= LARDON3D_VISUAL_INDEX_SEGMENT_MAX) { + return LARDON3D_VISUAL_INDEX_LIMIT; + } + Lardon3DProjectDbFeatureSet sets[LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX]; + size_t set_count = 0; + loaded = lardon3d_project_db_list_visual_index_pending( + database, index_id, after, sets, maximum_feature_sets, &set_count); + if (loaded != LARDON3D_PROJECT_DB_OK) { + return db_result(loaded); + } + if (set_count == 0) { + return LARDON3D_VISUAL_INDEX_NO_CHANGE; + } + if (after_select) { + after_select(hook_userdata); + } + Posting *postings = calloc(SEGMENT_POSTING_MAX, sizeof(*postings)); + if (!postings) { + return LARDON3D_VISUAL_INDEX_IO_ERROR; + } + size_t posting_count = 0; + Lardon3DVisualIndexResult result = LARDON3D_VISUAL_INDEX_OK; + uint64_t ids[LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX]; + for (size_t i = 0; i < set_count && result == LARDON3D_VISUAL_INDEX_OK; ++i) { + ids[i] = sets[i].feature_set_id; + result = append_feature_postings(project_path, &sets[i], index.max_features_per_set, postings, + SEGMENT_POSTING_MAX, &posting_count); + } + unsigned char hash[32]; + char path[4096]; + uint64_t size = 0; + bool durable = false; + if (result == LARDON3D_VISUAL_INDEX_OK) { + result = create_segment_file(project_path, &index, postings, posting_count, set_count, hash, + path, &size, &durable); + } + free(postings); + if (result != LARDON3D_VISUAL_INDEX_OK && + result != LARDON3D_VISUAL_INDEX_PUBLISHED_NOT_DURABLE) { + return result; + } + struct timespec now; + if (clock_gettime(CLOCK_REALTIME, &now) != 0) { + return LARDON3D_VISUAL_INDEX_IO_ERROR; + } + Lardon3DProjectDbVisualIndexSegment segment = { + .visual_index_id = index_id, + .generation = segment_count + 1, + .size_bytes = size, + .posting_count = posting_count, + .feature_set_count = (uint32_t)set_count, + .durability = durable ? LARDON3D_DB_VISUAL_INDEX_DURABLE + : LARDON3D_DB_VISUAL_INDEX_PUBLISHED_NOT_DURABLE, + .producer_task_id = producer_task_id, + .created_at = now.tv_sec, + }; + memcpy(segment.sha256, hash, 32); + snprintf(segment.path, sizeof(segment.path), "%s", path); + Lardon3DProjectDbVisualIndexSegment published; + loaded = lardon3d_project_db_publish_visual_index_segment(database, &segment, ids, set_count, + &published); + if (loaded != LARDON3D_PROJECT_DB_OK) { + return db_result(loaded); + } + *last = ids[set_count - 1]; + *indexed_count = set_count; + return durable ? LARDON3D_VISUAL_INDEX_OK + : LARDON3D_VISUAL_INDEX_PUBLISHED_NOT_DURABLE; +} + +Lardon3DVisualIndexResult lardon3d_visual_index_update_once( + const char *project_path, Lardon3DProjectDb *database, uint64_t index_id, + uint64_t producer_task_id, uint64_t after, size_t maximum_feature_sets, uint64_t *last, + size_t *indexed_count) { + return update_once_internal(project_path, database, index_id, producer_task_id, after, + maximum_feature_sets, last, indexed_count, NULL, NULL); +} + +#ifdef LARDON3D_VISUAL_INDEX_TESTING +Lardon3DVisualIndexResult lardon3d_visual_index_test_update_once( + const char *project_path, Lardon3DProjectDb *database, uint64_t index_id, + uint64_t producer_task_id, uint64_t after, size_t maximum_feature_sets, uint64_t *last, + size_t *indexed_count, Lardon3DVisualIndexAfterSelectHook after_select, + void *hook_userdata) { + return update_once_internal(project_path, database, index_id, producer_task_id, after, + maximum_feature_sets, last, indexed_count, after_select, + hook_userdata); +} +#endif + +static Lardon3DVisualIndexResult open_segment( + const char *project_path, const Lardon3DProjectDbVisualIndex *index, + const Lardon3DProjectDbVisualIndexSegment *segment, int *descriptor, uint64_t *posting_count) { + char path[PATH_MAX]; + int n = snprintf(path, sizeof(path), "%s/%s", project_path, segment->path); + if (n <= 0 || (size_t)n >= sizeof(path)) { + return LARDON3D_VISUAL_INDEX_INVALID_ARGUMENT; + } + int fd = open(path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (fd < 0) { + return errno == ENOENT ? LARDON3D_VISUAL_INDEX_NOT_FOUND : LARDON3D_VISUAL_INDEX_IO_ERROR; + } + struct stat status; + unsigned char header[SEGMENT_HEADER_SIZE]; + unsigned char hash[32]; + if (fstat(fd, &status) != 0 || !S_ISREG(status.st_mode) || status.st_size < SEGMENT_HEADER_SIZE || + (uint64_t)status.st_size != segment->size_bytes || !read_exact(fd, header, sizeof(header), 0) || + !hash_fd(fd, hash) || memcmp(hash, segment->sha256, 32) != 0) { + close(fd); + return LARDON3D_VISUAL_INDEX_CORRUPT; + } + uint32_t version = get_u32(header + 8); + if (memcmp(header, "L3DVIDX\0", 8) != 0) { + close(fd); + return LARDON3D_VISUAL_INDEX_CORRUPT; + } + if (version != 1) { + close(fd); + return LARDON3D_VISUAL_INDEX_UNSUPPORTED_VERSION; + } + uint64_t count = get_u64(header + 24); + if (get_u32(header + 12) != SEGMENT_HEADER_SIZE || get_u32(header + 16) != index->table_count || + get_u32(header + 20) != index->key_bits || get_u64(header + 40) != SEGMENT_HEADER_SIZE || + count > SEGMENT_POSTING_MAX || count > (UINT64_MAX - SEGMENT_HEADER_SIZE) / POSTING_SIZE || + get_u64(header + 48) != SEGMENT_HEADER_SIZE + count * POSTING_SIZE || + get_u64(header + 48) != (uint64_t)status.st_size || + memcmp(header + 56, index->index_parameter_fingerprint, 32) != 0 || + memcmp(header + 88, index->feature_parameter_fingerprint, 32) != 0) { + close(fd); + return LARDON3D_VISUAL_INDEX_CORRUPT; + } + for (size_t i = 120; i < 128; ++i) { + if (header[i] != 0) { + close(fd); + return LARDON3D_VISUAL_INDEX_CORRUPT; + } + } + *descriptor = fd; + *posting_count = count; + return LARDON3D_VISUAL_INDEX_OK; +} + +static bool read_posting(int descriptor, uint64_t index, Posting *posting) { + unsigned char record[POSTING_SIZE]; + if (!read_exact(descriptor, record, sizeof(record), + (off_t)(SEGMENT_HEADER_SIZE + index * POSTING_SIZE))) { + return false; + } + posting->table = get_u32(record); + posting->key = get_u32(record + 4); + posting->feature_set_id = get_u64(record + 8); + posting->feature_index = get_u32(record + 16); + return get_u32(record + 20) == 0 && posting->table < LARDON3D_VISUAL_INDEX_TABLE_COUNT && + posting->feature_set_id > 0 && posting->feature_index < 8192; +} + +static int posting_key_compare(const Posting *posting, uint32_t table, uint32_t key) { + if (posting->table != table) { + return posting->table < table ? -1 : 1; + } + return posting->key < key ? -1 : posting->key > key; +} + +static bool bucket_range(int descriptor, uint64_t count, uint32_t table, uint32_t key, + uint64_t *begin, uint64_t *end) { + uint64_t low = 0; + uint64_t high = count; + Posting posting; + while (low < high) { + uint64_t middle = low + (high - low) / 2; + if (!read_posting(descriptor, middle, &posting)) { + return false; + } + if (posting_key_compare(&posting, table, key) < 0) { + low = middle + 1; + } else { + high = middle; + } + } + *begin = low; + high = count; + while (low < high) { + uint64_t middle = low + (high - low) / 2; + if (!read_posting(descriptor, middle, &posting)) { + return false; + } + if (posting_key_compare(&posting, table, key) <= 0) { + low = middle + 1; + } else { + high = middle; + } + } + *end = low; + return true; +} + +static void add_evidence(Accumulator *accumulators, size_t *count, uint64_t feature_set_id, + uint32_t query_feature) { + for (size_t i = 0; i < *count; ++i) { + if (accumulators[i].feature_set_id == feature_set_id) { + if (accumulators[i].last_query_feature != query_feature) { + ++accumulators[i].evidence; + accumulators[i].last_query_feature = query_feature; + } + return; + } + } + if (*count < LARDON3D_VISUAL_INDEX_CANDIDATE_MAX) { + accumulators[*count] = (Accumulator){feature_set_id, 1, query_feature}; + ++*count; + } +} + +static Lardon3DVisualIndexResult count_segment_buckets( + const char *project_path, const Lardon3DProjectDbVisualIndex *index, + const Lardon3DProjectDbVisualIndexSegment *segment, const unsigned char *descriptors, + uint32_t query_count, uint32_t *bucket_sizes) { + int descriptor = -1; + uint64_t posting_count = 0; + Lardon3DVisualIndexResult result = + open_segment(project_path, index, segment, &descriptor, &posting_count); + if (result != LARDON3D_VISUAL_INDEX_OK) { + return result; + } + for (uint32_t query = 0; query < query_count; ++query) { + for (uint32_t table = 0; table < index->table_count; ++table) { + size_t slot = (size_t)query * index->table_count + table; + if (bucket_sizes[slot] > index->max_bucket_postings) { + continue; + } + uint64_t begin; + uint64_t end; + uint32_t key = lsh_key(descriptors + (size_t)query * DESCRIPTOR_SIZE, table); + if (!bucket_range(descriptor, posting_count, table, key, &begin, &end)) { + close(descriptor); + return LARDON3D_VISUAL_INDEX_CORRUPT; + } + uint64_t available = index->max_bucket_postings + 1U - bucket_sizes[slot]; + uint64_t amount = end - begin; + bucket_sizes[slot] += (uint32_t)(amount < available ? amount : available); + } + } + close(descriptor); + return LARDON3D_VISUAL_INDEX_OK; +} + +static Lardon3DVisualIndexResult query_segment( + const char *project_path, const Lardon3DProjectDbVisualIndex *index, + const Lardon3DProjectDbVisualIndexSegment *segment, const unsigned char *descriptors, + uint32_t query_count, const uint32_t *bucket_sizes, uint64_t self, + Accumulator *accumulators, size_t *accumulator_count) { + int descriptor = -1; + uint64_t posting_count = 0; + Lardon3DVisualIndexResult result = + open_segment(project_path, index, segment, &descriptor, &posting_count); + if (result != LARDON3D_VISUAL_INDEX_OK) { + return result; + } + Posting posting; + for (uint32_t query = 0; query < query_count; ++query) { + for (uint32_t table = 0; table < index->table_count; ++table) { + if (bucket_sizes[(size_t)query * index->table_count + table] > + index->max_bucket_postings) { + continue; + } + uint64_t begin; + uint64_t end; + uint32_t key = lsh_key(descriptors + (size_t)query * DESCRIPTOR_SIZE, table); + if (!bucket_range(descriptor, posting_count, table, key, &begin, &end)) { + close(descriptor); + return LARDON3D_VISUAL_INDEX_CORRUPT; + } + for (uint64_t offset = begin; offset < end; ++offset) { + if (!read_posting(descriptor, offset, &posting)) { + close(descriptor); + return LARDON3D_VISUAL_INDEX_CORRUPT; + } + if (posting.feature_set_id != self) { + add_evidence(accumulators, accumulator_count, posting.feature_set_id, query); + } + } + } + } + close(descriptor); + return LARDON3D_VISUAL_INDEX_OK; +} + +static int compare_candidate(const void *left, const void *right) { + const Lardon3DVisualIndexCandidate *a = left; + const Lardon3DVisualIndexCandidate *b = right; + if (a->score != b->score) { + return a->score > b->score ? -1 : 1; + } + if (a->evidence_count != b->evidence_count) { + return a->evidence_count > b->evidence_count ? -1 : 1; + } + if (a->image_id != b->image_id) { + return a->image_id < b->image_id ? -1 : 1; + } + return a->feature_set_id < b->feature_set_id ? -1 : a->feature_set_id > b->feature_set_id; +} + +Lardon3DVisualIndexResult lardon3d_visual_index_query( + const char *project_path, Lardon3DProjectDb *database, uint64_t index_id, + uint64_t query_id, const Lardon3DVisualIndexQueryOptions *options, + Lardon3DVisualIndexCandidate *results, size_t capacity, size_t *result_count) { + if (result_count) { + *result_count = 0; + } + if (!project_path || !database || index_id == 0 || query_id == 0 || !options || !results || + !result_count || options->top_k == 0 || options->top_k > LARDON3D_VISUAL_INDEX_TOP_K_MAX || + capacity < options->top_k || options->minimum_evidence_count > 1024 || + options->scanset_filter > LARDON3D_VISUAL_INDEX_OTHER_SCANSETS) { + return LARDON3D_VISUAL_INDEX_INVALID_ARGUMENT; + } + Lardon3DProjectDbVisualIndex index; + Lardon3DProjectDbFeatureSet query_set; + Lardon3DProjectDbResult db = lardon3d_project_db_load_visual_index(database, index_id, &index); + if (db == LARDON3D_PROJECT_DB_OK) { + db = lardon3d_project_db_load_feature_set(database, query_id, &query_set); + } + if (db != LARDON3D_PROJECT_DB_OK) { + return db_result(db); + } + if (query_set.descriptor_type != index.descriptor_type || + query_set.descriptor_dimension != index.descriptor_dimension || + query_set.extractor_version != index.extractor_version || + strcmp(query_set.extractor_kind, index.extractor_kind) != 0 || + memcmp(query_set.parameter_fingerprint, index.feature_parameter_fingerprint, 32) != 0) { + return LARDON3D_VISUAL_INDEX_INCOMPATIBLE; + } + Lardon3DProjectDbImage query_image; + Lardon3DProjectDbImageAsset query_asset; + db = lardon3d_project_db_load_image(database, query_set.image_id, &query_image, &query_asset); + if (db != LARDON3D_PROJECT_DB_OK) { + return db_result(db); + } + uint32_t query_count = query_set.feature_count < index.max_features_per_set + ? query_set.feature_count + : index.max_features_per_set; + unsigned char *descriptors = query_count ? malloc((size_t)query_count * DESCRIPTOR_SIZE) : NULL; + if (query_count && !descriptors) { + return LARDON3D_VISUAL_INDEX_IO_ERROR; + } + Lardon3DFeatureReader *reader = NULL; + Lardon3DFeatureFileMetadata metadata; + Lardon3DFeatureStoreResult opened = + lardon3d_feature_reader_open(project_path, &query_set, &reader, &metadata); + for (uint32_t start = 0; opened == LARDON3D_FEATURE_STORE_OK && start < query_count;) { + uint32_t amount = query_count - start; + if (amount > LARDON3D_FEATURE_READER_RANGE_MAX) { + amount = LARDON3D_FEATURE_READER_RANGE_MAX; + } + opened = lardon3d_feature_reader_descriptors(reader, start, + descriptors + (size_t)start * DESCRIPTOR_SIZE, + amount, (size_t)amount * DESCRIPTOR_SIZE); + start += amount; + } + lardon3d_feature_reader_close(reader); + if (opened != LARDON3D_FEATURE_STORE_OK) { + free(descriptors); + return opened == LARDON3D_FEATURE_STORE_UNSUPPORTED_VERSION + ? LARDON3D_VISUAL_INDEX_UNSUPPORTED_VERSION + : LARDON3D_VISUAL_INDEX_CORRUPT; + } + Lardon3DProjectDbVisualIndexSegment segments[LARDON3D_VISUAL_INDEX_SEGMENT_MAX]; + size_t segment_count = 0; + db = lardon3d_project_db_list_visual_index_segments(database, index_id, 0, segments, + LARDON3D_VISUAL_INDEX_SEGMENT_MAX, + &segment_count); + if (db != LARDON3D_PROJECT_DB_OK) { + free(descriptors); + return db_result(db); + } + Accumulator *accumulators = calloc(LARDON3D_VISUAL_INDEX_CANDIDATE_MAX, sizeof(*accumulators)); + if (!accumulators) { + free(descriptors); + return LARDON3D_VISUAL_INDEX_IO_ERROR; + } + size_t accumulator_count = 0; + Lardon3DVisualIndexResult result = LARDON3D_VISUAL_INDEX_OK; + size_t bucket_count = (size_t)query_count * index.table_count; + uint32_t *bucket_sizes = bucket_count ? calloc(bucket_count, sizeof(*bucket_sizes)) : NULL; + if (bucket_count && !bucket_sizes) { + free(descriptors); + free(accumulators); + return LARDON3D_VISUAL_INDEX_IO_ERROR; + } + for (size_t i = 0; i < segment_count && result == LARDON3D_VISUAL_INDEX_OK; ++i) { + result = count_segment_buckets(project_path, &index, &segments[i], descriptors, query_count, + bucket_sizes); + } + for (size_t i = 0; i < segment_count && result == LARDON3D_VISUAL_INDEX_OK; ++i) { + result = query_segment(project_path, &index, &segments[i], descriptors, query_count, + bucket_sizes, query_id, accumulators, &accumulator_count); + } + free(bucket_sizes); + free(descriptors); + Lardon3DVisualIndexCandidate *candidates = + calloc(LARDON3D_VISUAL_INDEX_CANDIDATE_MAX, sizeof(*candidates)); + if (!candidates) { + free(accumulators); + return LARDON3D_VISUAL_INDEX_IO_ERROR; + } + size_t output = 0; + for (size_t i = 0; i < accumulator_count && result == LARDON3D_VISUAL_INDEX_OK; ++i) { + if (accumulators[i].evidence < options->minimum_evidence_count) { + continue; + } + Lardon3DProjectDbFeatureSet set; + Lardon3DProjectDbImage image; + Lardon3DProjectDbImageAsset asset; + db = lardon3d_project_db_load_feature_set(database, accumulators[i].feature_set_id, &set); + if (db == LARDON3D_PROJECT_DB_OK) { + db = lardon3d_project_db_load_image(database, set.image_id, &image, &asset); + } + if (db != LARDON3D_PROJECT_DB_OK) { + result = db_result(db); + break; + } + bool same_asset = asset.asset_id == query_asset.asset_id; + bool scan_allowed = options->scanset_filter == LARDON3D_VISUAL_INDEX_ANY_SCANSET || + (options->scanset_filter == LARDON3D_VISUAL_INDEX_SAME_SCANSET && + image.scanset_id == query_image.scanset_id) || + (options->scanset_filter == LARDON3D_VISUAL_INDEX_OTHER_SCANSETS && + image.scanset_id != query_image.scanset_id); + if (!scan_allowed || (options->exclude_same_asset && same_asset) || + image.image_id == query_image.image_id || output >= LARDON3D_VISUAL_INDEX_CANDIDATE_MAX) { + continue; + } + candidates[output++] = (Lardon3DVisualIndexCandidate){ + .feature_set_id = set.feature_set_id, + .image_id = image.image_id, + .scanset_id = image.scanset_id, + .score = query_count ? (double)accumulators[i].evidence / query_count : 0.0, + .evidence_count = accumulators[i].evidence, + .same_image_asset = same_asset, + }; + } + free(accumulators); + if (result != LARDON3D_VISUAL_INDEX_OK) { + free(candidates); + return result; + } + qsort(candidates, output, sizeof(*candidates), compare_candidate); + if (output > options->top_k) { + output = options->top_k; + } + memcpy(results, candidates, output * sizeof(*results)); + free(candidates); + *result_count = output; + return LARDON3D_VISUAL_INDEX_OK; +} diff --git a/src/visual_index_task.c b/src/visual_index_task.c new file mode 100644 index 0000000..8ae47a0 --- /dev/null +++ b/src/visual_index_task.c @@ -0,0 +1,222 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +typedef struct { + char project_path[PATH_MAX]; + Lardon3DProjectDb *database; + Lardon3DResourceGovernor *governor; + Lardon3DProjectDbVisualIndexUpdateTask parameters; +} VisualIndexTaskContext; + +static void destroy_context(void *userdata) { free(userdata); } + +static void runtime_state(const VisualIndexTaskContext *context, Lardon3DAppState *state) { + lardon3d_app_state_init(state); + state->project_loaded = true; + state->project_db = context->database; + state->resource_governor = context->governor; + snprintf(state->project_path, sizeof(state->project_path), "%s", context->project_path); +} + +static uint64_t elapsed_ns(struct timespec begin, struct timespec end) { + uint64_t seconds = end.tv_sec >= begin.tv_sec ? (uint64_t)(end.tv_sec - begin.tv_sec) : 0; + long nanoseconds = end.tv_nsec - begin.tv_nsec; + if (nanoseconds < 0 && seconds > 0) { + --seconds; + nanoseconds += 1000000000L; + } + return seconds <= UINT64_MAX / 1000000000ULL + ? seconds * 1000000000ULL + (uint64_t)nanoseconds + : UINT64_MAX; +} + +static void finished(const Lardon3DTask *task, void *userdata) { +#ifdef LARDON3D_VISUAL_INDEX_TASK_TESTING + const char *skip = getenv("LARDON3D_TEST_VISUAL_INDEX_SKIP_FINISHED_CHECKPOINT"); + if (skip && strcmp(skip, "1") == 0) { + return; + } +#endif + VisualIndexTaskContext *context = userdata; + Lardon3DAppState state; + runtime_state(context, &state); + (void)lardon3d_project_checkpoint_visual_index_update_task(&state, task, + &context->parameters); +} + +static bool run(Lardon3DTask *task, void *userdata) { + VisualIndexTaskContext *context = userdata; + Lardon3DAppState state; + runtime_state(context, &state); + for (;;) { + if (!lardon3d_task_checkpoint(task)) { + return false; + } + Lardon3DTaskExecutionContract contract; + if (!lardon3d_task_execution_contract(task, &contract) || contract.batch_size == 0 || + contract.batch_size > LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX) { + return lardon3d_task_fail(task, "Contrat Visual Index invalide."); + } + struct timespec begin; + struct timespec end; + clock_gettime(CLOCK_MONOTONIC, &begin); + uint64_t last = context->parameters.after_feature_set_id; + size_t indexed = 0; + Lardon3DVisualIndexResult result = lardon3d_visual_index_update_once( + context->project_path, context->database, context->parameters.visual_index_id, + lardon3d_task_id(task), context->parameters.after_feature_set_id, contract.batch_size, + &last, &indexed); + clock_gettime(CLOCK_MONOTONIC, &end); + if (result == LARDON3D_VISUAL_INDEX_NO_CHANGE) { + return lardon3d_task_set_progress(task, 100, "Visual Index à jour."); + } + if (result != LARDON3D_VISUAL_INDEX_OK && + result != LARDON3D_VISUAL_INDEX_PUBLISHED_NOT_DURABLE) { + return lardon3d_task_fail(task, "Mise à jour Visual Index impossible."); + } + context->parameters.after_feature_set_id = last; + lardon3d_resource_governor_record_batch(context->governor, LARDON3D_RESOURCE_TASK_CPU, + indexed, elapsed_ns(begin, end), 0); + if (!lardon3d_task_set_progress(task, 99, "Segment Visual Index publié.") || + lardon3d_project_checkpoint_visual_index_update_task(&state, task, + &context->parameters) != + LARDON3D_PROJECT_TASK_CHECKPOINT_OK) { + return lardon3d_task_fail(task, "Checkpoint Visual Index impossible."); + } +#ifdef LARDON3D_VISUAL_INDEX_TASK_TESTING + const char *pause = getenv("LARDON3D_TEST_VISUAL_INDEX_PAUSE_AFTER_SEGMENT"); + if (pause && strcmp(pause, "1") == 0) { + (void)lardon3d_task_pause(task); + } +#endif + Lardon3DResourceReservation *reservation = NULL; + if (!lardon3d_task_sequence_break(task, context->governor, &reservation, &contract)) { + return false; + } + } +} + +static VisualIndexTaskContext *make_context( + const Lardon3DTaskReconstructionContext *runtime, + const Lardon3DProjectDbVisualIndexUpdateTask *parameters) { + if (!runtime || !runtime->project_path || !runtime->project_db || + !runtime->resource_governor || !parameters) { + return NULL; + } + VisualIndexTaskContext *context = calloc(1, sizeof(*context)); + if (!context) { + return NULL; + } + int written = snprintf(context->project_path, sizeof(context->project_path), "%s", + runtime->project_path); + if (written <= 0 || (size_t)written >= sizeof(context->project_path)) { + free(context); + return NULL; + } + context->database = runtime->project_db; + context->governor = runtime->resource_governor; + context->parameters = *parameters; + return context; +} + +bool lardon3d_visual_index_update_reconstruct(const Lardon3DTaskDurableSnapshot *snapshot, + void *userdata, + Lardon3DTaskKindBinding *binding) { + Lardon3DTaskReconstructionContext *runtime = userdata; + if (!snapshot || !runtime || !binding) { + return false; + } + Lardon3DProjectDbVisualIndexUpdateTask parameters; + if (lardon3d_project_db_load_visual_index_update_task(runtime->project_db, snapshot->id, + ¶meters) != LARDON3D_PROJECT_DB_OK) { + return false; + } + VisualIndexTaskContext *context = make_context(runtime, ¶meters); + if (!context) { + return false; + } + *binding = (Lardon3DTaskKindBinding){.callback = run, + .userdata = context, + .userdata_destroy = destroy_context, + .finished_callback = finished, + .finished_userdata = context}; + return true; +} + +Lardon3DTask *lardon3d_project_create_visual_index_update_task( + Lardon3DAppState *state, uint64_t visual_index_id, uint64_t *task_id) { + if (task_id) { + *task_id = 0; + } + if (!state || !state->project_loaded || !state->project_db || !state->resource_governor || + !task_id || visual_index_id == 0) { + return NULL; + } + Lardon3DProjectDbVisualIndex index; + if (lardon3d_project_db_load_visual_index(state->project_db, visual_index_id, &index) != + LARDON3D_PROJECT_DB_OK) { + return NULL; + } + uint64_t id = 0; + if (lardon3d_project_db_allocate_task_id(state->project_db, &id) != LARDON3D_PROJECT_DB_OK) { + return NULL; + } + Lardon3DProjectDbVisualIndexUpdateTask parameters = { + .task_id = id, + .visual_index_id = visual_index_id, + .after_feature_set_id = 0, + }; + Lardon3DTaskReconstructionContext runtime = {state->project_path, state->project_db, + state->resource_governor}; + VisualIndexTaskContext *context = make_context(&runtime, ¶meters); + if (!context) { + return NULL; + } + Lardon3DResourceEstimate estimate = {.memory_fixed_bytes = 8ULL * 1024 * 1024, + .memory_bytes_per_item = 2ULL * 1024 * 1024, + .minimum_batch_size = 1, + .maximum_batch_size = 16, + .desired_cpu_threads = 1, + .desired_io_slots = 1, + .task_class = LARDON3D_RESOURCE_TASK_CPU}; + Lardon3DTask *task = lardon3d_task_create_typed( + "Mise à jour Visual Index", &estimate, LARDON3D_VISUAL_INDEX_UPDATE_TASK_KIND, + LARDON3D_VISUAL_INDEX_UPDATE_TASK_KIND_VERSION, run, context, destroy_context); + if (!task || !lardon3d_task_assign_id(task, id) || + !lardon3d_task_set_finished_callback(task, finished, context) || + lardon3d_project_checkpoint_visual_index_update_task(state, task, ¶meters) != + LARDON3D_PROJECT_TASK_CHECKPOINT_OK) { + lardon3d_task_destroy(task); + return NULL; + } + *task_id = id; + return task; +} + +bool lardon3d_project_enqueue_visual_index_update(Lardon3DAppState *state, + uint64_t visual_index_id, + uint64_t *task_id) { + if (!state || !state->task_queue) { + return false; + } + Lardon3DTask *task = + lardon3d_project_create_visual_index_update_task(state, visual_index_id, task_id); + if (!task) { + return false; + } + if (!lardon3d_task_queue_add(state->task_queue, task, NULL)) { + lardon3d_task_destroy(task); + return false; + } + return true; +} diff --git a/tests/test_feature_task.c b/tests/test_feature_task.c index 1050b6f..c0250f9 100644 --- a/tests/test_feature_task.c +++ b/tests/test_feature_task.c @@ -11,6 +11,8 @@ #include #include +#include +#include #include #include #include @@ -149,6 +151,33 @@ static bool run_test(void) { LARDON3D_FEATURE_STORE_OK); lardon3d_feature_reader_close(reader); + Lardon3DVisualIndexConfiguration index_configuration = {1, 256, 128}; + uint64_t visual_index_id = 0; + CHECK(lardon3d_visual_index_create(state.project_db, &set, &index_configuration, + &visual_index_id) == LARDON3D_VISUAL_INDEX_OK); + CHECK(setenv("LARDON3D_TEST_VISUAL_INDEX_PAUSE_AFTER_SEGMENT", "1", 1) == 0 && + setenv("LARDON3D_TEST_VISUAL_INDEX_SKIP_FINISHED_CHECKPOINT", "1", 1) == 0); + uint64_t visual_task_id = 0; + CHECK(lardon3d_project_enqueue_visual_index_update(&state, visual_index_id, &visual_task_id)); + CHECK(wait_state(state.task_queue, visual_task_id, TASK_PAUSED, &snapshot)); + lardon3d_task_queue_destroy(state.task_queue); + state.task_queue = NULL; + lardon3d_project_close(&state); + lardon3d_resource_governor_destroy(state.resource_governor); + state.resource_governor = NULL; + CHECK(unsetenv("LARDON3D_TEST_VISUAL_INDEX_PAUSE_AFTER_SEGMENT") == 0 && + unsetenv("LARDON3D_TEST_VISUAL_INDEX_SKIP_FINISHED_CHECKPOINT") == 0); + lardon3d_app_state_init(&state); + CHECK(runtime(&state) && lardon3d_project_open(&state, "Features")); + CHECK(lardon3d_project_last_recovery_summary(&state, &summary) && summary.resumed == 1); + CHECK(wait_state(state.task_queue, visual_task_id, TASK_COMPLETED, &snapshot)); + Lardon3DProjectDbVisualIndexSegment visual_segments[2]; + size_t visual_segment_count = 0; + CHECK(lardon3d_project_db_list_visual_index_segments( + state.project_db, visual_index_id, 0, visual_segments, 2, &visual_segment_count) == + LARDON3D_PROJECT_DB_OK && + visual_segment_count == 1); + CHECK(write_pgm(source)); Lardon3DProjectDbImage missing_image; Lardon3DProjectDbImageAsset missing_asset; diff --git a/tests/test_project_db.c b/tests/test_project_db.c index fd1aa16..d723f0a 100644 --- a/tests/test_project_db.c +++ b/tests/test_project_db.c @@ -89,7 +89,7 @@ static bool create_future_database(const char *path) { } bool ok = sqlite3_exec(connection, "CREATE TABLE metadata(key TEXT PRIMARY KEY,value INTEGER NOT NULL);" - "INSERT INTO metadata VALUES('schema_version',6);", + "INSERT INTO metadata VALUES('schema_version',7);", NULL, NULL, NULL) == SQLITE_OK; return sqlite3_close(connection) == SQLITE_OK && ok; } @@ -227,7 +227,7 @@ static bool run_test(void) { char database_path[512], artifact_path[512], future_path[512], corrupt_path[512]; char legacy_path[512], failed_migration_path[512], v2_path[512]; char failed_v3_migration_path[512], v3_path[512], failed_v4_path[512]; - char v4_path[512], failed_v5_path[512]; + char v4_path[512], failed_v5_path[512], failed_v6_path[512]; CHECK(snprintf(database_path, sizeof(database_path), "%s/project.db", directory) > 0); CHECK(snprintf(artifact_path, sizeof(artifact_path), "%s/artifact.bin", directory) > 0); CHECK(snprintf(future_path, sizeof(future_path), "%s/future.db", directory) > 0); @@ -244,11 +244,13 @@ static bool run_test(void) { CHECK(snprintf(v4_path, sizeof(v4_path), "%s/v4.db", directory) > 0); CHECK(snprintf(failed_v5_path, sizeof(failed_v5_path), "%s/failed-v5-migration.db", directory) > 0); + CHECK(snprintf(failed_v6_path, sizeof(failed_v6_path), "%s/failed-v6-migration.db", directory) > + 0); char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY]; Lardon3DProjectDb *database = NULL; CHECK(lardon3d_project_db_open(database_path, &database, error) == LARDON3D_PROJECT_DB_OK); - CHECK(database && lardon3d_project_db_schema_version(database) == 5); + CHECK(database && lardon3d_project_db_schema_version(database) == 6); bool legacy_pending = true; CHECK(lardon3d_project_db_legacy_catalog_pending(database, &legacy_pending) == LARDON3D_PROJECT_DB_OK && @@ -436,7 +438,7 @@ static bool run_test(void) { LARDON3D_PROJECT_DB_INVALID_ARGUMENT); lardon3d_project_db_close(contexts[0].database); database = NULL; - CHECK(query_integer(database_path, "SELECT value FROM metadata WHERE key='schema_version'", 5)); + CHECK(query_integer(database_path, "SELECT value FROM metadata WHERE key='schema_version'", 6)); CHECK(query_integer(database_path, "SELECT count(*) FROM tasks WHERE task_id=1", 1)); CHECK(lardon3d_project_db_open(database_path, &database, error) == LARDON3D_PROJECT_DB_OK); CHECK(lardon3d_project_db_load_task(database, 1, &task) == LARDON3D_PROJECT_DB_OK); @@ -454,7 +456,7 @@ static bool run_test(void) { CHECK(create_v1_database(legacy_path)); CHECK(lardon3d_project_db_open(legacy_path, &database, error) == LARDON3D_PROJECT_DB_OK); - CHECK(lardon3d_project_db_schema_version(database) == 5); + CHECK(lardon3d_project_db_schema_version(database) == 6); CHECK(lardon3d_project_db_get_project(database, &loaded_project) == LARDON3D_PROJECT_DB_OK && strcmp(loaded_project.stable_id, "legacy-project") == 0); CHECK(lardon3d_project_db_load_task(database, 9, &task) == LARDON3D_PROJECT_DB_OK); @@ -464,7 +466,7 @@ static bool run_test(void) { LARDON3D_PROJECT_DB_OK); lardon3d_project_db_close(database); database = NULL; - CHECK(query_integer(legacy_path, "SELECT value FROM metadata WHERE key='schema_version'", 5)); + CHECK(query_integer(legacy_path, "SELECT value FROM metadata WHERE key='schema_version'", 6)); CHECK(create_v1_database(failed_migration_path)); CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V2", "1", 1) == 0); @@ -489,7 +491,7 @@ static bool run_test(void) { LARDON3D_PROJECT_DB_OK); lardon3d_project_db_close(database); database = NULL; - CHECK(query_integer(v2_path, "SELECT value FROM metadata WHERE key='schema_version'", 5)); + CHECK(query_integer(v2_path, "SELECT value FROM metadata WHERE key='schema_version'", 6)); CHECK(create_v2_database(failed_v3_migration_path)); CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V3", "1", 1) == 0); @@ -517,7 +519,7 @@ static bool run_test(void) { LARDON3D_PROJECT_DB_OK); lardon3d_project_db_close(database); database = NULL; - CHECK(query_integer(v3_path, "SELECT value FROM metadata WHERE key='schema_version'", 5)); + CHECK(query_integer(v3_path, "SELECT value FROM metadata WHERE key='schema_version'", 6)); CHECK(create_v3_database(failed_v4_path)); CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V4", "1", 1) == 0); @@ -534,7 +536,7 @@ static bool run_test(void) { fprintf(stderr, "Migration v4 (%d): %s\n", (int)v4_result, error); } CHECK(v4_result == LARDON3D_PROJECT_DB_OK); - CHECK(lardon3d_project_db_schema_version(database) == 5); + CHECK(lardon3d_project_db_schema_version(database) == 6); CHECK(lardon3d_project_db_load_task(database, 9, &task) == LARDON3D_PROJECT_DB_OK); CHECK(lardon3d_project_db_load_artifact(database, "legacy-artifact", &loaded_artifact) == LARDON3D_PROJECT_DB_OK); @@ -550,6 +552,15 @@ static bool run_test(void) { failed_v5_path, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='feature_sets'", 0)); + CHECK(create_v4_database(failed_v6_path)); + CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V6", "1", 1) == 0); + CHECK(lardon3d_project_db_open(failed_v6_path, &database, error) == LARDON3D_PROJECT_DB_IO_ERROR); + CHECK(unsetenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V6") == 0); + CHECK(query_integer(failed_v6_path, "SELECT value FROM metadata WHERE key='schema_version'", 4)); + CHECK(query_integer( + failed_v6_path, + "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='visual_indexes'", 0)); + CHECK(unlink(artifact_path) == 0); CHECK(unlink(database_path) == 0); CHECK(unlink(future_path) == 0); @@ -562,6 +573,7 @@ static bool run_test(void) { CHECK(unlink(failed_v4_path) == 0); CHECK(unlink(v4_path) == 0); CHECK(unlink(failed_v5_path) == 0); + CHECK(unlink(failed_v6_path) == 0); CHECK(rmdir(directory) == 0); return true; } diff --git a/tests/test_visual_index.c b/tests/test_visual_index.c new file mode 100644 index 0000000..6778bb1 --- /dev/null +++ b/tests/test_visual_index.c @@ -0,0 +1,811 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#define CHECK(condition) \ + do { \ + if (!(condition)) { \ + fprintf(stderr, "Échec ligne %d : %s\n", __LINE__, #condition); \ + return false; \ + } \ + } while (0) + +typedef struct { + const char *root; + Lardon3DProjectDb *database; + uint64_t index_id; + uint64_t query_id; + bool ok; +} QueryThread; + +typedef struct { + pthread_barrier_t barrier; + const char *root; + Lardon3DProjectDb *database; + uint64_t index_id; + Lardon3DVisualIndexResult result[2]; +} UpdateRace; + +typedef struct { + UpdateRace *race; + size_t slot; +} UpdateRaceThread; + +static size_t candidate_rank(const Lardon3DVisualIndexCandidate *candidates, size_t count, + uint64_t feature_set_id) { + for (size_t i = 0; i < count; ++i) { + if (candidates[i].feature_set_id == feature_set_id) { + return i; + } + } + return SIZE_MAX; +} + +static bool join_path(char output[PATH_MAX], const char *left, const char *right) { + int written = snprintf(output, PATH_MAX, "%s/%s", left, right); + return written > 0 && (size_t)written < PATH_MAX; +} + +static bool remove_tree(const char *path) { + struct stat status; + if (lstat(path, &status) != 0) { + return errno == ENOENT; + } + if (!S_ISDIR(status.st_mode)) { + return unlink(path) == 0; + } + DIR *directory = opendir(path); + if (!directory) { + return false; + } + bool ok = true; + for (struct dirent *entry = readdir(directory); entry; entry = readdir(directory)) { + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) { + continue; + } + char child[PATH_MAX]; + if (!join_path(child, path, entry->d_name) || !remove_tree(child)) { + ok = false; + } + } + return closedir(directory) == 0 && rmdir(path) == 0 && ok; +} + +static bool write_source(const char *path, unsigned char value) { + int descriptor = open(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0600); + if (descriptor < 0) { + return false; + } + unsigned char bytes[64]; + memset(bytes, value, sizeof(bytes)); + bool ok = write(descriptor, bytes, sizeof(bytes)) == (ssize_t)sizeof(bytes); + return close(descriptor) == 0 && ok; +} + +static unsigned char scene_pixel(unsigned int x, unsigned int y, unsigned int seed) { + uint32_t value = x * 2654435761U ^ y * 2246822519U ^ seed * 3266489917U; + value ^= value >> 13; + value *= 1274126177U; + return (unsigned char)(value >> 24); +} + +static bool write_scene(const char *path, unsigned int shift, unsigned int seed) { + FILE *file = fopen(path, "wb"); + if (!file || fprintf(file, "P5\n256 256\n255\n") <= 0) { + if (file) { + fclose(file); + } + return false; + } + for (unsigned int y = 0; y < 256; ++y) { + for (unsigned int x = 0; x < 256; ++x) { + unsigned char value = x >= shift ? scene_pixel(x - shift, y, seed) : 0; + if (fwrite(&value, 1, 1, file) != 1) { + fclose(file); + return false; + } + } + } + return fclose(file) == 0; +} + +static bool write_transformed_scene(const char *path, unsigned int seed, bool crop, + double angle_degrees) { + unsigned int width = crop ? 192U : 256U; + FILE *file = fopen(path, "wb"); + if (!file || fprintf(file, "P5\n%u %u\n255\n", width, width) <= 0) { + if (file) { + fclose(file); + } + return false; + } + double angle = angle_degrees * 3.14159265358979323846 / 180.0; + double cosine = cos(angle); + double sine = sin(angle); + for (unsigned int y = 0; y < width; ++y) { + for (unsigned int x = 0; x < width; ++x) { + double output_x = crop ? (double)x + 32.0 : (double)x; + double output_y = crop ? (double)y + 32.0 : (double)y; + double dx = output_x - 127.5; + double dy = output_y - 127.5; + int source_x = (int)(127.5 + cosine * dx + sine * dy + 0.5); + int source_y = (int)(127.5 - sine * dx + cosine * dy + 0.5); + unsigned char value = source_x >= 0 && source_x < 256 && source_y >= 0 && source_y < 256 + ? scene_pixel((unsigned int)source_x, (unsigned int)source_y, seed) + : 0; + if (fwrite(&value, 1, 1, file) != 1) { + fclose(file); + return false; + } + } + } + return fclose(file) == 0; +} + +static bool publish_real_features(Lardon3DAppState *state, uint64_t image_id, + const char *source, Lardon3DProjectDbFeatureSet *set) { + Lardon3DFeatureExtractorParameters parameters = {512, 4, 20}; + Lardon3DExtractedFeatures features; + if (lardon3d_feature_extract_orb(source, ¶meters, &features) != + LARDON3D_FEATURE_EXTRACT_OK) { + return false; + } + Lardon3DFeatureStoreResult result = + lardon3d_feature_store_publish(state, image_id, 0, ¶meters, &features, set); + lardon3d_extracted_features_destroy(&features); + return result == LARDON3D_FEATURE_STORE_OK || + result == LARDON3D_FEATURE_STORE_PUBLISHED_NOT_DURABLE; +} + +static bool publish_product_features(Lardon3DAppState *state, uint64_t image_id, + const char *source, Lardon3DProjectDbFeatureSet *set) { + Lardon3DFeatureExtractorParameters parameters = {640, 4, 20}; + Lardon3DExtractedFeatures features; + if (lardon3d_feature_extract_orb(source, ¶meters, &features) != + LARDON3D_FEATURE_EXTRACT_OK) { + return false; + } + Lardon3DFeatureStoreResult result = + lardon3d_feature_store_publish(state, image_id, 0, ¶meters, &features, set); + lardon3d_extracted_features_destroy(&features); + return result == LARDON3D_FEATURE_STORE_OK || + result == LARDON3D_FEATURE_STORE_PUBLISHED_NOT_DURABLE; +} + +static bool publish_features(Lardon3DAppState *state, uint64_t image_id, unsigned int family, + Lardon3DProjectDbFeatureSet *set) { + enum { COUNT = 16 }; + Lardon3DFeatureKeypoint keypoints[COUNT]; + unsigned char descriptors[COUNT * 32]; + for (uint32_t i = 0; i < COUNT; ++i) { + keypoints[i] = (Lardon3DFeatureKeypoint){(float)i, (float)i, 8.0F, 0.0F, + (float)(COUNT - i), 0}; + for (uint32_t byte = 0; byte < 32; ++byte) { + descriptors[i * 32 + byte] = + family == 0 || (family == 1 && i < 10) + ? (unsigned char)(i * 17U + byte * 29U) + : (unsigned char)(family * 73U + i * 31U + byte * 7U); + } + } + Lardon3DExtractedFeatures features = { + .image_width = 64, + .image_height = 64, + .feature_count = COUNT, + .keypoints = keypoints, + .descriptors = descriptors, + }; + Lardon3DFeatureExtractorParameters parameters = {64, 4, 20}; + Lardon3DFeatureStoreResult result = + lardon3d_feature_store_publish(state, image_id, 0, ¶meters, &features, set); + return result == LARDON3D_FEATURE_STORE_OK || + result == LARDON3D_FEATURE_STORE_PUBLISHED_NOT_DURABLE; +} + +static bool publish_empty_features(Lardon3DAppState *state, uint64_t image_id, + Lardon3DProjectDbFeatureSet *set) { + Lardon3DExtractedFeatures features = {.image_width = 64, .image_height = 64}; + Lardon3DFeatureExtractorParameters parameters = {64, 4, 20}; + Lardon3DFeatureStoreResult result = + lardon3d_feature_store_publish(state, image_id, 0, ¶meters, &features, set); + return result == LARDON3D_FEATURE_STORE_OK || + result == LARDON3D_FEATURE_STORE_PUBLISHED_NOT_DURABLE; +} + +static bool publish_burst_features(Lardon3DAppState *state, uint64_t image_id, unsigned int kind, + Lardon3DProjectDbFeatureSet *set) { + enum { COUNT = 16 }; + Lardon3DFeatureKeypoint keypoints[COUNT]; + unsigned char descriptors[COUNT * 32]; + memset(descriptors, 0, sizeof(descriptors)); + for (uint32_t i = 0; i < COUNT; ++i) { + keypoints[i] = (Lardon3DFeatureKeypoint){(float)i, (float)i, 8.0F, 0.0F, + (float)(COUNT - i), 0}; + if ((kind == 0 && i >= 8) || (kind == 2 && i < 8)) { + uint32_t detail = kind == 0 ? i - 8 : i; + for (uint32_t byte = 0; byte < 32; ++byte) { + descriptors[i * 32 + byte] = (unsigned char)(detail * 37U + byte * 19U + 1U); + } + } else if (kind == 2 && i >= 8) { + memset(descriptors + i * 32, (int)(i + 31U), 32); + } + } + Lardon3DExtractedFeatures features = {64, 64, COUNT, keypoints, descriptors}; + Lardon3DFeatureExtractorParameters parameters = {63, 4, 20}; + Lardon3DFeatureStoreResult result = + lardon3d_feature_store_publish(state, image_id, 0, ¶meters, &features, set); + return result == LARDON3D_FEATURE_STORE_OK || + result == LARDON3D_FEATURE_STORE_PUBLISHED_NOT_DURABLE; +} + +static void update_race_barrier(void *userdata) { + UpdateRace *race = userdata; + (void)pthread_barrier_wait(&race->barrier); +} + +static void *update_race_thread(void *userdata) { + UpdateRaceThread *thread = userdata; + uint64_t last = 0; + size_t indexed = 0; + thread->race->result[thread->slot] = lardon3d_visual_index_test_update_once( + thread->race->root, thread->race->database, thread->race->index_id, 0, 0, 16, &last, + &indexed, update_race_barrier, thread->race); + return NULL; +} + +static bool sha256_bytes(const unsigned char *bytes, size_t size, unsigned char hash[32]) { + EVP_MD_CTX *context = EVP_MD_CTX_new(); + unsigned int hash_size = 0; + bool ok = context && EVP_DigestInit_ex(context, EVP_sha256(), NULL) == 1 && + EVP_DigestUpdate(context, bytes, size) == 1 && + EVP_DigestFinal_ex(context, hash, &hash_size) == 1 && hash_size == 32; + EVP_MD_CTX_free(context); + return ok; +} + +static bool replace_segment_metadata(const char *database_path, uint64_t segment_id, + const unsigned char hash[32], const char *path, + uint64_t size) { + sqlite3 *connection = NULL; + sqlite3_stmt *statement = NULL; + bool ok = sqlite3_open(database_path, &connection) == SQLITE_OK && + sqlite3_prepare_v2(connection, + "UPDATE visual_index_segments SET sha256=?1,path=?2,size_bytes=?3 " + "WHERE visual_index_segment_id=?4", -1, &statement, NULL) == + SQLITE_OK; + if (ok) { + sqlite3_bind_blob(statement, 1, hash, 32, SQLITE_TRANSIENT); + sqlite3_bind_text(statement, 2, path, -1, SQLITE_TRANSIENT); + sqlite3_bind_int64(statement, 3, (sqlite3_int64)size); + sqlite3_bind_int64(statement, 4, (sqlite3_int64)segment_id); + ok = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(connection) == 1; + } + sqlite3_finalize(statement); + return sqlite3_close(connection) == SQLITE_OK && ok; +} + +static bool publish_mutated_segment(const char *root, const char *database_path, + const Lardon3DProjectDbVisualIndexSegment *segment, + const unsigned char *bytes, size_t size, char path[PATH_MAX], + unsigned char hash[32]) { + char hex[65]; + if (!sha256_bytes(bytes, size, hash)) { + return false; + } + for (size_t i = 0; i < 32; ++i) { + snprintf(hex + i * 2, 3, "%02x", hash[i]); + } + char directory[PATH_MAX]; + if (snprintf(path, PATH_MAX, "assets/visual-index/%.2s/%s", hex, hex) <= 0 || + snprintf(directory, sizeof(directory), "%s/assets/visual-index/%.2s", root, hex) <= 0) { + return false; + } + if (mkdir(directory, 0700) != 0 && errno != EEXIST) { + return false; + } + char absolute[PATH_MAX]; + if (!join_path(absolute, root, path)) { + return false; + } + int descriptor = open(absolute, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0600); + if (descriptor < 0) { + return false; + } + bool ok = write(descriptor, bytes, size) == (ssize_t)size && close(descriptor) == 0; + return ok && replace_segment_metadata(database_path, segment->visual_index_segment_id, hash, + path, size); +} + +static void *query_thread(void *userdata) { + QueryThread *context = userdata; + Lardon3DVisualIndexQueryOptions options = { + .top_k = 4, + .minimum_evidence_count = 1, + .scanset_filter = LARDON3D_VISUAL_INDEX_ANY_SCANSET, + }; + Lardon3DVisualIndexCandidate candidates[LARDON3D_VISUAL_INDEX_TOP_K_MAX]; + size_t count = 0; + context->ok = lardon3d_visual_index_query(context->root, context->database, context->index_id, + context->query_id, &options, candidates, 4, &count) == + LARDON3D_VISUAL_INDEX_OK && + count > 0; + return NULL; +} + +static bool insert_scale_fixture(const char *database_path, uint64_t image_asset_id, + uint64_t feature_asset_id, + const Lardon3DProjectDbFeatureSet *prototype, uint64_t first, + uint64_t count) { + sqlite3 *connection = NULL; + if (sqlite3_open(database_path, &connection) != SQLITE_OK || + sqlite3_exec(connection, "PRAGMA foreign_keys=ON;BEGIN IMMEDIATE", NULL, NULL, NULL) != + SQLITE_OK) { + sqlite3_close(connection); + return false; + } + sqlite3_stmt *scan = NULL; + sqlite3_stmt *image = NULL; + sqlite3_stmt *feature = NULL; + bool ok = sqlite3_prepare_v2(connection, + "INSERT INTO scansets(scanset_id,name,created_at,updated_at) " + "VALUES(?1,'scale',0,0)", -1, &scan, NULL) == SQLITE_OK && + sqlite3_prepare_v2(connection, + "INSERT INTO images(image_id,scanset_id,asset_id,original_name," + "source_path,imported_at) VALUES(?1,?2,?3,'scale','scale',0)", -1, + &image, NULL) == SQLITE_OK && + sqlite3_prepare_v2(connection, + "INSERT INTO feature_sets(feature_set_id,image_id,feature_asset_id," + "extractor_kind,extractor_version,parameter_fingerprint," + "source_image_sha256,feature_count,descriptor_type," + "descriptor_dimension,created_at) VALUES(?1,?2,?3,?4,?5,?6,?7," + "?8,?9,?10,0)", -1, &feature, NULL) == SQLITE_OK; + for (uint64_t i = 0; i < count && ok; ++i) { + sqlite3_int64 id = (sqlite3_int64)(first + i); + sqlite3_bind_int64(scan, 1, id); + ok = sqlite3_step(scan) == SQLITE_DONE; + sqlite3_reset(scan); + sqlite3_clear_bindings(scan); + sqlite3_bind_int64(image, 1, id); + sqlite3_bind_int64(image, 2, id); + sqlite3_bind_int64(image, 3, (sqlite3_int64)image_asset_id); + ok = ok && sqlite3_step(image) == SQLITE_DONE; + sqlite3_reset(image); + sqlite3_clear_bindings(image); + sqlite3_bind_int64(feature, 1, id); + sqlite3_bind_int64(feature, 2, id); + sqlite3_bind_int64(feature, 3, (sqlite3_int64)feature_asset_id); + sqlite3_bind_text(feature, 4, prototype->extractor_kind, -1, SQLITE_TRANSIENT); + sqlite3_bind_int64(feature, 5, prototype->extractor_version); + sqlite3_bind_blob(feature, 6, prototype->parameter_fingerprint, 32, SQLITE_TRANSIENT); + sqlite3_bind_blob(feature, 7, prototype->source_image_sha256, 32, SQLITE_TRANSIENT); + sqlite3_bind_int64(feature, 8, prototype->feature_count); + sqlite3_bind_int64(feature, 9, prototype->descriptor_type); + sqlite3_bind_int64(feature, 10, prototype->descriptor_dimension); + ok = ok && sqlite3_step(feature) == SQLITE_DONE; + sqlite3_reset(feature); + sqlite3_clear_bindings(feature); + } + sqlite3_finalize(scan); + sqlite3_finalize(image); + sqlite3_finalize(feature); + ok = ok && sqlite3_exec(connection, "COMMIT", NULL, NULL, NULL) == SQLITE_OK; + if (!ok) { + sqlite3_exec(connection, "ROLLBACK", NULL, NULL, NULL); + } + return sqlite3_close(connection) == SQLITE_OK && ok; +} + +static bool run_test(void) { + char root[] = "/tmp/lardon3d-visual-index-XXXXXX"; + CHECK(mkdtemp(root)); + char internal[PATH_MAX]; + char database_path[PATH_MAX]; + char source_a[PATH_MAX]; + char source_b[PATH_MAX]; + char source_c[PATH_MAX]; + char scene_a[PATH_MAX]; + char scene_b[PATH_MAX]; + char scene_c[PATH_MAX]; + char product_paths[6][PATH_MAX]; + CHECK(join_path(internal, root, ".lardon3d") && mkdir(internal, 0700) == 0 && + join_path(database_path, root, "project.db") && join_path(source_a, root, "a.bin") && + join_path(source_b, root, "b.bin") && join_path(source_c, root, "c.bin") && + join_path(scene_a, root, "scene-a.pgm") && join_path(scene_b, root, "scene-b.pgm") && + join_path(scene_c, root, "scene-c.pgm") && + join_path(product_paths[0], root, "product-a1.pgm") && + join_path(product_paths[1], root, "product-a2.pgm") && + join_path(product_paths[2], root, "product-a3.pgm") && + join_path(product_paths[3], root, "product-b1-crop.pgm") && + join_path(product_paths[4], root, "product-b2-rotation.pgm") && + join_path(product_paths[5], root, "product-b3.pgm") && write_source(source_a, 1) && + write_source(source_b, 2) && write_source(source_c, 3) && write_scene(scene_a, 0, 11) && + write_scene(scene_b, 5, 11) && write_scene(scene_c, 0, 97) && + write_scene(product_paths[0], 0, 21) && write_scene(product_paths[1], 0, 43) && + write_scene(product_paths[2], 0, 89) && + write_transformed_scene(product_paths[3], 21, true, 0.0) && + write_transformed_scene(product_paths[4], 43, false, 8.0) && + write_scene(product_paths[5], 0, 127)); + Lardon3DProjectDb *database = NULL; + char error[256]; + CHECK(lardon3d_project_db_open(database_path, &database, error) == LARDON3D_PROJECT_DB_OK); + Lardon3DAppState state; + lardon3d_app_state_init(&state); + state.project_loaded = true; + state.project_db = database; + CHECK(snprintf(state.project_path, sizeof(state.project_path), "%s", root) > 0); + Lardon3DProjectDbScanSet scan_a; + Lardon3DProjectDbScanSet scan_b; + CHECK(lardon3d_image_catalog_create_scanset(&state, "A", &scan_a) && + lardon3d_image_catalog_create_scanset(&state, "B", &scan_b)); + Lardon3DProjectDbImage image_a; + Lardon3DProjectDbImage image_b; + Lardon3DProjectDbImage image_c; + Lardon3DProjectDbImage image_a_copy; + Lardon3DProjectDbImage empty_image; + Lardon3DProjectDbImageAsset asset; + CHECK(lardon3d_image_catalog_import_file(&state, scan_a.scanset_id, source_a, 0, &image_a, + &asset) == LARDON3D_IMAGE_CATALOG_IMPORTED && + lardon3d_image_catalog_import_file(&state, scan_b.scanset_id, source_b, 0, &image_b, + &asset) == LARDON3D_IMAGE_CATALOG_IMPORTED && + lardon3d_image_catalog_import_file(&state, scan_b.scanset_id, source_c, 0, &image_c, + &asset) == LARDON3D_IMAGE_CATALOG_IMPORTED && + lardon3d_image_catalog_import_file(&state, scan_b.scanset_id, source_a, 0, &image_a_copy, + &asset) == LARDON3D_IMAGE_CATALOG_IMPORTED && + image_a_copy.asset_id == image_a.asset_id && + lardon3d_image_catalog_import_file(&state, scan_a.scanset_id, source_c, 0, &empty_image, + &asset) == LARDON3D_IMAGE_CATALOG_IMPORTED); + Lardon3DProjectDbFeatureSet feature_a; + Lardon3DProjectDbFeatureSet feature_b; + Lardon3DProjectDbFeatureSet feature_c; + Lardon3DProjectDbFeatureSet feature_a_copy; + Lardon3DProjectDbFeatureSet empty_feature; + CHECK(publish_features(&state, image_a.image_id, 0, &feature_a) && + publish_features(&state, image_b.image_id, 1, &feature_b) && + publish_features(&state, image_c.image_id, 2, &feature_c) && + publish_features(&state, image_a_copy.image_id, 0, &feature_a_copy) && + publish_empty_features(&state, empty_image.image_id, &empty_feature)); + Lardon3DProjectDbFeatureSet burst_query; + Lardon3DProjectDbFeatureSet burst_pattern; + Lardon3DProjectDbFeatureSet burst_related; + CHECK(publish_burst_features(&state, image_a.image_id, 0, &burst_query) && + publish_burst_features(&state, image_b.image_id, 1, &burst_pattern) && + publish_burst_features(&state, image_c.image_id, 2, &burst_related)); + Lardon3DVisualIndexConfiguration burst_configuration = {1, 16, 8}; + uint64_t burst_index = 0; + uint64_t last = 0; + size_t indexed = 0; + CHECK(lardon3d_visual_index_create(database, &burst_query, &burst_configuration, &burst_index) == + LARDON3D_VISUAL_INDEX_OK && + lardon3d_visual_index_update_once(root, database, burst_index, 0, 0, 16, &last, + &indexed) == LARDON3D_VISUAL_INDEX_OK && + indexed == 3); + Lardon3DVisualIndexCandidate candidates[LARDON3D_VISUAL_INDEX_TOP_K_MAX]; + size_t count = 0; + Lardon3DVisualIndexQueryOptions options = { + .top_k = 4, + .minimum_evidence_count = 1, + .scanset_filter = LARDON3D_VISUAL_INDEX_ANY_SCANSET, + }; + CHECK(lardon3d_visual_index_query(root, database, burst_index, burst_query.feature_set_id, + &options, candidates, 4, &count) == + LARDON3D_VISUAL_INDEX_OK && + candidate_rank(candidates, count, burst_related.feature_set_id) != SIZE_MAX && + candidate_rank(candidates, count, burst_pattern.feature_set_id) == SIZE_MAX); + Lardon3DVisualIndexConfiguration configuration = {1, 16, 4096}; + uint64_t index_id = 0; + CHECK(lardon3d_visual_index_create(database, &feature_a, &configuration, &index_id) == + LARDON3D_VISUAL_INDEX_OK && + index_id > 0); + CHECK(lardon3d_visual_index_update_once(root, database, index_id, 0, 0, 16, &last, &indexed) == + LARDON3D_VISUAL_INDEX_OK && + indexed == 5 && last == empty_feature.feature_set_id); + CHECK(lardon3d_visual_index_update_once(root, database, index_id, 0, last, 16, &last, &indexed) == + LARDON3D_VISUAL_INDEX_NO_CHANGE && + indexed == 0); + Lardon3DProjectDbFeatureSet no_pending[16]; + size_t no_pending_count = 0; + CHECK(lardon3d_project_db_list_visual_index_pending(database, index_id, 0, no_pending, 16, + &no_pending_count) == + LARDON3D_PROJECT_DB_OK && + no_pending_count == 0); + options = (Lardon3DVisualIndexQueryOptions){ + .top_k = 4, + .minimum_evidence_count = 2, + .scanset_filter = LARDON3D_VISUAL_INDEX_OTHER_SCANSETS, + }; + CHECK(lardon3d_visual_index_query(root, database, index_id, feature_a.feature_set_id, &options, + candidates, 4, &count) == LARDON3D_VISUAL_INDEX_OK && + count >= 1); + bool found_related = false; + for (size_t i = 0; i < count; ++i) { + found_related = found_related || + (candidates[i].feature_set_id == feature_b.feature_set_id && + candidates[i].image_id == image_b.image_id && + candidates[i].evidence_count >= 10 && candidates[i].scanset_id == + scan_b.scanset_id); + } + CHECK(found_related); + bool found_same_asset = false; + for (size_t i = 0; i < count; ++i) { + if (candidates[i].feature_set_id == feature_a_copy.feature_set_id) { + found_same_asset = candidates[i].same_image_asset && candidates[i].evidence_count == 16 && + candidates[i].score == 1.0; + } + } + CHECK(found_same_asset); + options.exclude_same_asset = true; + CHECK(lardon3d_visual_index_query(root, database, index_id, feature_a.feature_set_id, &options, + candidates, 4, &count) == LARDON3D_VISUAL_INDEX_OK); + for (size_t i = 0; i < count; ++i) { + CHECK(candidates[i].feature_set_id != feature_a_copy.feature_set_id); + } + options.exclude_same_asset = false; + options.scanset_filter = LARDON3D_VISUAL_INDEX_ANY_SCANSET; + CHECK(lardon3d_visual_index_query(root, database, index_id, empty_feature.feature_set_id, + &options, candidates, 4, &count) == + LARDON3D_VISUAL_INDEX_OK && + count == 0); + options.scanset_filter = LARDON3D_VISUAL_INDEX_SAME_SCANSET; + CHECK(lardon3d_visual_index_query(root, database, index_id, feature_a.feature_set_id, &options, + candidates, 4, &count) == LARDON3D_VISUAL_INDEX_OK && + count == 0); + + Lardon3DProjectDbImage real_a; + Lardon3DProjectDbImage real_b; + Lardon3DProjectDbImage real_c; + CHECK(lardon3d_image_catalog_import_file(&state, scan_a.scanset_id, scene_a, 0, &real_a, + &asset) == LARDON3D_IMAGE_CATALOG_IMPORTED && + lardon3d_image_catalog_import_file(&state, scan_b.scanset_id, scene_b, 0, &real_b, + &asset) == LARDON3D_IMAGE_CATALOG_IMPORTED && + lardon3d_image_catalog_import_file(&state, scan_b.scanset_id, scene_c, 0, &real_c, + &asset) == LARDON3D_IMAGE_CATALOG_IMPORTED); + Lardon3DProjectDbFeatureSet real_feature_a; + Lardon3DProjectDbFeatureSet real_feature_b; + Lardon3DProjectDbFeatureSet real_feature_c; + CHECK(publish_real_features(&state, real_a.image_id, scene_a, &real_feature_a) && + publish_real_features(&state, real_b.image_id, scene_b, &real_feature_b) && + publish_real_features(&state, real_c.image_id, scene_c, &real_feature_c)); + uint64_t real_index = 0; + Lardon3DVisualIndexConfiguration real_configuration = {1, 512, 256}; + CHECK(lardon3d_visual_index_create(database, &real_feature_a, &real_configuration, &real_index) == + LARDON3D_VISUAL_INDEX_OK && + setenv("LARDON3D_TEST_VISUAL_INDEX_FAIL_DIR_FSYNC", "1", 1) == 0 && + lardon3d_visual_index_update_once(root, database, real_index, 0, 0, 16, &last, &indexed) == + LARDON3D_VISUAL_INDEX_PUBLISHED_NOT_DURABLE && + indexed == 3 && unsetenv("LARDON3D_TEST_VISUAL_INDEX_FAIL_DIR_FSYNC") == 0); + options = (Lardon3DVisualIndexQueryOptions){8, 2, LARDON3D_VISUAL_INDEX_OTHER_SCANSETS, false}; + CHECK(lardon3d_visual_index_query(root, database, real_index, real_feature_a.feature_set_id, + &options, candidates, 8, &count) == + LARDON3D_VISUAL_INDEX_OK && + count > 0 && candidates[0].feature_set_id == real_feature_b.feature_set_id); + Lardon3DProjectDbImage product_images[6]; + Lardon3DProjectDbFeatureSet product_features[6]; + for (size_t i = 0; i < 6; ++i) { + uint64_t scanset = i < 3 ? scan_a.scanset_id : scan_b.scanset_id; + CHECK(lardon3d_image_catalog_import_file(&state, scanset, product_paths[i], 0, + &product_images[i], &asset) == + LARDON3D_IMAGE_CATALOG_IMPORTED && + publish_product_features(&state, product_images[i].image_id, product_paths[i], + &product_features[i])); + } + Lardon3DVisualIndexConfiguration product_configuration = {1, 640, 256}; + uint64_t product_index = 0; + CHECK(lardon3d_visual_index_create(database, &product_features[0], &product_configuration, + &product_index) == LARDON3D_VISUAL_INDEX_OK && + lardon3d_visual_index_update_once(root, database, product_index, 0, 0, 16, &last, + &indexed) == LARDON3D_VISUAL_INDEX_OK && + indexed == 6); + options = (Lardon3DVisualIndexQueryOptions){6, 1, LARDON3D_VISUAL_INDEX_ANY_SCANSET, false}; + CHECK(lardon3d_visual_index_query(root, database, product_index, + product_features[0].feature_set_id, &options, candidates, 6, + &count) == LARDON3D_VISUAL_INDEX_OK); + size_t crop_rank = candidate_rank(candidates, count, product_features[3].feature_set_id); + size_t unrelated_rank = candidate_rank(candidates, count, product_features[5].feature_set_id); + CHECK(crop_rank != SIZE_MAX && (unrelated_rank == SIZE_MAX || crop_rank < unrelated_rank)); + size_t low_evidence_count = count; + options.minimum_evidence_count = 8; + CHECK(lardon3d_visual_index_query(root, database, product_index, + product_features[0].feature_set_id, &options, candidates, 6, + &count) == LARDON3D_VISUAL_INDEX_OK && + count <= low_evidence_count); + options = (Lardon3DVisualIndexQueryOptions){6, 1, + LARDON3D_VISUAL_INDEX_OTHER_SCANSETS, false}; + CHECK(lardon3d_visual_index_query(root, database, product_index, + product_features[0].feature_set_id, &options, candidates, 6, + &count) == LARDON3D_VISUAL_INDEX_OK); + for (size_t i = 0; i < count; ++i) { + CHECK(candidates[i].scanset_id == scan_b.scanset_id); + } + options.scanset_filter = LARDON3D_VISUAL_INDEX_SAME_SCANSET; + CHECK(lardon3d_visual_index_query(root, database, product_index, + product_features[0].feature_set_id, &options, candidates, 6, + &count) == LARDON3D_VISUAL_INDEX_OK); + for (size_t i = 0; i < count; ++i) { + CHECK(candidates[i].scanset_id == scan_a.scanset_id); + } + options.scanset_filter = LARDON3D_VISUAL_INDEX_ANY_SCANSET; + CHECK(lardon3d_visual_index_query(root, database, product_index, + product_features[1].feature_set_id, &options, candidates, 6, + &count) == LARDON3D_VISUAL_INDEX_OK); + size_t rotation_rank = candidate_rank(candidates, count, product_features[4].feature_set_id); + unrelated_rank = candidate_rank(candidates, count, product_features[5].feature_set_id); + CHECK(rotation_rank != SIZE_MAX && + (unrelated_rank == SIZE_MAX || rotation_rank < unrelated_rank)); + Lardon3DVisualIndexConfiguration retry_configuration = {1, 15, 64}; + uint64_t retry_index = 0; + CHECK(lardon3d_visual_index_create(database, &feature_a, &retry_configuration, &retry_index) == + LARDON3D_VISUAL_INDEX_OK && + setenv("LARDON3D_TEST_PROJECT_DB_FAIL_VISUAL_SEGMENT", "1", 1) == 0 && + lardon3d_visual_index_update_once(root, database, retry_index, 0, 0, 16, &last, &indexed) == + LARDON3D_VISUAL_INDEX_DB_BUSY && + unsetenv("LARDON3D_TEST_PROJECT_DB_FAIL_VISUAL_SEGMENT") == 0 && + lardon3d_visual_index_update_once(root, database, retry_index, 0, 0, 16, &last, &indexed) == + LARDON3D_VISUAL_INDEX_OK && + indexed == 5); + Lardon3DVisualIndexConfiguration race_configuration = {1, 14, 64}; + uint64_t race_index = 0; + CHECK(lardon3d_visual_index_create(database, &feature_a, &race_configuration, &race_index) == + LARDON3D_VISUAL_INDEX_OK); + UpdateRace race = {.root = root, .database = database, .index_id = race_index}; + UpdateRaceThread race_threads[2] = {{&race, 0}, {&race, 1}}; + pthread_t update_threads[2]; + CHECK(pthread_barrier_init(&race.barrier, NULL, 2) == 0 && + pthread_create(&update_threads[0], NULL, update_race_thread, &race_threads[0]) == 0 && + pthread_create(&update_threads[1], NULL, update_race_thread, &race_threads[1]) == 0 && + pthread_join(update_threads[0], NULL) == 0 && pthread_join(update_threads[1], NULL) == 0 && + pthread_barrier_destroy(&race.barrier) == 0); + CHECK((race.result[0] == LARDON3D_VISUAL_INDEX_OK) != + (race.result[1] == LARDON3D_VISUAL_INDEX_OK)); + CHECK(race.result[0] == LARDON3D_VISUAL_INDEX_OK || + race.result[0] == LARDON3D_VISUAL_INDEX_DB_ERROR || + race.result[0] == LARDON3D_VISUAL_INDEX_DB_BUSY); + CHECK(race.result[1] == LARDON3D_VISUAL_INDEX_OK || + race.result[1] == LARDON3D_VISUAL_INDEX_DB_ERROR || + race.result[1] == LARDON3D_VISUAL_INDEX_DB_BUSY); + CHECK(lardon3d_visual_index_update_once(root, database, race_index, 0, 0, 16, &last, + &indexed) == LARDON3D_VISUAL_INDEX_NO_CHANGE); + options.top_k = LARDON3D_VISUAL_INDEX_TOP_K_MAX + 1; + CHECK(lardon3d_visual_index_query(root, database, index_id, feature_a.feature_set_id, &options, + candidates, 4, &count) == + LARDON3D_VISUAL_INDEX_INVALID_ARGUMENT); + QueryThread contexts[4]; + pthread_t threads[4]; + for (size_t i = 0; i < 4; ++i) { + contexts[i] = (QueryThread){root, database, index_id, feature_a.feature_set_id, false}; + CHECK(pthread_create(&threads[i], NULL, query_thread, &contexts[i]) == 0); + } + for (size_t i = 0; i < 4; ++i) { + CHECK(pthread_join(threads[i], NULL) == 0 && contexts[i].ok); + } + Lardon3DProjectDbVisualIndexSegment segments[2]; + size_t segment_count = 0; + CHECK(lardon3d_project_db_list_visual_index_segments(database, index_id, 0, segments, 2, + &segment_count) == LARDON3D_PROJECT_DB_OK && + segment_count == 1); + char segment_path[PATH_MAX]; + CHECK(join_path(segment_path, root, segments[0].path)); + unsigned char *segment_bytes = malloc((size_t)segments[0].size_bytes); + CHECK(segment_bytes); + int descriptor = open(segment_path, O_RDWR | O_CLOEXEC); + unsigned char original; + CHECK(descriptor >= 0 && + pread(descriptor, segment_bytes, (size_t)segments[0].size_bytes, 0) == + (ssize_t)segments[0].size_bytes && + pread(descriptor, &original, 1, 12) == 1); + unsigned char original_hash[32]; + char original_path[PATH_MAX]; + memcpy(original_hash, segments[0].sha256, sizeof(original_hash)); + CHECK(snprintf(original_path, sizeof(original_path), "%s", segments[0].path) > 0); + options = (Lardon3DVisualIndexQueryOptions){4, 1, LARDON3D_VISUAL_INDEX_ANY_SCANSET, false}; + unsigned char *coherent = malloc((size_t)segments[0].size_bytes); + char coherent_path[PATH_MAX]; + unsigned char coherent_hash[32]; + CHECK(coherent); + memcpy(coherent, segment_bytes, (size_t)segments[0].size_bytes); + coherent[8] = 2; + coherent[9] = coherent[10] = coherent[11] = 0; + CHECK(publish_mutated_segment(root, database_path, &segments[0], coherent, + (size_t)segments[0].size_bytes, coherent_path, coherent_hash) && + lardon3d_visual_index_query(root, database, index_id, feature_a.feature_set_id, &options, + candidates, 4, &count) == + LARDON3D_VISUAL_INDEX_UNSUPPORTED_VERSION && + replace_segment_metadata(database_path, segments[0].visual_index_segment_id, + original_hash, original_path, segments[0].size_bytes)); + memcpy(coherent, segment_bytes, (size_t)segments[0].size_bytes); + memset(coherent + 24, 0xff, 8); + CHECK(publish_mutated_segment(root, database_path, &segments[0], coherent, + (size_t)segments[0].size_bytes, coherent_path, coherent_hash) && + lardon3d_visual_index_query(root, database, index_id, feature_a.feature_set_id, &options, + candidates, 4, &count) == LARDON3D_VISUAL_INDEX_CORRUPT && + replace_segment_metadata(database_path, segments[0].visual_index_segment_id, + original_hash, original_path, segments[0].size_bytes)); + memcpy(coherent, segment_bytes, (size_t)segments[0].size_bytes); + uint64_t overflow_count = (UINT64_MAX - 128U) / 24U + 1U; + for (size_t i = 0; i < 8; ++i) { + coherent[24 + i] = (unsigned char)(overflow_count >> (i * 8)); + } + CHECK(publish_mutated_segment(root, database_path, &segments[0], coherent, + (size_t)segments[0].size_bytes, coherent_path, coherent_hash) && + lardon3d_visual_index_query(root, database, index_id, feature_a.feature_set_id, &options, + candidates, 4, &count) == LARDON3D_VISUAL_INDEX_CORRUPT && + replace_segment_metadata(database_path, segments[0].visual_index_segment_id, + original_hash, original_path, segments[0].size_bytes)); + free(coherent); + unsigned char mutated = (unsigned char)(original ^ 0x40U); + CHECK(pwrite(descriptor, &mutated, 1, 12) == 1 && close(descriptor) == 0); + options = (Lardon3DVisualIndexQueryOptions){4, 1, LARDON3D_VISUAL_INDEX_ANY_SCANSET, false}; + CHECK(lardon3d_visual_index_query(root, database, index_id, feature_a.feature_set_id, &options, + candidates, 4, &count) == LARDON3D_VISUAL_INDEX_CORRUPT); + descriptor = open(segment_path, O_RDWR | O_CLOEXEC); + CHECK(descriptor >= 0 && pwrite(descriptor, &original, 1, 12) == 1 && close(descriptor) == 0); + char hidden_path[PATH_MAX]; + CHECK(snprintf(hidden_path, sizeof(hidden_path), "%s.hidden", segment_path) > 0 && + rename(segment_path, hidden_path) == 0); + CHECK(lardon3d_visual_index_query(root, database, index_id, feature_a.feature_set_id, &options, + candidates, 4, &count) == LARDON3D_VISUAL_INDEX_NOT_FOUND && + rename(hidden_path, segment_path) == 0); + descriptor = open(segment_path, O_RDWR | O_CLOEXEC); + CHECK(descriptor >= 0 && ftruncate(descriptor, (off_t)segments[0].size_bytes - 1) == 0 && + close(descriptor) == 0); + CHECK(lardon3d_visual_index_query(root, database, index_id, feature_a.feature_set_id, &options, + candidates, 4, &count) == LARDON3D_VISUAL_INDEX_CORRUPT); + descriptor = open(segment_path, O_WRONLY | O_TRUNC | O_CLOEXEC); + CHECK(descriptor >= 0 && + write(descriptor, segment_bytes, (size_t)segments[0].size_bytes) == + (ssize_t)segments[0].size_bytes && + close(descriptor) == 0); + free(segment_bytes); + lardon3d_project_db_close(database); + database = NULL; + state.project_db = NULL; + CHECK(insert_scale_fixture(database_path, image_a.asset_id, feature_a.feature_asset_id, + &feature_a, 10000, 50000)); + CHECK(lardon3d_project_db_open(database_path, &database, error) == LARDON3D_PROJECT_DB_OK); + state.project_db = database; + Lardon3DVisualIndexConfiguration capacity_configuration = {1, 15, 4096}; + uint64_t capacity_index = 0; + CHECK(lardon3d_visual_index_create(database, &feature_a, &capacity_configuration, + &capacity_index) == LARDON3D_VISUAL_INDEX_OK); + size_t total_indexed = 0; + last = 0; + for (size_t batch = 0; batch < LARDON3D_VISUAL_INDEX_SEGMENT_MAX; ++batch) { + Lardon3DVisualIndexResult updated = + lardon3d_visual_index_update_once(root, database, capacity_index, 0, last, 16, &last, + &indexed); + CHECK(updated == LARDON3D_VISUAL_INDEX_OK); + total_indexed += indexed; + } + CHECK(total_indexed == 4096 && + lardon3d_visual_index_update_once(root, database, capacity_index, 0, last, 16, &last, + &indexed) == LARDON3D_VISUAL_INDEX_LIMIT); + Lardon3DProjectDbFeatureSet pending[16]; + size_t pending_count = 0; + CHECK(lardon3d_project_db_list_visual_index_pending(database, capacity_index, last, pending, 16, + &pending_count) == + LARDON3D_PROJECT_DB_OK && + pending_count == 16); + options = (Lardon3DVisualIndexQueryOptions){10, 1, LARDON3D_VISUAL_INDEX_ANY_SCANSET, false}; + CHECK(lardon3d_visual_index_query(root, database, capacity_index, feature_a.feature_set_id, + &options, + candidates, 10, &count) == LARDON3D_VISUAL_INDEX_OK && + count == 10); + lardon3d_project_db_close(database); + CHECK(remove_tree(root)); + return true; +} + +int main(void) { return run_test() ? 0 : 1; }