From 663176f8841461f5a6badee734a3f1eef4e62583 Mon Sep 17 00:00:00 2001 From: fy59 Date: Sat, 29 Aug 2026 16:46:17 +0200 Subject: [PATCH] feat: freeze pre-sfm execution and compute resources --- README.md | 15 +- docs/architecture/candidate_pair.md | 43 +- docs/architecture/feature_store.md | 20 +- docs/architecture/geometric_verification.md | 25 +- docs/architecture/geometric_verifier.md | 78 +- docs/architecture/internal_parallelism.md | 288 ++++ docs/architecture/match_result.md | 2 +- docs/architecture/matcher.md | 31 +- .../precision_feature_pipeline.md | 22 +- docs/architecture/project_database.md | 24 +- docs/architecture/resource_governor.md | 70 +- docs/architecture/task_kind_registry.md | 26 +- docs/architecture/task_system.md | 37 +- docs/architecture/track_builder.md | 6 + docs/architecture/visual_index.md | 40 +- docs/roadmap/roadmap.md | 87 +- include/lardon3d/feature_extractor.h | 7 +- include/lardon3d/geometric_verifier.h | 29 +- include/lardon3d/matcher_task.h | 26 + include/lardon3d/project_db.h | 24 + include/lardon3d/task_checkpoint.h | 23 + meson.build | 41 +- src/acquisition_campaign_task.cpp | 10 +- src/app.c | 16 +- src/candidate_pair_gen.c | 81 +- src/candidate_pair_gen_internal.h | 43 + src/candidate_pair_task.c | 329 +++- src/feature_extractor_opencv.cpp | 10 +- src/feature_task.c | 8 +- src/geometric_verifier.cpp | 109 +- src/geometric_verifier_task.c | 44 +- src/incremental_reconstruction_task.cpp | 7 +- src/matcher.cpp | 125 +- src/matcher_internal.h | 45 + src/matcher_task.c | 394 ++++- src/photo_quality.cpp | 200 ++- src/photo_quality_task.cpp | 7 +- src/project.c | 175 ++- src/project_db.c | 109 ++ src/raw_development_task.cpp | 7 +- src/sift_task.c | 13 +- src/sparse_sfm_task.cpp | 7 +- src/task_checkpoint.c | 73 + src/task_kind_registry.c | 104 +- src/track_builder_task.cpp | 7 +- src/visual_index.c | 116 +- src/visual_index_internal.h | 18 + src/visual_index_task.c | 13 +- tests/pre_sfm_real_execution.cpp | 1318 +++++++++++++++++ tests/test_candidate_pair_gen.c | 68 +- tests/test_candidate_pair_task.c | 483 +++++- tests/test_feature_store.c | 143 +- tests/test_feature_task.c | 71 + tests/test_geometric_verifier_core.cpp | 210 ++- tests/test_geometric_verifier_task.c | 190 ++- tests/test_matcher_task.c | 386 ++++- tests/test_photo_quality.cpp | 66 + tests/test_project.c | 119 +- tests/test_task_checkpoint.c | 16 + tests/test_task_kind_registry.c | 18 + tests/test_track_builder_project.cpp | 21 + tests/test_visual_index.c | 97 ++ 62 files changed, 5770 insertions(+), 470 deletions(-) create mode 100644 docs/architecture/internal_parallelism.md create mode 100644 src/candidate_pair_gen_internal.h create mode 100644 src/matcher_internal.h create mode 100644 src/visual_index_internal.h create mode 100644 tests/pre_sfm_real_execution.cpp diff --git a/README.md b/README.md index aa0ac7f..40ca957 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,8 @@ persistante, enrichissable et versionnable. - **Feature Store v1/v2** : ORB U8×32, SIFT/RootSIFT F32×128 et lecture typée bornée - **Image View** : vues triées et filtrées pour la TUI - **Task** : moteur de tâches avec pause/reprise, annulation et séquences -- **Task Checkpoint v1** : snapshot durable, fichier atomique et reprise sûre +- **Task Checkpoint v1** : snapshot durable, protocole `.chk.next` → SQLite → + `.chk` sous verrou par tâche, et reprise sûre - **Project Database v22** : fondations v20/v21 préservées, avec snapshot d'exécution sélectionnée, publication bornée des représentations, attache de scope de calibration et tâche durable mince `raw.develop` — PASS / FROZEN @@ -61,8 +62,11 @@ persistante, enrichissable et versionnable. OpenMVS binaire little-endian validé (en-tête <= 1 MiB en octets bruts, LF/CRLF acceptés, CR seul malformé rejeté, ligne <= 64 KiB), fusionné en mode 0 — PASS / FROZEN -- **Geometric Verification Model v1** : identité, masque d'inliers et modèle 3×3 persistants -- **Geometric Verifier v1** : Fundamental USAC/MAGSAC, reprise et lots resource-aware +- **Geometric Verification Model** : identité, masque d'inliers et modèle 3×3 + persistants pour les policies Verifier v1/v2 historiques et v3 courante +- **Geometric Verifier v3** : Fundamental USAC/MAGSAC, reprise et lots resource-aware +- **Internal Parallelism + Compute Resources v1** : parallélisme interne borné, + sorties canoniques et admission Governor — PASS / FROZEN - **Task Kind Registry** : identité métier durable et reconstruction runtime explicite - **Recovery projet** : reprise automatique sélective et bornée des imports récupérables - **Task Queue** : file FIFO avec sélection adaptative et backpressure @@ -144,6 +148,7 @@ Acquisitions - [Registry des types de tâches](docs/architecture/task_kind_registry.md) - [File de tâches](docs/architecture/task_queue.md) - [Resource Governor](docs/architecture/resource_governor.md) +- [Parallélisme interne borné](docs/architecture/internal_parallelism.md) - [Pipeline sensible aux ressources](docs/architecture/resource_aware_pipeline.md) - [Intégration Scheduler ↔ Governor](docs/architecture/scheduler_resource_integration.md) - [Pipeline de reconstruction](docs/architecture/reconstruction_pipeline.md) @@ -199,8 +204,8 @@ Pour les changements sensibles à la mémoire ou à la concurrence, ajouter ASan ## Statut Lardon3D est en développement actif. La persistance des tâches, le catalogue, -le Feature Store multipasse, le Visual Index ORB, Candidate Pair Generator -Matcher v1, Geometric Verification Model v1 et Geometric Verifier Fundamental v1 +le Feature Store multipasse, le Visual Index ORB, Candidate Pair Generator, +Matcher v1, Geometric Verification Model et Geometric Verifier Fundamental v3 sont implémentés. Le runtime Feature + Matcher + Verifier emploie des tâches durables, de petits lots, le Resource Governor interactif et un hot path Vulkan ORB exact avec fallback CPU. La feasibility Vulkan SIFT/RootSIFT a été rejetée ; ces deux matchers diff --git a/docs/architecture/candidate_pair.md b/docs/architecture/candidate_pair.md index 45736be..7179029 100644 --- a/docs/architecture/candidate_pair.md +++ b/docs/architecture/candidate_pair.md @@ -219,9 +219,9 @@ La fonction retourne le dernier FeatureSet traité. ### Unité de travail -Un FeatureSet source et sa requête Visual Index associée. Chaque séquence -traite un lot borné de FeatureSets (1 à 64 selon le contrat Governor), -page par page (PAGE_SIZE = 64). +Un membership FeatureSet source du Visual Index et sa requête associée. Chaque +séquence traite un lot borné de memberships (1 à 64 selon le contrat Governor), +en ordre croissant d'ID sans supposer des IDs contigus. ### Checkpoint @@ -236,10 +236,20 @@ DB, et les paires déjà persistées sont ignorées par `find avant create`. À l'ouverture du projet, la tâche est automatiquement restaurée via la registry production et resoumise à la queue. +Les anciens snapshots v1 produits avec l'estimation opérationnelle exacte +128 Kio fixes, 64 Kio par item, lot 1–64, CPU 1, IO 1 et GPU 0 sont normalisés +éphémèrement par la registry à la forme courante CPU12 avant admission. Le +snapshot durable original reste la source du reconstructeur ; aucun checkpoint +d'estimation seule n'est stagé, promu ou publié sous le même résumé. Une panne +pré-terminale répète donc cette normalisation exacte. Aucun autre snapshot, +curseur ou paramètre scientifique n'est réinterprété. + ### Intégration scheduler La tâche utilise le scheduler générique via le pattern standard : -- Estimation immuable (128 Kio fixes, 256 Kio par item, lot 1–64) +- Estimation opérationnelle (256 Kio fixes, 64 Kio par item, lot 1–64). + La Queue conserve un callback ; jusqu'à douze participants CPU admis peuvent + calculer une fenêtre interne bornée sans modifier l'identité scientifique. - Réservation CPU + IO avant exécution - `lardon3d_task_sequence_break()` entre chaque lot pour réadmission Governor - Callback terminal checkpoint après `COMPLETED`/`FAILED`/`CANCELLED` @@ -263,21 +273,23 @@ bool lardon3d_candidate_pair_generate_reconstruct( ### Garantie actuelle -Le generator s'exécute sur un seul FeatureSet à la fois. -Le mutex DB protège les création concurrentes. +La Queue conserve un callback actif. À l'intérieur de la Task, jusqu'à douze +threads CPU admis calculent en parallèle une fenêtre d'au plus deux sources par +thread. Chaque participant possède un handle DB de lecture privé. Le thread +propriétaire publie ensuite seul et dans l'ordre canonique des sources. Voir le +[contrat de parallélisme interne](internal_parallelism.md). ### Atomicité -Deux workers créant simultanément la même paire : -- Ne corrompent pas la DB -- Ne créent pas deux lignes (UNIQUE constraint) -- Retournent OK pour l'un, NOT_FOUND→create pour l'autre +Les workers internes ne créent aucune paire. Après leur jointure, le +propriétaire applique seul `find avant create`. La contrainte UNIQUE reste une +protection persistante, pas un mécanisme d'ordonnancement parallèle. ### Limites -Le pattern `find + create` n'est pas atomique entre les deux appels. -Avec un seul worker, c'est suffisant. Avec plusieurs workers, -la contrainte UNIQUE assure l'unicité mais peut causer un retry. +Le pattern `find + create` n'est pas atomique entre les deux appels. Le chemin +de Task n'introduit aucun writer concurrent ; la sémantique existante reste +inchangée pour les autres appelants éventuels. ## Bornes et ressources @@ -287,8 +299,9 @@ la contrainte UNIQUE assure l'unicité mais peut causer un retry. ### Mémoire -Allocation par requête : `256 * sizeof(Lardon3DVisualIndexCandidate)` -≈ 256 * 40 = 10 240 octets (10 Kio). +Allocation de requête bornée par le top-K, plus un résultat de propositions +borné par le même maximum. La Task limite sa fenêtre à 24 sources et annonce +256 Kio fixes plus 64 Kio par item au Governor. ### Complexité diff --git a/docs/architecture/feature_store.md b/docs/architecture/feature_store.md index 66f100a..19d34c0 100644 --- a/docs/architecture/feature_store.md +++ b/docs/architecture/feature_store.md @@ -28,10 +28,22 @@ porte sur les 24 octets canoniques `L3DORBP1`, version et trois entiers little-endian. Il ne dépend ni du padding, ni de la locale. Le contrat ORB v1 produit 32 octets binaires par point. -OpenCV n'est pas configuré via un état global par Lardon3D. Le scheduler actuel -n'a qu'un worker ; OpenCV peut néanmoins employer son backend parallèle -interne. Le contrôle fin de ce parallélisme devra précéder les futurs pools de -workers. +Au démarrage, avant la création des workers, Lardon3D configure la limite de +threads interne process-wide d'OpenCV à +`min(12, logical_cpu_count - system_cpu_reserve)`. La borne douze est +opérationnelle et correspond au plafond audité. `features.extract` demande +jusqu'à douze threads au Resource Governor ; celui-ci réduit l'admission au +budget hôte, donc à la limite OpenCV configurée. La Queue conserve un callback actif ; aucun pool Lardon3D +supplémentaire n'est créé pour ORB. + +Cette limite est une configuration opérationnelle, jamais un paramètre +scientifique : elle n'entre ni dans le fingerprint ORB v1, ni dans l'identité +FeatureSet, ni dans le Feature File. L'audit contrôlé OpenCV 5.0.0 aux limites +1, 2, 4, 8 et 12 exige le même count, le même ordre et les mêmes valeurs +binary32 des six champs keypoint persistés, les mêmes lignes/octets descriptor +et le même SHA-256 du Feature File. `class_id` est une donnée interne +`cv::KeyPoint` non publiée ; elle ne fait pas partie du record Feature File v1/v2 +gelé et ne peut donc définir une identité scientifique. La limite de 100 000 000 pixels est vérifiée après `cv::imread` : l'API utilisée ne fournit pas de sonde de dimensions multi-format fiable sans décodage. Le pic diff --git a/docs/architecture/geometric_verification.md b/docs/architecture/geometric_verification.md index 698d76d..20b332e 100644 --- a/docs/architecture/geometric_verification.md +++ b/docs/architecture/geometric_verification.md @@ -2,8 +2,12 @@ ## Scope -Geometric Verification Model v1 est le contrat scientifique persistant placé après le Matcher. -Il stocke un résultat terminé, compact et immutable. Il n'est ni un moteur de calcul ni une tâche. +Geometric Verification Model est le contrat persistant placé après le Matcher. +Sa représentation stocke les identités scientifiques Geometric Verifier v1/v2 +historiques et v3 courantes, sans changement de schéma : `verifier_version` et +`parameter_fingerprint` appartiennent déjà à l'identité exacte. Il stocke un +résultat terminé, compact et immutable. Il n'est ni un moteur de calcul ni une +tâche. Aucun RANSAC, USAC, MAGSAC, calcul d'inliers ou backend géométrique n'appartient à ce ticket. ## Position in reconstruction pipeline @@ -43,7 +47,7 @@ binaires, little-endian. Aucun timestamp, résultat, PID, durée ou identifiant ## Verifier kind -v1 supporte uniquement `FUNDAMENTAL`, valeur persistante stable 1. Aucun comportement fictif +Le modèle supporte uniquement `FUNDAMENTAL`, valeur persistante stable 1. Aucun comportement fictif `ESSENTIAL` ou `HOMOGRAPHY` n'est réservé dans l'API publique. ## Persistent states @@ -82,9 +86,10 @@ jusqu'à 32 fois plus grande au cas dense et aurait un encodage supplémentaire - kind, version et fingerprint ont une sérialisation stable ; - une ligne publiée est complète et immutable. -Exemple : pour 100 matches, FUNDAMENTAL v1/fingerprint X peut publier REJECTED avec 23 inliers, -un masque de 13 octets et aucun modèle. Une autre identité peut publier VERIFIED avec 67 inliers, -le même format de masque et une matrice 3×3 finie. +Exemple : pour 100 matches, une identité FUNDAMENTAL v1, v2 ou v3/fingerprint +X peut publier REJECTED avec 23 inliers, un masque de 13 octets et aucun modèle. +Une autre identité peut publier VERIFIED avec 67 inliers, le même format de +masque et une matrice 3×3 finie. ## Persistence semantics @@ -149,9 +154,9 @@ Après commit, le résultat est complet et réutilisable après réouverture. Av ne laisse aucune ligne partielle. Un loader rejette toute ligne incohérente comme corruption au lieu de réparer ou d'interpréter au mieux. -## Future verifier execution contract +## Verifier execution contract -Le prochain ticket prendra un Match Result et son Match File borné. L'accès nécessaire existe via +L'exécution prend un Match Result et son Match File borné. L'accès nécessaire existe via `lardon3d_feature_reader_keypoints()`, borné à 256 keypoints par appel ; l'intégration devra relier les deux Feature Sets et les indices du Match File sans modifier le Feature Store. Le verifier estimera hors transaction, dérivera état/masque/modèle, publiera en une courte transaction, @@ -165,8 +170,8 @@ fingerprint. Aucun nombre de threads ou hardware ID n'est un paramètre scientif ## Explicitly out of scope -Le calcul géométrique, le choix RANSAC/USAC/MAGSAC, OpenCV geometry, GPU, Vulkan, OpenCL, shader, -task kind, worker, checkpoint et nouvelle orchestration sont explicitement hors périmètre. +GPU, Vulkan, OpenCL, shader et nouvelle orchestration restent hors périmètre de ce contrat de +persistance. ## Versioning diff --git a/docs/architecture/geometric_verifier.md b/docs/architecture/geometric_verifier.md index c2b37c3..b49ed2d 100644 --- a/docs/architecture/geometric_verifier.md +++ b/docs/architecture/geometric_verifier.md @@ -1,4 +1,4 @@ -# Geometric Verifier v1 +# Geometric Verifier v1 / v2 / v3 ## Scope @@ -7,6 +7,14 @@ Fundamental `GEOMETRIC_REJECTED` ou `GEOMETRIC_VERIFIED`. Le contrat persistant [`geometric_verification.md`](geometric_verification.md). Tracks, pose, Essential, compétition Homography, triangulation et SfM sont hors périmètre. +V1 et v2 restent des identités scientifiques historiques et immutables : leurs versions, +fingerprints, lignes et résultats existants ne sont jamais réinterprétés. V2 conserve l'estimator, +les paramètres, l'ordre et l'acceptance v1, mais ajoute avant USAC le support minimal par +observations canoniques distinctes décrit ci-dessous. V3 est la policy de production courante : +après les mêmes validations intégrales, elle compose ce support v2 avec la preuve exacte de +faisabilité d'acceptation `match_count >= min_inlier_count`. Chaque policy possède sa version et son +fingerprint distincts ; Project DB v22 stocke déjà ces champs et ne nécessite aucune migration. + ## Inputs Le parent DB fournit les deux Feature Set IDs, le compte, le chemin, la taille et le SHA-256 du @@ -31,6 +39,28 @@ L'entrée `i` de l'estimator correspond exactement à l'entrée `i` du Match Fil File impose déjà des indices A strictement croissants ; le verifier ne trie et ne filtre pas les correspondances. Toute corruption d'index est une erreur d'exécution, jamais un rejet scientifique. +V2 distingue le nombre brut de lignes des observations canoniques distinctes. Les identités sont +`A=(feature_set_id_a, feature_index_a)` et +`B=(feature_set_id_b, feature_index_b)`. Après validation intégrale du parent, du Match asset, des +Feature Sets et Feature assets, v2 exige au moins sept A distincts **et** sept B distincts. Sinon il +publie `GEOMETRIC_REJECTED`, `inlier_count=0`, masque intégralement nul de longueur exactement +`ceil(match_count/8)`, sans modèle et sans appel USAC. + +Ce préflight ne modifie jamais l'évidence Matcher : aucune déduplication, tri, contrainte +one-to-one, unicité de coordonnées, limite de multiplicité, analyse de rang/conditionnement/ +colinéarité ou compétition Homography n'est appliquée. Des IDs distincts ayant les mêmes +coordonnées restent des observations distinctes. Le Match File canonique rendant A strictement +croissant, une insuffisance A implique en pratique moins de sept lignes valides ; B peut en revanche +être insuffisant malgré un grand nombre de lignes brutes. + +V3 exécute ensuite USAC seulement si `match_count >= min_inlier_count`. Lorsque cette inégalité +échoue, au plus `match_count` bits du masque pourraient être inliers : l'acceptation est donc +mathématiquement impossible. V3 publie alors le même rejet zéro borné sans appel estimator. La +borne vient du paramètre durable, jamais d'une constante `16`. À l'égalité, l'entrée reste éligible. +Ce contrat n'ajoute aucune règle `N<20`, rang, coordonnées, homographie, déduplication ou retry ; +toute entrée qui franchit les deux préflights appelle l'USAC inchangé et toute exception inattendue +reste un échec Task sans publication. + ## Coordinate representation Le stockage source reste `binary32`. Sur 1024 points, bruit 0,75 px et 50 % d'outliers, Point2f et @@ -91,7 +121,7 @@ quatre premiers octets sont décodés little-endian et les 31 bits faibles alime ## Parameter fingerprint -Le fingerprint v1 est SHA-256 des 84 octets suivants. Les entiers sont little-endian ; les doubles +Le fingerprint v1/v2/v3 est SHA-256 des 84 octets suivants. Les entiers sont little-endian ; les doubles sont leurs bits IEEE-754 binary64 écrits comme `uint64_t` little-endian. NaN/Inf sont refusés et le seul champ autorisant zéro signé, `min_inlier_ratio`, normalise `-0.0` en `+0.0`. Aucun octet ne provient d'un dump de structure. @@ -101,7 +131,7 @@ provient d'un dump de structure. | 0 | 8 | domaine ASCII `L3DGVFP1` | | 8 | 4 | version encodage = 1 | | 12 | 4 | kind FUNDAMENTAL = 1 | -| 16 | 4 | verifier version = 1 | +| 16 | 4 | verifier version = 1, 2 ou 3 | | 20 | 4 | algorithme USAC_MAGSAC explicite = 1 | | 24 | 8 | threshold binary64 | | 32 | 8 | confidence binary64 | @@ -122,10 +152,19 @@ provient d'un dump de structure. | 79 | 4 | polisher iterations = 3 | | 83 | 1 | réservé nul | -Le vector golden de la configuration production commence par les 84 octets hexadécimaux +Le vector golden v1 de la configuration historique commence par les 84 octets hexadécimaux `4c33444756465031...0300000000` et donne le SHA-256 `ddb44bb070c62be66c405946e89cbb49c084f8f30a21d6f408dc239225b7bbd0`. Pour un Match File SHA composé de 31 octets nuls puis `01`, cette configuration donne la seed décimale `1910542150`. +V2 conserve cet encodage et place `2` au champ `verifier version`; son fingerprint diffère donc +obligatoirement même lorsque les sept paramètres numériques sont identiques. La seed dérivée suit +ce fingerprint v2 et appartient à cette nouvelle identité. Pour la configuration production, le +fingerprint v2 est `7868a893437ee611a10008a093286997212fa8bd80b2afd2bb1d11f04f01c5ae` ; +le même Match SHA golden donne la seed décimale `1528046088`. +V3 conserve encore exactement les 84 octets et place `3` au champ version. Pour la configuration +production, son fingerprint est +`6944a471d611d8ffc59dac7cf15a5b79b97e2371d4c51785c477d68c1577f74c` ; le même Match SHA golden +donne la seed décimale `188721673`. Les politiques de ressources, hardware, PSI, lot, worker et réservation CPU ne sont ni des champs ni des entrées. @@ -160,11 +199,14 @@ popcount sont couverts avec le Model v1 inchangé. ## Scientific rejection -Un nombre de matches inférieur au minimum réel, l'absence de modèle sur entrée valide ou l'échec -de l'acceptance policy produit un résultat scientifique REJECTED cohérent. +En v1, un nombre brut de matches inférieur au minimum réel produit le rejet zéro historique. En +v2, moins de sept observations canoniques distinctes sur A ou B produit le rejet zéro décrit dans +`Input ordering`. V3 conserve ce test et rejette également quand le nombre brut est strictement +inférieur au `min_inlier_count` configuré. L'absence de modèle sur entrée valide ou l'échec de +l'acceptance policy produit un résultat scientifique REJECTED cohérent. -Le minimum USAC observé est sept. Moins de sept matches produit un masque zéro sans appel OpenCV ; -sept à quinze peuvent produire une hypothèse mais ne franchissent pas le support production. +Le minimum USAC observé est sept. Sept à quinze observations supportées peuvent produire une +hypothèse mais ne franchissent pas nécessairement le support d'acceptation production. ## Execution failure @@ -185,9 +227,11 @@ Une unité atomique est un Match Result, au maximum 8192 correspondances. Le Mat n'est utilisé. À 8192 matches, les allocations directement contrôlées maximales sont 98 304 octets d'entries, -393 216 octets de keypoints A/B, 262 144 octets de Point2d A/B, 1024 octets de bitset, environ -8192 octets de mask OpenCV et 72 octets de modèle, soit environ 745 Kio hors petits objets et -scratch OpenCV. Aucun descriptor ni matrice A×B n'est lu. Massif mesure 2,445 Mio de heap au pic +393 216 octets de keypoints A/B, 262 144 octets de Point2d A/B, deux cartes v2 de présence de +8192 octets, 1024 octets de bitset, environ 8192 octets de mask OpenCV et 72 octets de modèle, soit +environ 761 Kio hors petits objets et scratch OpenCV. Les cartes appartiennent à une paire et sont +libérées au retour ; leur borne Feature Store est opérationnelle et n'ajoute aucune limite +scientifique. Aucun descriptor ni matrice A×B n'est lu. Massif mesure 2,445 Mio de heap au pic du test E2E complet, incluant SQLite, OpenCV, fixtures Feature Store et toutes les séquences de test. Une réservation conservatrice de 4 Mio par job couvre ce profil mesuré. @@ -219,6 +263,12 @@ calculé à la création pour validation à la reconstruction. Aucun `cv::Mat`, paramètre Governor ou donnée hardware n'y appartient. La tâche calcule hors transaction et publie chaque résultat par transaction courte avant avancement du curseur. +Le payload v22 ne duplique pas `verifier_version`. À la reconstruction, le fingerprint exact est +comparé aux encodages supportés v1, v2 et v3 des sept paramètres durables : cela restaure sans +ambiguïté chaque policy, sans migration ni nouveau système d'identité. Toute nouvelle tâche +sélectionne v3 ; le payload et l'ABI publique de configuration restent inchangés. Une tâche v3 +démarre avec une identité neuve et ne reprend ni ne ré-étiquette une tâche v1/v2. + Le Task Kind production est `geometric_verifier.run` version 1. Il pagine les Match Results par ID strictement croissant avec une page de `batch + 1`, et traite des lots Governor 1/2/4/8. Les parents autres que `MATCHED` avec `match_count > 0` sont seulement traversés par le curseur. Une @@ -283,6 +333,12 @@ Gate A couvre corpus, comparaison, seed et repeatability. Gate B couvre fingerpr canonicalisation, mapping bit à bit, frontières d'acceptation, E2E DB, reuse, corruption, publication, 8192 matches et ASan/UBSan. Gate C couvre Task, publication avant curseur et reprise. +La référence A6000 v3 complète contient 37 805 parents `MATCHED`. Le préflight v3 rejette à zéro +9 368 parents avec `N<16`, puis le support distinct A/B rejette 117 parents supplémentaires avec +`N>=16`. Les 28 320 autres parents appellent USAC sous leur seed v3 exacte ; ils terminent tous par +un modèle ou une absence de modèle, sans exception estimator. Cette exécution démarre une tâche v3 +neuve et ne reprend ni ne ré-étiquette la tâche v2 historique 1385. + Gate D a exécuté 1000 parents configurés dans la vraie Task, puis les reprises et variantes de configuration du test : environ 2001 traversées réutilisées en 5,870 s, soit environ 341/s. Ce run valide pagination, checkpoints, Governor et reuse ; il n'est pas une mesure de latence MAGSAC diff --git a/docs/architecture/internal_parallelism.md b/docs/architecture/internal_parallelism.md new file mode 100644 index 0000000..6be9b2b --- /dev/null +++ b/docs/architecture/internal_parallelism.md @@ -0,0 +1,288 @@ +# Parallélisme interne borné + +## Statut + +**PASS / FROZEN — INTERNAL_PARALLELISM_COMPUTE_RESOURCES_V1.** Ce document +décrit le contrat validé pour `features.extract`, `features.extract.sift`, +`visual_index.update`, `candidate_pair.generate` et `matcher.run`. Ce gel porte +sur le parallélisme interne borné, les réservations Governor et les sorties +canoniques ; il ne transforme pas les résultats de microbenchmarks GPU en une +preuve de débit durable sur corpus sous pression hôte. + +## Frontière runtime + +La Task Queue conserve un seul callback actif. Une Task peut exploiter à +l'intérieur de ce callback les `cpu_threads` effectivement admis par le +Resource Governor. Ce parallélisme interne ne crée ni pool global, ni deuxième +scheduler, ni file de travail persistante. L'affinité CPU éventuelle appartient +au lanceur ou à l'hôte : la bibliothèque ne choisit et ne persiste aucun ID de +CPU. + +## Audit GPU — 29 août 2026 + +Cet audit est une constatation de capacité et une recommandation de périmètre ; +il n'ajoute aucun backend ni aucune politique du Governor. La machine contrôlée +expose `AMD Radeon 780M Graphics (RADV PHOENIX)`, iGPU Vulkan API 1.4.354 sous +Mesa 26.2.1. OpenCV 5.0.0 y indique le chargement dynamique de Vulkan et +d'OpenCL, avec TBB comme framework parallèle ; ses modules CUDA ne sont pas +disponibles. L'outil `clinfo` n'était pas installé : cette absence ne prouve ni +ne valide une exécution OpenCL. Elle ne constitue donc pas une couture GPU +utilisable par Lardon3D. + +La 780M est UMA. Toute mémoire de travail GPU, toute double résidence +CPU/GPU et tout staging host-visible consomment la RAM hôte et doivent être +comptés une seule fois par le Resource Governor, conformément à la règle UMA +du [Resource Boundary](resource_boundary.md#selected-gpu) ; ils ne créent pas +un budget VRAM indépendant. L'admission, la durée de réservation et la +libération restent du ressort du Governor et de la Task admise. Les copies ou +synchronisations hôte↔GPU, y compris sur mémoire partagée, restent un coût de +transfert/synchronisation à mesurer : l'UMA ne les rend pas gratuites. + +| Étape | Backend courant / couture GPU existante | Bibliothèque ou API requise si étude ultérieure | Compatibilité, sortie et coût | Recommandation de l'audit | +| --- | --- | --- | --- | +| `features.extract` (ORB/SIFT) | Extraction OpenCV CPU ; aucune couture GPU de production. | Aucune API d'extraction GPU n'est validée. Les indicateurs OpenCV Vulkan/OpenCL dynamiques ne fournissent pas à eux seuls un backend d'extraction ; CUDA est indisponible dans ce build. | Il faudrait prouver l'algorithme, les keypoints, descripteurs, ordre et Feature File produits. Les descripteurs et leurs buffers devraient être résidents ou stagés en RAM UMA ; le transfert et la synchronisation risquent de dominer les images bornées. Bénéfice non mesuré, complexité élevée. | Rester CPU. Une étude ne peut commencer qu'avec une API/backend concret et une preuve d'équivalence. | +| `visual_index.update` | Construction et publication CPU ; aucune couture GPU. | Aucun backend/API GPU validé. | Les postings, leur compaction, tri total et publication déterministe sont aujourd'hui CPU. Un backend devrait préserver exactement `table_id,key24,feature_set_id,feature_index`, le segment, SHA-256 et les memberships ; il ajouterait double résidence UMA et synchronisation pour un bénéfice non mesuré. Complexité élevée. | Rester CPU ; aucun chantier GPU n'est justifié par cet audit. | +| `candidate_pair.generate` | Requête Visual Index, top-K et publication CPU ; aucune couture GPU. | Aucun backend/API GPU validé. | Les scores, top-K, tie-breaks, normalisation et ordre de publication sont canoniques. Une accélération devrait rendre ces résultats identiques malgré les accès DB et les petits résultats bornés ; copies/synchronisations UMA et l'accès persistant réduisent le bénéfice attendu. Bénéfice non mesuré, complexité élevée. | Rester CPU ; ne pas introduire un backend GPU ou une seconde politique de sélection. | +| `matcher.run` | BFMatcher CPU ; couture existante limitée au backend Vulkan ORB sériel. Le mode parallèle CPU ne réserve ni n'utilise le GPU. | Vulkan compute existant pour ORB/Hamming top-2 uniquement. Aucun chemin Vulkan/OpenCL/CUDA validé pour SIFT/RootSIFT. | ORB Vulkan déjà prouve une parité top-2 et Match File complète avec le CPU, ce qui permet l'identité persistante commune. Les dispatchs restent soumis aux buffers descriptors et à la synchronisation UMA. SIFT/RootSIFT Vulkan n'est pas équivalent sur égalités adversariales et reste CPU. | Conserver le chemin ORB Vulkan sériel explicitement admis et le CPU parallèle séparé. Ne pas étendre à SIFT/RootSIFT ni fusionner les modes sans nouvelle validation. | + +L'attente « sortie identique » est une exigence de preuve, non une présomption +attachée au GPU. Un futur backend ne peut partager une identité, un fingerprint +ou une version scientifique existants que si l'équivalence complète des sorties +canoniques est démontrée à la frontière concernée. Si ses sorties diffèrent, il +doit recevoir un backend/version et une identité scientifique versionnés, avec +une décision et une validation dédiées ; il ne doit jamais être choisi comme un +fallback transparent. Le mode ORB Vulkan sériel existant est l'exception déjà +prouvée à cette règle d'identité commune. + +### Décision GPU finale — Radeon 780M + +La validation de production de `matcher.run` distingue l'identité scientifique +de son mode opérationnel. Les API historiques créent toujours le Matcher CPU +parallèle (`CPU=8`, `GPU=0`). L'API additive de mode explicite peut créer ORB +Vulkan seulement avant admission, avec `CPU=1`, un slot GPU et 640 Kio UMA. +Cette réservation couvre les buffers persistants A (256 Kio), B (256 Kio) et +top-2 (128 Kio); les 10 Mio de stage Matcher restent la mémoire CPU par paire. +Une panne de dispatch reprend entièrement le top-2 CPU avant toute publication; +elle ne publie jamais une sortie GPU partielle ni ne change fingerprint, +identité Match Result ou SHA du Match File. + +Sur l'hôte contrôlé (`AMD Radeon 780M Graphics (RADV PHOENIX)`, Vulkan 1.4.354, +Mesa 26.2.1), le backend a créé le device et exécuté les dispatchs réels. Les +mesures de kernel chaud, distinctes du coût Task/SQLite/checkpoint, sont : + +| Paire ORB | CPU BFMatcher | Vulkan | Accélération Vulkan | +| --- | ---: | ---: | ---: | +| 768 × 768 | 0,67 ms | 0,34 ms | 1,97× | +| 4096 × 4096 | 17,00 ms | 1,62 ms | 10,49× | +| 8192 × 8192 | 67,54 ms | 3,91 ms | 17,27× | + +La parité top-2, Match File et publication durable est couverte par une Task +Queue réelle à 769 × 769 descriptors, par une répétition déterministe et par +le fallback forcé. La reprise accepte seulement les signatures entières CPU8, +Vulkan CPU1 et les deux signatures CPU12 historiques; celles-ci sont +normalisées éphémèrement vers leur forme courante avant admission. Une signature voisine +est rejetée. La configuration portable `-Dvulkan_orb=disabled` refuse le mode +Vulkan avant allocation de Task ID et conserve le Matcher CPU. + +`HOST_PRESSURE_CONTAMINATED=TRUE` pour une comparaison de débit de corpus : au +moment de l'audit, `MemAvailable` était ~7,6 Gio, 7,6 Gio de swap étaient en +usage, même si la PSI mémoire courante était nulle. Il n'existe donc pas de +mesure défendable CPU t1/t6/t12 contre corpus Vulkan dans cette tranche. Le +Matcher CPU durable actuel plafonne en outre à huit participants utiles; un +CPU t12 ne serait pas une comparaison de Task valide. Les chiffres ci-dessus +sont volontairement limités au kernel commun et ne sélectionnent pas une +politique AUTO ou le backend par défaut. + +Les classifications finales sont : + +| Domaine | Classification | Évidence déterminante | +| --- | --- | --- | +| `candidate_pair.generate` | `CANDIDATE_GPU=REJECTED_WITH_MEASURED_REASON` | Le coût est Visual Index, filtrage, branchement et publication SQLite ordonnée; aucune primitive GPU existante ne préserve ces identités et le CPU parallèle Candidate a déjà démontré 7,114× à t12. | +| `features.extract` | `FEATURE_GPU=REJECTED_WITH_MEASURED_OR_IMPLEMENTATION_EVIDENCE` | OpenCV 5.0.0 installé n'expose aucun ORB/SIFT Vulkan/OpenCL utilisable, CUDA est absent, et aucun seam existant ne peut prouver les Feature Files byte-identiques. | +| `visual_index.update` | `VISUAL_INDEX_GPU=REJECTED_WITH_MEASURED_REASON` | Postings, tri total, SHA et publication déterministe sont CPU; aucun kernel GPU borné existant ne couvre cette frontière. | +| `matcher.run` ORB | `MATCHER_GPU=EXISTING_BACKEND_VALIDATED_BUT_CPU_PREFERRED` | Le backend est réellement admis, dispatché et exact, mais la pression hôte et l'absence d'une comparaison de débit Task/corpus saine interdisent de préférer Vulkan au CPU parallèle comme défaut. | + +Cette dernière classification ne rejette pas le backend : ORB Vulkan reste un +mode explicite validé. Elle interdit seulement de présenter les microbenchmarks +comme une preuve que le mode sériel GPU bat le Matcher CPU parallèle dans un +corpus durable complet. + +## Extraction OpenCV + +Le processus configure une seule fois la limite de threads interne OpenCV, +avant la création de la Queue, au budget interactif audité +`min(12, logical_cpu_count - system_cpu_reserve)`. Les Tasks ORB et SIFT demandent +le plafond douze au Governor, qui réduit l'admission à ce même budget ; la +réservation vit pendant l'exécution bornée d'une +image et est libérée avec la Task. OpenCV possède son fan-out interne, tandis +que la Queue reste propriétaire de l'unique callback actif. + +Le nombre admis est une politique de ressources, pas une identité scientifique. +L'audit OpenCV 5.0.0 contrôlé à 1/2/4/8/12 threads obtient des Feature Files +ORB v1 et SIFT v2 byte-identiques. Aucun fingerprint, format ou schéma n'est +modifié. Les utilisateurs imbriqués d'OpenCV qui réduisent temporairement cette +limite doivent restaurer sa valeur avant de libérer leur réservation ; ils ne +peuvent pas faire varier la configuration pendant une extraction concurrente. + +Pour Candidate Generation, l'estimation demande jusqu'à douze threads CPU et +un slot I/O. Le callback de Queue compte comme un de ces threads ; il crée donc +au plus `cpu_threads - 1` threads enfants. Tous les enfants sont joints avant +la fin de la séquence, la libération de réservation ou +`lardon3d_task_sequence_break()`. + +## Calcul et publication Visual Index + +Une séquence sélectionne le même préfixe durable d'au plus seize Feature Sets +qu'en mode sériel. Jusqu'à douze participants effectivement admis lisent les +Feature Files immuables ; chaque reader, son descripteur de fichier et sa +tranche de 256 descripteurs appartiennent à un seul participant. Aucun enfant +n'utilise le handle Project DB partagé. + +Chaque Feature Set écrit dans une tranche disjointe du buffer de postings borné +du segment. Après jointure, le callback compacte seul ces tranches dans l'ordre +de sélection, puis le tri total existant impose +`table_id,key24,feature_set_id,feature_index`. Le callback est l'unique +propriétaire de la sérialisation, du hash, de la publication asset et de la +transaction segment + memberships. Le nombre de participants ne change donc +ni octet, SHA-256, chemin, membership, génération, fingerprint, ni résultat de +requête. + +Une erreur de lecture interdit toute publication. Si la création d'un enfant +échoue, le callback calcule après les jointures uniquement les tranches restées +sans producteur. Le curseur n'avance qu'après commit du segment ; le checkpoint +Task suit ce commit, et tous les enfants sont joints avant `sequence_break`. +Le buffer de postings n'est pas multiplié : ses tranches privées partitionnent +la capacité fixe existante. Les coûts supplémentaires sont au plus une pile et +un reader/FD Feature File par participant admis. + +## Calcul et publication Candidate + +**CANDIDATE_PARALLELISM_IMPLEMENTATION=PASS.** Le parallélisme interne +Candidate est validé lorsque le Governor admet à la fois plusieurs CPUs et un +lot contenant plusieurs memberships indépendants. Les benchmarks CPU 1/6/12 +antérieurs restent valides pour leurs intervalles mesurés ; une preuve de débit +durable sur machine réelle exige en plus que chaque frontière observée admette +un lot supérieur à un. + +L'admission CPU et l'admission de lot sont deux contrats opérationnels +différents. `desired_cpu_threads` borne le fan-out disponible, tandis que +`batch_size` borne strictement le nombre de memberships que la Task peut +prendre dans la séquence courante. Ainsi, `cpu_threads=12` avec +`batch_size=1` est architecturalement valide mais ne contient qu'une source +indépendante : Candidate calcule un seul participant utile. Il ne doit jamais +emprunter du travail à une séquence ultérieure pour remplir les CPUs admis. + +Le test Candidate couvre désormais six Feature Sets et trois fenêtres +productives saines : l'admission initiale, puis deux réadmissions productives +après `sequence_break`, admettent chacune deux sources et obtiennent deux +participants utiles. Une troisième réadmission après `sequence_break` observe +le suffixe vide requis pour terminer. La sortie reste canonique, le curseur +durable est contigu et aucune paire n'est dupliquée. Le même test vérifie le +complément sûr : un lot admis d'un seul item, même avec douze CPUs disponibles, +produit exactement un participant par frontière productive. Cette réduction +est `EXPECTED_RESOURCE_SAFETY_BEHAVIOR`, pas une perte de parallélisme +Candidate. + +**REAL_HOST_CURRENT_STATE=GOVERNOR_PRESSURE_THROTTLING_ACTIVE.** Le 29 août +2026, un run S21 réel a conservé son estimate Candidate CPU-12 et son format +de checkpoint courant (`256 KiB + 64 KiB/item`, lot `1..64`) après 237 +séquences, mais le Governor a admis des lots d'un sous pression globale. Les +faits retenus sont : zram `6059936 / 6291452 KiB` (~96,3 %), swapfile ~1,92 +GiB, tandis que Candidate restait ~55 MiB RSS, `VmSwap=0` et +`MemAvailable` ~7,6 GiB. La disparition ultérieure du swap `/dev/sdb1` vers +14:00 a modifié la configuration de swap de l'hôte, mais elle est postérieure +à l'effondrement initial du lot et n'en est pas la cause initiale prouvée. +Cette décision reflète +des signaux de swap/PSI globaux, non la réservation Candidate elle-même. Les +seuils Gate G ne sont pas modifiés pour améliorer un benchmark ; une preuve +réelle soutenue doit attendre un état hôte qui admet répétitivement +`batch_size > 1`. + +Les sources sont les memberships durables du Visual Index, pagés en ordre +croissant de `feature_set_id`. Les IDs peuvent être clairsemés. Une fenêtre +contient au plus `2 * cpu_threads` sources et jamais plus de 24. Chaque résultat +conserve au plus le top-K existant de 256 propositions ; l'estimation de Task +reste conservatrice à 256 Kio fixes et 64 Kio par item admis. + +Chaque participant possède son propre handle `project.db` pour les lectures et +le ferme avant de terminer. Le calcul parallèle charge le Feature Set source, +exécute exactement la requête Visual Index existante et forme les paires +canoniques en mémoire. Il ne publie rien. + +La reprise normalise uniquement l'estimation Candidate v1 sérielle historique +de forme exacte vers la demande courante de douze participants. Cette +estimation effective reste privée et n'est pas checkpointée avant admission ; le Governor peut +toujours la réduire à son budget disponible. Le Task ID, le curseur typé, le +lot 1–64 et les paramètres scientifiques restent inchangés. Les autres kinds +ne passent pas par cette normalisation Candidate. Matcher possède sa propre +normalisation bornée des deux formes CPU12 historiques, décrite ci-dessous. + +Après la jointure complète, le thread propriétaire de la Task publie seul, dans +l'ordre croissant des sources puis dans l'ordre des résultats de requête. Il +emploie le comportement existant `find_candidate_pair` puis +`create_candidate_pair`. Les scores, top-K, tie-breaks, normalisation de paire, +fingerprints et identités scientifiques ne changent pas. Le nombre de threads +ne peut donc modifier ni les identités persistées, ni leur ordre de publication, +ni leur cardinalité. + +## Échec, reprise et progression + +Une erreur de calcul à la source `S` interdit toute publication de `S` et de +toute source suivante déjà calculée dans la fenêtre. Les sources antérieures +entièrement publiées forment le seul préfixe susceptible d'avancer le curseur. +Une reprise revoit éventuellement un préfixe publié avant un crash mais non +checkpointé ; l'idempotence existante fait converger ce rejeu sans deviner +d'identité. + +La progression est le rang du curseur parmi les memberships durables ordonnés, +divisé par leur nombre total. Elle n'utilise ni la valeur numérique du +`feature_set_id`, ni un compteur process-local, ni le nombre de séquences. Elle +est donc monotone pendant l'exécution et reconstruite à l'identique après +reprise pour un même état de memberships. Les valeurs incomplètes sont bornées +à 0–99 ; 100 n'est publié qu'après observation d'un suffixe vide et achèvement +de la Task. + +## Calcul et publication Matcher + +Les sources Matcher sont les Candidate Pairs durables, pagées en ordre +croissant de `candidate_pair_id`. Une fenêtre contient au plus +`2 * cpu_threads` paires et jamais plus de huit. Le callback Queue compte comme +un participant et crée donc au plus `cpu_threads - 1` enfants ; tous sont +joints avant publication, libération de réservation ou `sequence_break`. + +Chaque participant calcule un Match File temporaire privé. OpenCV est configuré +à un thread interne pendant la séquence afin que `BFMatcher` ne crée pas un +second fan-out sous les participants admis. Le backend Vulkan ORB partagé reste +utilisé uniquement par un mode déclaré sériel avant admission, dont l'estimation +réserve un thread CPU et le GPU. Le mode parallèle reste CPU-only même si le +Governor ne lui accorde finalement qu'un participant ; il ne sélectionne donc +jamais tardivement une ressource non réservée. Ce choix runtime ne modifie ni le +fingerprint, ni Lowe ratio, ni les correspondances brutes, ni l'identité Match +Result. + +Les APIs historiques sélectionnent toujours le mode CPU parallèle ; le mode +ORB Vulkan est demandé par une API additive explicite et n'est jamais activé +par variable d'environnement ou réduction Governor. Sa création exige ORB, un +GPU sélectionné et un backend présent. Après reprise, la forme immutable GPU +peut conserver le fallback CPU exact si le backend runtime n'est plus +disponible. Les signatures courantes complètes sont CPU8/GPU0 et +CPU1/GPU1/640 Kio ; les signatures historiques CPU12 correspondantes sont +normalisées éphémèrement comme un tout avant admission. Toute forme +voisine est rejetée afin de ne pas inférer un mode depuis un champ isolé. + +Après jointure, seul le callback propriétaire publie les stages, dans l'ordre +croissant de `candidate_pair_id`, par le chemin atomique Match Store existant. +Une erreur de calcul ou de publication interdit la paire fautive et tout son +suffixe déjà calculé. Le curseur avance exclusivement avec le préfixe contigu +durablement publié. Les temporaires non consommés sont supprimés sur succès, +échec et annulation ; les assets partagés ne sont jamais supprimés par ce +nettoyage. Le rejeu conserve l'idempotence Match Result existante. + +## Hors périmètre + +- parallélisme entre Tasks ou plusieurs callbacks Queue actifs ; +- pool de workers global ou persistant ; +- modification du schéma Project DB ; +- affinité, pinning ou IDs de CPU dans l'API ; +- modification de la sélection/scoring/fingerprint Candidate ; +- changement des décisions FROZEN du Resource Governor. diff --git a/docs/architecture/match_result.md b/docs/architecture/match_result.md index 8aa6f9a..c44401b 100644 --- a/docs/architecture/match_result.md +++ b/docs/architecture/match_result.md @@ -64,4 +64,4 @@ Le prochain modèle possède ses propres statistiques géométriques, dont un Geometric Verification Model v1 est désormais l'enfant scientifique persistant de ce résultat dans Project DB v12. -NEXT: GEOMETRIC VERIFIER v1 +NEXT: GEOMETRIC VERIFIER v3 diff --git a/docs/architecture/matcher.md b/docs/architecture/matcher.md index fe36ab1..b2970c2 100644 --- a/docs/architecture/matcher.md +++ b/docs/architecture/matcher.md @@ -83,14 +83,21 @@ flottante ne garantit pas le top-2 OpenCV sur les égalités adversariales et le gain n'est présent que sur les grandes paires carrées. Ils restent intégralement sur BFMatcher CPU. -`matcher.run` v1 orchestre le Matcher sans connaître son backend interne. La +**IMPLEMENTATION IN PROGRESS — parallélisme interne P4.** `matcher.run` v1 +orchestre le Matcher sans connaître son backend interne. La tâche persiste uniquement la configuration, l'identité des Feature Sets à sélectionner et un curseur Candidate Pair. Une paire est atomique et publiée immédiatement. Les lots 1/2/4/8 sont séparés par checkpoint et `task_sequence_break()`. La tâche utilise l'unique Resource Governor avec une estimation couvrant ce working set ; aucune seconde logique de budget n'est -introduite. Sa ligne durable `matcher_tasks` appartient au schéma Project DB -v11 ; le Match Result reste le contrat publié en v10. +introduite. Dans un lot, des fenêtres bornées calculent au plus deux paires par +thread CPU effectivement admis, avec un plafond de huit. Le callback Queue +compte comme un participant, joint au plus `cpu_threads - 1` enfants, puis +publie seul les stages en ordre `candidate_pair_id`. OpenCV reste à un thread +interne pendant cette séquence, ce qui interdit un fan-out BFMatcher imbriqué. +Le curseur ne suit que le préfixe contigu durable ; le premier échec scelle tout +le suffixe calculé. Sa ligne durable `matcher_tasks` appartient au schéma +Project DB v11 ; le Match Result reste le contrat publié en v10. Project DB v12 ajoute un enfant `Geometric Verification Result` référencé par `match_result_id`. Le Matcher ne calcule, ne stocke et ne valide aucun inlier @@ -105,8 +112,20 @@ canonicalisation et persistance restent communs. Sur Radeon 780M, la parité top-2 avec OpenCV est exacte et le gain warm est supérieur à 90 % à 4096/8192. Le backend de production conserve exactement cette frontière. Sa parité entière permet au CPU et à Vulkan de partager l'identité persistante. La sélection est -une politique runtime ; le CPU reste le fallback portable. Le contrat détaillé -est décrit dans [vulkan_matcher.md](vulkan_matcher.md). +une politique runtime explicite ; le CPU reste le fallback portable. Les APIs +historiques de création/soumission choisissent toujours le mode CPU parallèle. +Les variantes additives `*_with_mode` permettent de demander ORB Vulkan avant +l'admission ; elles refusent SIFT/RootSIFT, un profil sans GPU sélectionné ou +un backend absent. Ce mode immutable demande exactement CPU 1, GPU 1 et +640 Kio GPU/UMA. Un échec Vulkan pendant une exécution déjà admise reprend le +top-2 CPU exact sans changer fingerprint ni résultat. Le contrat détaillé est +décrit dans [vulkan_matcher.md](vulkan_matcher.md). + +La reprise reconnaît uniquement quatre estimations complètes : CPU8/GPU0 et +CPU1/GPU1/640 Kio courantes, plus CPU12/GPU0 et CPU12/GPU1/640 Kio +historiques. Les deux formes CPU12 sont normalisées éphémèrement vers leur +forme courante avant admission, sans checkpoint d'estimation seule. +Une forme voisine est rejetée ; aucun champ isolé ne sert à deviner le backend. ## Déterminisme et fingerprint @@ -117,4 +136,4 @@ mêmes paires attendues, l'ordre et la sérialisation stables. Aucune promesse bit-à-bit cross-platform n'est faite pour les distances L2 flottantes. ORB/Hamming bénéficie d'une garantie plus forte grâce à sa distance entière. -NEXT: GEOMETRIC VERIFIER v1 +NEXT: GEOMETRIC VERIFIER v3 diff --git a/docs/architecture/precision_feature_pipeline.md b/docs/architecture/precision_feature_pipeline.md index e4ec4e5..53048b3 100644 --- a/docs/architecture/precision_feature_pipeline.md +++ b/docs/architecture/precision_feature_pipeline.md @@ -56,11 +56,23 @@ validé. Après crash, la registry statique reconstruit le même `task_id`; l'im est recommencée, sans micro-checkpoint trompeur. Aucun Feature Set partiel ne devient READY. Le SHA-256 de l'Image Asset géré est vérifié avant décodage. -L'estimation SIFT réserve un slot CPU, un slot IO, aucun GPU et environ -1,06 Gio structurels : image décodée, pyramides OpenCV, candidats et -descriptors. Le pic de lot est zéro. OpenCV peut utiliser son backend parallèle -interne ; Lardon3D ne modifie aucun état global OpenCV. Le scheduler v1 reste à -un worker, ce qui borne actuellement l'oversubscription entre tâches. +L'estimation SIFT demande jusqu'à douze threads CPU ; le Governor réduit ce +plafond à la limite interne OpenCV configurée au démarrage. Elle réserve aussi +un slot IO, aucun GPU et environ 1,06 Gio structurels : +image décodée, pyramides OpenCV, candidats et descriptors. Le pic de lot est +zéro. Le Resource Governor admet donc le fan-out OpenCV complet ; la Queue +conserve un seul callback actif et ne superpose pas un second pool SIFT. + +Le nombre de threads reste opérationnel et absent du fingerprint SIFT/RootSIFT. +La reprise accepte la forme CPU12 courante et normalise uniquement l'ancienne +forme CPU1 complète et exacte dans la copie privée restaurée. Elle ne publie +aucun checkpoint d'estimation seule ; une forme voisine est rejetée. +L'audit contrôlé OpenCV 5.0.0 à 1, 2, 4, 8 et 12 threads compare exactement le +count, l'ordre et les valeurs binary32 des champs keypoint persistés, les +octaves, toutes les lignes/octets F32×128 et le SHA-256 du Feature File v2. +Toutes les configurations convergent vers les mêmes octets durables. Le champ +OpenCV `class_id`, non transporté par le record FeatureSet gelé, n'est ni +persisté ni utilisé comme identité scientifique. Le modèle n'est pas présenté comme un RSS mesuré : l'image grayscale atteint 100 MB à la limite post-décodage, le plafond défensif de candidats représente diff --git a/docs/architecture/project_database.md b/docs/architecture/project_database.md index 502f58e..1eef01d 100644 --- a/docs/architecture/project_database.md +++ b/docs/architecture/project_database.md @@ -539,7 +539,11 @@ Project Database miniature. La stratégie v1 retient un résumé logique interrogable dans SQLite et une référence vers le fichier checkpoint. Le fichier checkpoint validé reste la source complète pour `lardon3d_task_restore()` ; la DB seule ne reconstruit -jamais une tâche. Un écart ou un fichier invalide interdit la reprise. +jamais une tâche. La cohérence compare uniquement les champs que la DB stocke +dans `tasks` : `task_id`, nom, états saved/recovery, progression et compteur de +séquences. Elle ne prétend pas comparer l'estimation complète ni les timestamps +du snapshot. Une divergence de ce résumé ou un fichier invalide interdit la +reprise. ## Schéma v7 implémenté @@ -1050,9 +1054,21 @@ DB existante sans ligne projet ne peut être initialisée que si l'INI possède déjà son identité. Les checkpoints sont référencés par chemins relatifs portables : -`.lardon3d/checkpoints/.chk`. L'inventaire projet pagine les tâches DB, -résout ce chemin sous la racine, charge le checkpoint et vérifie la cohérence du -snapshot avec le résumé DB. +`.lardon3d/checkpoints/.chk`. Leur publication générique met d'abord +en durabilité `.chk.next`, commit ensuite SQLite, puis promeut `.next` vers le +canonique sous `.chk.lock`. Le verrou est consultatif et vivant seulement pour +le processus ; il sérialise writer et reprise, mais n'est pas une donnée de +récupération. + +L'inventaire projet utilise sa page DB seulement comme découverte. Après avoir +obtenu `.chk.lock`, il recharge la ligne DB, car elle peut avoir changé pendant +l'attente. Il choisit le canonique codec/version valide dont les champs du +résumé DB correspondent exactement. Ce choix est prioritaire et ignore une +`.next` périmée ou corrompue. Si le canonique ne correspond pas, une `.next` +codec/version valide dont le même résumé correspond est promue sous le verrou +puis utilisée. L'absence de `.next` reste normale pour un projet v22 legacy +contenant seulement `.chk`; toute autre absence, corruption, version future ou +divergence classe seulement la tâche comme non récupérable. Après copie du record hors mutex SQLite, l'inventaire consulte la registry. Il distingue `LEGACY_UNTYPED`, `UNKNOWN_TASK_KIND` et diff --git a/docs/architecture/resource_governor.md b/docs/architecture/resource_governor.md index e40c271..4013295 100644 --- a/docs/architecture/resource_governor.md +++ b/docs/architecture/resource_governor.md @@ -2,10 +2,12 @@ ## SIFT v1A -Une extraction SIFT réserve séparément un slot CPU et un slot IO, aucun GPU, -pour une image. L'estimation structurelle conservatrice est environ 1,06 Gio -(décodage, pyramides, candidats et F32×128), lot 1, pic de record batch zéro. -OpenCV peut employer son parallélisme interne ; aucun état global n'est modifié. +Une extraction SIFT demande jusqu'à douze threads CPU, un slot IO, aucun GPU, +pour une image. Le Governor réduit ce plafond au budget hôte, identique à la +limite OpenCV process-wide configurée avant les workers. L'estimation +structurelle conservatrice est environ 1,06 Gio (décodage, +pyramides, candidats et F32×128), lot 1, pic de record batch zéro. La +réservation couvre ainsi le fan-out interne sans créer un second pool runtime. ## Responsabilité @@ -126,29 +128,59 @@ monitoring live n'appartient à Gate G core. l'échantillon peut conserver taille/durée mais n'alimente jamais l'adaptation mémoire. - Pas de communication inter-classes de tâches -- `features.extract` réserve un lot de 1, un thread CPU et un slot I/O, avec +- `features.extract` réserve un lot de 1, demande jusqu'à douze threads CPU et + un slot I/O, avec 64 Mio fixes et 512 Mio par image. Cette estimation conservatrice couvre le chemin actuel sans prétendre mesurer les allocations internes d'OpenCV. `record_batch` couvre la validation source, le décodage, ORB, la publication et la finalisation DB ; `peak_memory_bytes == 0` signifie « mesure inconnue ». -- `visual_index.update` réserve un thread CPU, un slot I/O, 8 Mio fixes et - 2 Mio par Feature Set, par lots de 1 à 16. Le GPU vaut zéro. `record_batch` - compte uniquement les memberships commités et conserve la mémoire inconnue à zéro. -- `candidate_pair.generate` réserve un thread CPU, un slot I/O, 128 Kio fixes - et 256 Kio par Feature Set, par lots de 1 à 64. Le GPU vaut zéro. +- `visual_index.update` demande jusqu'à douze threads CPU, un slot I/O, 8 Mio + fixes et 2 Mio par Feature Set, par lots de 1 à 16. Le GPU vaut zéro. Le + callback compte comme participant et crée au plus `cpu_threads - 1` enfants, + tous joints avant publication et rupture de séquence. Chaque participant + possède au plus un reader/FD Feature File ; les tranches de postings privées + partitionnent le buffer borné du segment. `record_batch` compte uniquement + les memberships commités et conserve la mémoire inconnue à zéro. +- `candidate_pair.generate` demande jusqu'à douze threads CPU, un slot I/O, + 256 Kio fixes et 64 Kio par Feature Set, par lots de 1 à 64. Le GPU vaut + zéro. La reconstruction reconnaît uniquement l'ancienne estimation exacte + (128 Kio fixes, 64 Kio par item, un thread CPU, un slot I/O, aucun GPU, lots + 1 à 64, classe CPU) et la normalise éphémèrement vers l'estimation courante + complète ; aucun checkpoint d'estimation seule n'est publié et une forme + voisine n'est jamais réinterprétée comme legacy. `record_batch` compte le nombre de paires générées par séquence et la durée réelle du lot ; `peak_memory_bytes == 0` signifie « mesure inconnue ». - Chaque séquence interroge le Visual Index pour jusqu'à 64 Feature Sets et - persiste les paires candidates avec idempotence. -- `matcher.run` réserve douze threads CPU, un slot IO et un working set - contrôlé inférieur à environ 10 Mio au + Chaque séquence interroge le Visual Index pour jusqu'à 64 memberships. Le + calcul emploie des fenêtres internes d'au plus deux sources par thread admis + et 24 sources au total ; le propriétaire de Task persiste ensuite seul et en + ordre canonique. Cette estimation opérationnelle ne limite pas la taille + scientifique du dataset. +- `matcher.run` demande jusqu'à huit threads CPU, un slot IO et 10 Mio par + Candidate Pair admise, correspondant au working set contrôlé inférieur à + environ 10 Mio par paire au maximum SIFT/RootSIFT (8 Mio de descripteurs contigus, KNN `k=2`, sorties et fichier bornés), hors scratch interne OpenCV. Ses lots sont bornés à 1, 2, 4 - ou 8 Candidate Pairs et chaque paire libère ses buffers avant la suivante. - Quand le profil détecte un GPU et que le runtime possède le backend ORB, la - réservation ajoute un slot GPU et 640 Kio. Sur UMA ces 640 Kio sont aussi - débités du budget RAM. Sans GPU/backend, l'estimation reste CPU-only afin que - le fallback portable ne soit jamais refusé artificiellement. + ou 8 Candidate Pairs. **IMPLEMENTATION IN PROGRESS — P4 :** une fenêtre + contient au plus deux paires par thread CPU effectivement admis et huit + paires au total. Le callback Queue est un participant, crée au plus + `cpu_threads - 1` enfants et les joint avant publication et libération de la + réservation. Chaque paire conserve ses buffers dans un stage privé jusqu'à + sa publication ordonnée ou son nettoyage ; OpenCV reste à un thread interne + pour éviter une sursouscription imbriquée. Le Governor réserve donc au plus + 80 Mio contrôlés pour un lot de huit ; cette borne opérationnelle ne limite + pas la cardinalité scientifique du dataset. + Le mode d'exécution est fixé par l'estimation immutable avant admission. Le + mode parallèle par défaut est CPU-only et ne réserve aucun GPU, même si le + Governor réduit ensuite son admission à un thread. Un mode explicitement + sériel ORB, avec profil GPU et backend runtime disponibles, demande exactement + un thread CPU, un slot GPU et 640 Kio ; sur UMA ces 640 Kio sont aussi débités + du budget RAM. Sa reconstruction conserve ce mode à partir de l'estimation + durable, sans sélection tardive d'une ressource non réservée. + La reprise accepte les formes courantes exactes CPU8/GPU0 et + CPU1/GPU1/640 Kio ainsi que leurs prédécesseurs exacts CPU12. Elle normalise + éphémèrement CPU12 vers la forme courante du même mode avant admission. Toute + forme voisine est rejetée ; cette compatibilité opérationnelle ne persiste ni + ne déduit une identité backend dans Project DB. - `track_builder.run` réserve un worker CPU, aucun GPU et aucun fan-out GVR. L'estimation est `4 MiB + raw_inlier_edges * (48 + 2*160)` avec facteur 2 sous 400000 arêtes et facteur 8 au-delà, après vérification d'overflow. Le diff --git a/docs/architecture/task_kind_registry.md b/docs/architecture/task_kind_registry.md index e2e0533..34e57e3 100644 --- a/docs/architecture/task_kind_registry.md +++ b/docs/architecture/task_kind_registry.md @@ -31,6 +31,16 @@ transfert, la registry nettoie le userdata sur toute erreur ; après restauratio réussie, `Lardon3DTask` en devient propriétaire et le détruit une fois après la fin de l'exécution. Le constructeur métier n'est jamais appelé sous mutex DB. +La registry peut normaliser une ancienne estimation opérationnelle connue. Le +reconstructeur reçoit toujours le snapshot durable original afin de valider le +mode exact ; la registry applique ensuite l'estimation effective uniquement à +la copie privée transmise à la restauration de `Task`. Cette normalisation est +éphémère et déterministe : elle ne stage, ne promeut et ne publie aucun +checkpoint contenant seulement une estimation différente sous le même résumé. +Une panne pré-terminale peut donc répéter la normalisation exacte. Cette couture +ne peut modifier ni identité, paramètres scientifiques, progression ou curseur +métier, et toute forme voisine est rejetée. + ## Persistance et legacy Le checkpoint générique reste en version 1. Project Database v7 conserve le @@ -62,13 +72,27 @@ depuis `image_id` et ses paramètres bornés. **IMPLEMENTED** — `candidate_pair.generate`, version 1, recharge `visual_index_id + after_feature_set_id + top_k + minimum_evidence_count + scanset_filter + exclude_same_asset` depuis `candidate_pair_generate_tasks` -et reconstruit un contexte boundé. +et reconstruit un contexte boundé. La restauration reconnaît uniquement le +snapshot opérationnel sériel historique v1 exact et remplace éphémèrement sa +demande d'un thread CPU par douze avant admission. Le checkpoint historique et +le curseur typé restent inchangés. Les snapshots Candidate courants et tous les +autres kinds restent inchangés. **IMPLEMENTED** — `matcher.run`, version 1, recharge la configuration Matcher, l'identité Feature Set et le curseur `after_candidate_pair_id`. Il traite une Candidate Pair atomique à la fois dans des lots bornés à huit, checkpoint le curseur et repasse par le Governor entre les lots. La table durable `matcher_tasks` est introduite par Project DB v11, après le Match Result v10. +Son reconstructeur accepte les formes opérationnelles exactes CPU8/GPU0 et +CPU1/GPU1/640 Kio, puis normalise éphémèrement leurs deux prédécesseurs CPU12 +vers la forme courante correspondante avant l'admission. Une forme +voisine échoue au lieu de servir d'indice de backend ; le payload Project DB ne +change pas et ne persiste aucune identité matérielle. + +**IMPLEMENTED** — `features.extract.sift` et `features.extract.rootsift` +acceptent leur forme CPU12 courante et normalisent uniquement leur forme CPU1 +historique exacte. Cette compatibilité opérationnelle n'altère ni fingerprint, +Feature Set, checkpoint durable, ni politique scientifique. **IMPLEMENTED** — `geometric_verifier.run`, version 1, recharge la configuration Fundamental immuable, en revalide le fingerprint et reprend `after_match_result_id`. diff --git a/docs/architecture/task_system.md b/docs/architecture/task_system.md index 0dbbe8b..ffddb8a 100644 --- a/docs/architecture/task_system.md +++ b/docs/architecture/task_system.md @@ -51,11 +51,40 @@ une opération de réadmission, pas un état supplémentaire. | `lardon3d_task_restore_typed()` | Restaure une tâche typée et transfère l'ownership du userdata | | `lardon3d_task_checkpoint_save()` | Publie atomiquement un snapshot v1 | | `lardon3d_task_checkpoint_load()` | Lit et valide un checkpoint borné | +| `lardon3d_task_checkpoint_stage()` / `promote_staged()` | Publie d'abord `.next`, puis promeut sous le verrou par tâche | Après `rename`, un échec du `fsync` du répertoire retourne `LARDON3D_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE` : la publication est visible, mais sa durabilité après crash n'est pas confirmée. +### Publication et reprise projet + +Le protocole générique par tâche emploie le chemin canonique +`.lardon3d/checkpoints/.chk`, sa représentation staged `.chk.next` et +le verrou consultatif `.chk.lock`. Le verrou sérialise un writer et la reprise +sur le slot staged fixe ; il est détenu pour la séquence entière et sa fermeture +par le noyau après crash ne constitue pas un état durable. + +L'ordre de publication est strictement : codec durable de `.next`, commit +SQLite du résumé de tâche et de la référence checkpoint, puis promotion de +`.next` vers le fichier canonique et durabilité du répertoire. DB et système de +fichiers ne forment pas une transaction unique : si la promotion échoue ou sa +durabilité est incertaine après le commit SQLite, `.next` reste la +représentation de récupération possible. + +La reprise acquiert d'abord `.lock`, puis recharge la ligne SQLite : la page de +découverte peut être devenue périmée pendant l'attente. Elle valide le codec et +la version du canonique et compare seulement les champs effectivement stockés +dans le résumé DB — `task_id`, nom, états saved/recovery, progression et +compteur de séquences. Les timestamps et l'estimation complète ne sont pas +dupliqués par la DB et ne participent donc pas à ce test. Un canonique valide +dont ce résumé correspond est prioritaire ; une `.next` absente, périmée ou +corrompue est alors ignorée. Sinon, une `.next` valide qui correspond au même +résumé est promue sous le même verrou puis reprise. Un ancien projet v22 qui ne +possède que `.chk` reste ainsi récupérable. Toute autre absence, version non +supportée, corruption ou divergence interdit la reprise de cette tâche sans +affecter les autres entrées de l'inventaire. + ## Invariants 1. **Estimation immuable** : une fois créée, l'estimation d'une tâche ne change @@ -165,9 +194,11 @@ ni estimation sérialisée complète, ni callback, ni réservation, et ne rempla pas la validation du fichier checkpoint avant `task_restore()`. `lardon3d_project_checkpoint_task()` est la frontière runtime : elle capture le -snapshot, publie le fichier hors mutex de tâche, puis met à jour la DB. L'API -d'inventaire retourne des snapshots durables validés, mais ne peut pas appeler -`task_restore()` que via un descriptor connu ; aucun pointeur n'est persistant. +snapshot, stage le fichier hors mutex de tâche, met à jour la DB, puis promeut +le fichier canonique sous le verrou par tâche. L'API d'inventaire retourne des +snapshots dont le codec/version et le résumé DB ont été validés, mais ne peut +appeler `task_restore()` que via un descriptor connu ; aucun pointeur n'est +persistant. ## Limites diff --git a/docs/architecture/track_builder.md b/docs/architecture/track_builder.md index 4d4301e..db60808 100644 --- a/docs/architecture/track_builder.md +++ b/docs/architecture/track_builder.md @@ -66,6 +66,12 @@ The selector is exactly the FROZEN Track Model tuple only when its status is `GEOMETRIC_VERIFIED` and all three fields match exactly. `latest`, timestamps, greatest IDs and insertion order are forbidden. +Geometric Verifier v1, v2, and v3 are consequently separate evidence lineages. +A selector for any one version rejects every GVR from either other version, +even when they share a Match Result or numerical estimator parameters. The +current input selects exact v3 VERIFIED IDs and never mixes historical v1/v2 +evidence. No fallback or cross-version equivalence is inferred by Track Builder. + ## Input Scope The explicit supplied IDs are sorted numerically, unique, counted, and hashed diff --git a/docs/architecture/visual_index.md b/docs/architecture/visual_index.md index c8427ae..fbd4acf 100644 --- a/docs/architecture/visual_index.md +++ b/docs/architecture/visual_index.md @@ -56,9 +56,12 @@ ou endianness hôte n'intervient. `visual_index_id` est une identité SQLite Au plus `max_features_per_set` features sont indexées. La sélection v1 retient le préfixe de `feature_index` croissant. Les postings conservent l'indice -original. Un Feature Set vide est -membre valide sans posting. Le build ne garde qu'un Feature Set et une tranche -Feature Reader de 256 descripteurs en mémoire. +original. Un Feature Set vide est membre valide sans posting. Le build Task +peut lire en parallèle jusqu'à douze Feature Files, avec un reader et une +tranche de 256 descripteurs privés par participant effectivement admis. Chaque +Feature Set écrit dans une tranche privée de la capacité de postings déjà +réservée pour le segment ; le propriétaire compacte ensuite les tranches dans +l'ordre de sélection et applique seul l'ordre total persistant. ## Segments introduits en Project Database v6, conservés en v7 @@ -157,17 +160,36 @@ coopératives entre lectures et avant publication ; un segment déjà READY rest valide. La reprise recommence au dernier curseur commité et l'unicité des memberships rend le rejeu idempotent. -L'estimation réserve un thread CPU, un slot I/O, GPU zéro, 8 Mio fixes et 2 Mio -par Feature Set, lot 1..16. `record_batch` reçoit le nombre de Feature Sets -réellement commités, la durée réelle et `peak_memory_bytes=0` (inconnue). +**IMPLEMENTED — parallélisme interne borné.** La Queue exécute toujours un seul +callback. L'estimation demande jusqu'à douze threads CPU, un slot I/O, GPU zéro, +8 Mio fixes et 2 Mio par Feature Set, lot 1..16. Le callback compte comme un +participant et crée au plus `cpu_threads - 1` enfants. Chaque enfant lit +exclusivement des Feature Files immuables et écrit une tranche privée ; il ne +touche ni au handle Project DB partagé, ni au fichier de segment, ni au curseur. +Tous les enfants sont joints avant tri, sérialisation, publication asset et +transaction SQLite. + +La réduction emploie l'ordre total v1 +`table_id,key24,feature_set_id,feature_index`. Le fichier, son SHA-256, le +chemin, les memberships, la génération, le fingerprint et les résultats de +requête sont donc exactement identiques à un build avec un participant. Une +erreur de lecture dans une tranche interdit toute publication ; une création +de thread refusée est remplacée par le calcul de cette tranche sur le callback, +sans changer la réduction. Le curseur n'avance qu'après la publication +transactionnelle du segment, puis le checkpoint existant reste le seul point +de reprise Task. `record_batch` reçoit le nombre de Feature Sets réellement +commités, la durée réelle et `peak_memory_bytes=0` (inconnue). ## Complexité et limites Pour `D` descriptors échantillonnés, construction et disque sont `O(6D)`. Une requête effectue `O(6Q log P + H)` par segment (`Q<=1024`, `H` hits bornés), -pas `O(images²)`. La mémoire build est bornée par huit métadonnées, 256 -descripteurs et les postings d'un segment; la mémoire query par 4096 candidats, -256 postings et 256 résultats. À 3700 images et 512 features, environ 11,4 +pas `O(images²)`. La mémoire build est bornée par les métadonnées, les tranches +de 256 descripteurs privées des participants et les postings d'un segment ; les +tranches privées partitionnent le buffer de postings existant et ne le +dupliquent pas. Chaque participant garde au plus un reader/FD de Feature File. +La mémoire query est bornée par 4096 candidats, 256 postings et 256 résultats. +À 3700 images et 512 features, environ 11,4 millions de postings sont produits. Un test structurel persiste 50 000 Feature Sets puis confirme la pagination par 16 et le refus propre après 4096 memberships. Un index unique ne couvre donc pas encore 50 000 images : le risque principal est le nombre de segments diff --git a/docs/roadmap/roadmap.md b/docs/roadmap/roadmap.md index 92cde9e..7da3b85 100644 --- a/docs/roadmap/roadmap.md +++ b/docs/roadmap/roadmap.md @@ -181,21 +181,76 @@ présumer que chaque RAW a un JPEG sibling. La revue TUI détaillée, la guidanc de capture et les keyframes vidéo restent des intégrations ultérieures qui réutiliseront ce même chemin de qualité, sans second pipeline. -## NEXT REAL-DATA MILESTONE — SELECTED SCIENTIFIC EXECUTION ON ENGINE BAY INPUTS +## NEXT REAL-DATA MILESTONE — SELECTED SCIENTIFIC EXECUTION ON ENGINE BAY INPUTS — IMPLEMENTATION / VALIDATION IN PROGRESS -Après validation de Photo Quality Triage / Acquisition Selection, l'intégration -scientifique porte uniquement sur les acquisitions sélectionnées : +L'intégration réelle opt-in porte uniquement sur les acquisitions sélectionnées +des campagnes Engine Bay et s'arrête à la frontière pré-SfM validable. Pour +chaque campagne, un exécutable éphémère crée un projet temporaire Project DB v22 +distinct : Visual Index v1 est borné à 4096 images, tandis que l'ensemble +A6000+S21 en compte 4497. Cette séparation est opérationnelle ; elle ne redéfinit +ni Capture, ni Asset, ni `image_id`, ni l'identité scientifique des acquisitions. -1. matérialiser leurs représentations scientifiques ; -2. exécuter features, candidats, matching, vérification, tracks et Sparse SfM ; -3. évaluer la qualité de la reconstruction sparse multi-campagne ; -4. exécuter MVS/dense avec budgets et scratch contrôlés ; -5. publier durablement nuage dense et mesh ; -6. comparer le résultat aux tentatives photogrammétriques antérieures. +Le runner compose les API durables existantes, dans cet ordre : -Ce milestone est une intégration réelle, pas une nouvelle série de micro-gates -S3. Les antécédents d'OOM/SIGSEGV OpenMVS, pression swap, grands intermédiaires -et temps longs imposent reprise, progression et contrôle de ressources. +```text +qualité +→ campagne d'acquisition +→ snapshot immuable de la sélection +→ représentation (A6000 `raw.develop`, S21 JPEG SOURCE) +→ features ORB +→ Visual Index +→ candidats du même ScanSet +→ matching ORB +→ GV gelée +→ tracks +``` + +Il rouvre ensuite le Project DB afin de rapporter l'évidence durable produite. +Il n'introduit ni coordinateur réutilisable, ni DAG, ni sidecar, ni scheduler, +ni version Project DB v23 : Task, Queue, Scheduler et Resource Governor existants +conservent leurs responsabilités. + +Le Sparse SfM réel final reste `BLOCKED_BY_KNOWN_CALIBRATION_DATA` pour ces +campagnes ; aucune pseudo-calibration, interpolation de métadonnées ou inférence +d'identité ne le contourne. Dense, mesh et publication aval ne font pas partie de +ce jalon pré-SfM. Ce milestone est une intégration réelle, pas une nouvelle série +de micro-gates S3, et son statut ne revendique encore ni implémentation achevée, +ni validation réussie, ni PASS/FROZEN. + +## INTERNAL PARALLELISM + COMPUTE RESOURCES v1 — PASS / FROZEN + +La fondation scientifique pré-SfM courante comprend le parallélisme interne +borné des étapes Candidate, Matcher et Visual Index, l'audit du threading des +features, et l'admission/libération des ressources par le Resource Governor +existant. Elle ne crée ni parallélisme inter-Tasks, ni pool global, ni second +Scheduler, ni second Governor, ni persistance parallèle. Le contrat gelé est +celui du [parallélisme interne borné](../architecture/internal_parallelism.md). + +L'audit GPU vérifie les coutures réellement disponibles, les coûts UMA, les +réservations et la préservation des sorties canoniques. Il ne présume pas qu'un +GPU soit approprié ni qu'une sortie GPU partage automatiquement l'identité +scientifique CPU. Le gel ne revendique pas de comparaison de débit durable +CPU-versus-GPU sur corpus : la pression hôte a contaminé cette mesure. + +Un éventuel **GPU COMPUTE v1** est postérieur et optionnel. Il ne peut être +ouvert que si un audit futur et une preuve d'équivalence à la frontière concernée +le justifient ; sinon le chemin CPU demeure le chemin retenu. Il ne modifie pas +les contrats scientifiques, les résultats canoniques ni l'unique gouvernance des +ressources. + +L'ordre de travail autorisé à court terme est : + +```text +fondation scientifique pré-SfM courante +→ INTERNAL PARALLELISM + COMPUTE RESOURCES v1 + (Candidate, Matcher, Visual Index, audit feature threading, + correction progression/Resource Governor, audit GPU) +→ GPU COMPUTE v1 optionnel, seulement si audit et équivalence le justifient +→ poursuite pré-SfM réelle complète de S21 +→ acquisition dédiée de calibration +→ Sparse SfM réel +→ Dense / MVS +``` ## NEAR TERM @@ -504,9 +559,13 @@ L'ordre demeure sans ambiguïté : ```text CURRENT NEXT - acquisition physique dédiée de calibration + fondation scientifique pré-SfM courante + → INTERNAL PARALLELISM + COMPUTE RESOURCES v1 + → GPU COMPUTE v1 optionnel si audit/équivalence le justifient + → poursuite pré-SfM réelle complète de S21 + → acquisition physique dédiée de calibration → calibration connue validée → Sparse SfM réel multi-campagne - → dense/mesh → publication + → dense / MVS → publication → LATER Coverage Analysis → Coverage Viewer → suggestions de points de vue → localisation live → intégration HDMI/USB → Capture Guidance / Live Coverage diff --git a/include/lardon3d/feature_extractor.h b/include/lardon3d/feature_extractor.h index 4d9664d..7761b8e 100644 --- a/include/lardon3d/feature_extractor.h +++ b/include/lardon3d/feature_extractor.h @@ -68,8 +68,13 @@ typedef enum { LARDON3D_FEATURE_EXTRACT_ERROR } Lardon3DFeatureExtractResult; -/* Configuration OpenCV process-wide. À appeler avant le démarrage des workers. */ +/* Configure OpenCV's process-wide internal CPU-thread limit before workers start. + * `threads` must be in 1..INT_MAX. The setting is operational: it does not alter + * extractor fingerprints or FeatureSet identity. Callers must not race this API + * with extraction; the application runtime owns configuration and nested users + * must restore the preceding value before releasing their Task reservation. */ bool lardon3d_feature_opencv_configure_threads(unsigned int threads); +/* Return OpenCV's current process-wide limit, normalized to at least one thread. */ unsigned int lardon3d_feature_opencv_thread_count(void); bool lardon3d_feature_extractor_parameters_valid( diff --git a/include/lardon3d/geometric_verifier.h b/include/lardon3d/geometric_verifier.h index d10e8c3..ceb3efd 100644 --- a/include/lardon3d/geometric_verifier.h +++ b/include/lardon3d/geometric_verifier.h @@ -2,12 +2,17 @@ #define LARDON3D_GEOMETRIC_VERIFIER_H #include +#include #include #include enum { - LARDON3D_GEOMETRIC_VERIFIER_VERSION = 1, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V1 = 1, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V2 = 2, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3 = 3, + LARDON3D_GEOMETRIC_VERIFIER_VERSION = + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3, LARDON3D_GEOMETRIC_VERIFIER_FINGERPRINT_SIZE = 84, LARDON3D_GEOMETRIC_VERIFIER_MINIMUM_MATCHES = 7, }; @@ -45,6 +50,14 @@ bool lardon3d_geometric_verifier_fingerprint_bytes( const Lardon3DGeometricVerifierParameters *parameters, Lardon3DGeometricVerifierAlgorithm algorithm, uint32_t verifier_version, unsigned char bytes[LARDON3D_GEOMETRIC_VERIFIER_FINGERPRINT_SIZE]); +/* Computes the identity fingerprint for one supported scientific policy. + * Versions 1 through 3 are retained so durable historical tasks and GVRs + * remain reconstructable; new callers should select VERSION_V3. Returns false + * and writes a zero fingerprint for invalid parameters or versions. */ +bool lardon3d_geometric_verifier_fingerprint_for_version( + const Lardon3DGeometricVerifierParameters *parameters, + uint32_t verifier_version, unsigned char fingerprint[32]); +/* Convenience form for the current production policy (v3). */ void lardon3d_geometric_verifier_fingerprint( const Lardon3DGeometricVerifierParameters *parameters, unsigned char fingerprint[32]); @@ -56,10 +69,24 @@ Lardon3DGeometricVerifierResult lardon3d_geometric_verifier_verify_and_publish( uint64_t match_result_id, const Lardon3DGeometricVerifierParameters *parameters, Lardon3DProjectDbGeometricVerificationResult *result, bool *reused); +/* Verifies and atomically publishes or exactly reuses the requested v1/v2/v3 + * identity. Output is caller-owned. No scientific row is published for I/O, + * corruption, allocation, estimator, or database failure. C++ exceptions are + * contained by the implementation and never cross this C17 ABI. */ +Lardon3DGeometricVerifierResult +lardon3d_geometric_verifier_verify_and_publish_version( + const char *project_path, Lardon3DProjectDb *database, + uint64_t match_result_id, + const Lardon3DGeometricVerifierParameters *parameters, + uint32_t verifier_version, + Lardon3DProjectDbGeometricVerificationResult *result, bool *reused); #ifdef LARDON3D_GEOMETRIC_VERIFIER_TESTING void lardon3d_geometric_verifier_test_reset_estimator_calls(void); uint32_t lardon3d_geometric_verifier_test_estimator_calls(void); +bool lardon3d_geometric_verifier_test_has_minimal_support( + const uint32_t *indices_a, const uint32_t *indices_b, size_t count, + uint32_t feature_count_a, uint32_t feature_count_b); #endif #endif diff --git a/include/lardon3d/matcher_task.h b/include/lardon3d/matcher_task.h index c213a6d..9ba0065 100644 --- a/include/lardon3d/matcher_task.h +++ b/include/lardon3d/matcher_task.h @@ -23,14 +23,40 @@ typedef struct { Lardon3DMatcherParams matcher; } Lardon3DMatcherTaskConfiguration; +/* Selects only the admitted execution resources and runtime backend. Matcher + * parameters, fingerprints, Match Results, and Match Files are identical + * across modes. CPU_PARALLEL is portable and is the default used by the + * existing create/enqueue APIs. ORB_VULKAN requires an ORB configuration, a + * selected GPU, and an available Vulkan backend when the task is created. */ +typedef enum { + LARDON3D_MATCHER_TASK_MODE_CPU_PARALLEL = 0, + LARDON3D_MATCHER_TASK_MODE_ORB_VULKAN = 1, +} Lardon3DMatcherTaskMode; + Lardon3DTask *lardon3d_project_create_matcher_task( Lardon3DAppState *state, const Lardon3DMatcherTaskConfiguration *configuration, uint64_t *task_id); +/* Creates a Matcher task with an explicit immutable operational mode. On + * success, task_id receives the durable Task ID and the caller owns the Task. + * Invalid or unavailable mode selections return NULL and leave task_id zero. */ +Lardon3DTask *lardon3d_project_create_matcher_task_with_mode( + Lardon3DAppState *state, + const Lardon3DMatcherTaskConfiguration *configuration, + Lardon3DMatcherTaskMode mode, uint64_t *task_id); + bool lardon3d_project_enqueue_matcher_task( Lardon3DAppState *state, const Lardon3DMatcherTaskConfiguration *configuration, uint64_t *task_id); +/* Creates and transfers an explicitly selected Matcher task to state's Queue. + * Failure leaves task_id zero unless durable task creation succeeded before a + * Queue insertion failure, matching the existing enqueue ownership contract. */ +bool lardon3d_project_enqueue_matcher_task_with_mode( + Lardon3DAppState *state, + const Lardon3DMatcherTaskConfiguration *configuration, + Lardon3DMatcherTaskMode mode, uint64_t *task_id); + bool lardon3d_matcher_task_reconstruct( const Lardon3DTaskDurableSnapshot *snapshot, void *context, Lardon3DTaskKindBinding *binding); diff --git a/include/lardon3d/project_db.h b/include/lardon3d/project_db.h index 250a2de..9f12881 100644 --- a/include/lardon3d/project_db.h +++ b/include/lardon3d/project_db.h @@ -564,6 +564,13 @@ Lardon3DProjectDbResult lardon3d_project_db_find_track_by_observation( Lardon3DProjectDbResult lardon3d_project_db_open(const char *path, Lardon3DProjectDb **database, char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY]); +/* Copies the filename of the durable main database opened by this handle. + * The caller owns `path`, whose capacity must be at least + * LARDON3D_PROJECT_DB_PATH_CAPACITY. This identity lets private reader handles + * reopen the selected database without inferring a project-relative basename. */ +bool lardon3d_project_db_copy_path( + Lardon3DProjectDb *database, + char path[LARDON3D_PROJECT_DB_PATH_CAPACITY]); void lardon3d_project_db_close(Lardon3DProjectDb *database); bool lardon3d_project_db_last_error(Lardon3DProjectDb *database, char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY]); @@ -877,6 +884,23 @@ Lardon3DProjectDbResult lardon3d_project_db_list_visual_index_segments( Lardon3DProjectDbResult lardon3d_project_db_list_visual_index_pending( Lardon3DProjectDb *database, uint64_t visual_index_id, uint64_t after_feature_set_id, Lardon3DProjectDbFeatureSet *feature_sets, size_t capacity, size_t *count); +/* Lists durable membership Feature Set IDs in ascending numeric order. The + * cursor is an ID boundary, not a row count; sparse IDs are therefore valid. + * `capacity` is bounded by the catalog page maximum and output storage remains + * caller-owned for the duration of the call. A missing Visual Index returns + * `LARDON3D_PROJECT_DB_NOT_FOUND`; malformed arguments reset `count` to zero + * and return `LARDON3D_PROJECT_DB_INVALID_ARGUMENT`. */ +Lardon3DProjectDbResult lardon3d_project_db_list_visual_index_memberships( + Lardon3DProjectDb *database, uint64_t visual_index_id, uint64_t after_feature_set_id, + uint64_t *feature_set_ids, size_t capacity, size_t *count); +/* Returns the durable ordered membership rank at `after_feature_set_id` and + * the total membership count. These are source-membership counts, never an + * inference from the numeric Feature Set ID, and are safe for progress after + * restart. Both outputs are reset to zero on entry. A missing Visual Index + * returns `LARDON3D_PROJECT_DB_NOT_FOUND`. */ +Lardon3DProjectDbResult lardon3d_project_db_visual_index_membership_progress( + Lardon3DProjectDb *database, uint64_t visual_index_id, uint64_t after_feature_set_id, + uint64_t *completed_count, uint64_t *total_count); Lardon3DProjectDbResult lardon3d_project_db_publish_visual_index_segment( Lardon3DProjectDb *database, const Lardon3DProjectDbVisualIndexSegment *segment, const uint64_t *feature_set_ids, size_t feature_set_count, diff --git a/include/lardon3d/task_checkpoint.h b/include/lardon3d/task_checkpoint.h index c2af671..e46b8ed 100644 --- a/include/lardon3d/task_checkpoint.h +++ b/include/lardon3d/task_checkpoint.h @@ -18,6 +18,29 @@ typedef enum { LARDON3D_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE } Lardon3DTaskCheckpointResult; +/* Writes .next with the same file and parent-directory durability as a + * canonical checkpoint. The caller retains ownership of path; this API never + * replaces its canonical representation. */ +Lardon3DTaskCheckpointResult lardon3d_task_checkpoint_stage( + const char *path, + const Lardon3DTaskDurableSnapshot *snapshot +); +/* Loads the staged .next representation. It is optional so NOT_FOUND + * is a normal result for legacy v22 projects containing only . */ +Lardon3DTaskCheckpointResult lardon3d_task_checkpoint_load_staged( + const char *path, + Lardon3DTaskDurableSnapshot *snapshot, + uint32_t *format_version +); +/* Publishes a validated staged representation as canonical without removing + * .next until canonical file and directory durability have succeeded. A + * PUBLISHED_NOT_DURABLE result means the namespace changed but its directory + * sync failed; recovery must still validate the codec/version and the + * DB-stored task-summary fields before accepting either representation. */ +Lardon3DTaskCheckpointResult lardon3d_task_checkpoint_promote_staged(const char *path); +/* Removes stale staged state only after canonical publication is durable. */ +Lardon3DTaskCheckpointResult lardon3d_task_checkpoint_discard_staged(const char *path); + Lardon3DTaskCheckpointResult lardon3d_task_checkpoint_save( const char *path, const Lardon3DTaskDurableSnapshot *snapshot diff --git a/meson.build b/meson.build index c998998..c18d974 100644 --- a/meson.build +++ b/meson.build @@ -486,6 +486,45 @@ photo_quality_restart_scan = executable( dependencies: [threads, sqlite3, openssl, opencv, libraw, libexif, libpng, libdeflate], ) +# Opt-in real-data evidence runner. It is intentionally not registered with +# meson test: invocation requires explicit source roots and a fresh project +# directory, and may perform bounded but long RAW/ORB/matching work. +pre_sfm_real_execution = executable( + 'pre-sfm-real-execution', + sources: [ + 'tests/pre_sfm_real_execution.cpp', + 'src/app_state.c', + 'src/acquisition_pairing.cpp', 'src/acquisition_ingest.cpp', + 'src/acquisition_campaign.cpp', 'src/acquisition_campaign_task.cpp', + 'src/photo_quality.cpp', 'src/photo_quality_task.cpp', + 'src/raw_development.cpp', 'src/raw_development_task.cpp', + 'src/import.c', 'src/import_task.c', + 'src/image_catalog.c', 'src/image_catalog_persistent.c', 'src/image_view.c', + 'src/feature_extractor_opencv.cpp', 'src/feature_store.c', + 'src/feature_task.c', 'src/sift_task.c', 'src/precision_features.c', + 'src/visual_index.c', 'src/visual_index_task.c', + 'src/candidate_pair_gen.c', 'src/candidate_pair_task.c', + 'src/match_file.c', 'src/matcher.cpp', 'src/matcher_task.c', + 'src/geometric_verifier.cpp', 'src/geometric_verifier_task.c', + 'src/track_builder.cpp', 'src/track_builder_project.cpp', + 'src/track_builder_task.cpp', + 'src/project.c', 'src/project_db.c', 'src/project_db_sparse_sfm.c', + 'src/task.c', 'src/task_checkpoint.c', 'src/task_kind_registry.c', 'src/task_kinds.c', + 'src/task_queue.c', 'src/resource_governor.c', 'src/resource_snapshot.c', + 'src/hardware_profile.c', + ] + matcher_backend_sources, + include_directories: include_directories('include'), + c_args: [ + '-DLARDON3D_TRACK_BUILDER_TASK_AVAILABLE', + '-DLARDON3D_ACQUISITION_CAMPAIGN_TASK_AVAILABLE', + '-DLARDON3D_PHOTO_QUALITY_TASK_AVAILABLE', + '-DLARDON3D_RAW_DEVELOPMENT_TASK_AVAILABLE', + ], + dependencies: [ + threads, sqlite3, openssl, opencv, libraw, libexif, libpng, libdeflate, + ] + matcher_backend_dependencies, +) + feature_store_test = executable( 'test-feature-store', sources: [ @@ -608,7 +647,7 @@ matcher_task_test = executable( c_args: [ '-DLARDON3D_PROJECT_DB_TESTING', '-DLARDON3D_MATCHER_TASK_TESTING', - ], + ] + (matcher_vulkan_enabled ? ['-DLARDON3D_MATCHER_TASK_VULKAN'] : []), include_directories: include_directories('include'), dependencies: [threads, sqlite3, openssl, opencv] + matcher_backend_dependencies, ) diff --git a/src/acquisition_campaign_task.cpp b/src/acquisition_campaign_task.cpp index 709c71e..5017ad8 100644 --- a/src/acquisition_campaign_task.cpp +++ b/src/acquisition_campaign_task.cpp @@ -457,7 +457,15 @@ bool task_reconstruct_impl( p.scanset_id, r, blob.data(), p.request_size); if (!c) return false; - *b = {run, c, destroy, finished, c}; + // This Task kind has no legacy operational-resource reconciliation. Keep + // every optional Registry hook explicit so a future binding extension + // cannot accidentally inherit non-null recovery behavior. + *b = {}; + b->callback = run; + b->userdata = c; + b->userdata_destroy = destroy; + b->finished_callback = finished; + b->finished_userdata = c; return true; } diff --git a/src/app.c b/src/app.c index 62f528b..5d417df 100644 --- a/src/app.c +++ b/src/app.c @@ -31,13 +31,21 @@ lardon3d_app_run(void) || !lardon3d_resource_policy_default( &state.hardware_profile, &resource_policy - ) - || !lardon3d_feature_opencv_configure_threads( - state.hardware_profile.logical_cpu_count - - resource_policy.system_cpu_reserve )) { return EXIT_FAILURE; } + unsigned int feature_threads = state.hardware_profile.logical_cpu_count + - resource_policy.system_cpu_reserve; + if (feature_threads > 12) { + feature_threads = 12; + } + /* OpenCV owns an internal process-wide pool. Configure it once, before + * Queue workers exist, to the empirically audited part of the host-reserved + * CPU budget. Feature Tasks request the same count from the Governor. This + * operational ceiling is not part of a fingerprint or FeatureSet identity. */ + if (!lardon3d_feature_opencv_configure_threads(feature_threads)) { + return EXIT_FAILURE; + } state.resource_governor = lardon3d_resource_governor_create( &state.hardware_profile, &resource_policy diff --git a/src/candidate_pair_gen.c b/src/candidate_pair_gen.c index 951251f..bd3f917 100644 --- a/src/candidate_pair_gen.c +++ b/src/candidate_pair_gen.c @@ -10,6 +10,8 @@ #include +#include "candidate_pair_gen_internal.h" + static Lardon3DVisualIndexResult db_result(Lardon3DProjectDbResult result) { switch (result) { case LARDON3D_PROJECT_DB_OK: @@ -29,17 +31,15 @@ static Lardon3DVisualIndexResult db_result(Lardon3DProjectDbResult result) { } } -Lardon3DVisualIndexResult lardon3d_candidate_pair_generate( +Lardon3DVisualIndexResult lardon3d_candidate_pair_compute( const char *project_path, Lardon3DProjectDb *database, uint64_t visual_index_id, uint64_t source_feature_set_id, const Lardon3DVisualIndexQueryOptions *query_options, - Lardon3DCandidatePairGenStats *stats) { - if (stats) { - stats->generated_count = 0; - stats->skipped_count = 0; - stats->queried_count = 0; + Lardon3DCandidatePairComputation *computed) { + if (computed) { + memset(computed, 0, sizeof(*computed)); } if (!project_path || !database || visual_index_id == 0 || source_feature_set_id == 0 || - !query_options || !stats || query_options->top_k == 0 || + !query_options || !computed || query_options->top_k == 0 || query_options->top_k > LARDON3D_VISUAL_INDEX_TOP_K_MAX || query_options->minimum_evidence_count > 1024 || query_options->scanset_filter > LARDON3D_VISUAL_INDEX_OTHER_SCANSETS) { @@ -63,12 +63,8 @@ Lardon3DVisualIndexResult lardon3d_candidate_pair_generate( free(candidates); return result; } - stats->queried_count = (uint32_t)result_count; - time_t now = time(NULL); - if (now < 0) { - free(candidates); - return LARDON3D_VISUAL_INDEX_IO_ERROR; - } + computed->source_feature_set_id = source_feature_set_id; + computed->queried_count = (uint32_t)result_count; for (size_t i = 0; i < result_count; ++i) { uint64_t candidate_image_id = candidates[i].image_id; if (candidate_image_id == source_set.image_id) { @@ -78,29 +74,70 @@ Lardon3DVisualIndexResult lardon3d_candidate_pair_generate( : source_set.image_id; uint64_t image_b = candidate_image_id < source_set.image_id ? source_set.image_id : candidate_image_id; + computed->proposals[computed->proposal_count++] = + (Lardon3DCandidatePairProposal){.image_id_a = image_a, .image_id_b = image_b}; + } + free(candidates); + return LARDON3D_VISUAL_INDEX_OK; +} + +Lardon3DVisualIndexResult lardon3d_candidate_pair_publish( + Lardon3DProjectDb *database, const Lardon3DCandidatePairComputation *computed, + Lardon3DCandidatePairGenStats *stats) { + if (stats) { + memset(stats, 0, sizeof(*stats)); + } + if (!database || !computed || !stats || computed->source_feature_set_id == 0 || + computed->proposal_count > LARDON3D_VISUAL_INDEX_TOP_K_MAX) { + return LARDON3D_VISUAL_INDEX_INVALID_ARGUMENT; + } + stats->queried_count = computed->queried_count; + time_t now = time(NULL); + if (now < 0) { + return LARDON3D_VISUAL_INDEX_IO_ERROR; + } + /* Publication deliberately preserves proposal order, including duplicates. + * The established find-before-create behavior therefore retains its exact + * generated/skipped accounting while the UNIQUE key remains the authority. */ + for (size_t i = 0; i < computed->proposal_count; ++i) { Lardon3DProjectDbCandidatePair pair; - Lardon3DProjectDbResult found = - lardon3d_project_db_find_candidate_pair(database, image_a, image_b, &pair); + Lardon3DProjectDbResult found = lardon3d_project_db_find_candidate_pair( + database, computed->proposals[i].image_id_a, computed->proposals[i].image_id_b, &pair); if (found == LARDON3D_PROJECT_DB_OK) { ++stats->skipped_count; } else if (found == LARDON3D_PROJECT_DB_NOT_FOUND) { - Lardon3DProjectDbResult created = - lardon3d_project_db_create_candidate_pair(database, image_a, image_b, (int64_t)now, - &pair); + Lardon3DProjectDbResult created = lardon3d_project_db_create_candidate_pair( + database, computed->proposals[i].image_id_a, computed->proposals[i].image_id_b, + (int64_t)now, &pair); if (created != LARDON3D_PROJECT_DB_OK) { - free(candidates); return db_result(created); } ++stats->generated_count; } else { - free(candidates); return db_result(found); } } - free(candidates); return LARDON3D_VISUAL_INDEX_OK; } +Lardon3DVisualIndexResult lardon3d_candidate_pair_generate( + const char *project_path, Lardon3DProjectDb *database, uint64_t visual_index_id, + uint64_t source_feature_set_id, const Lardon3DVisualIndexQueryOptions *query_options, + Lardon3DCandidatePairGenStats *stats) { + if (stats) { + memset(stats, 0, sizeof(*stats)); + } + if (!stats) { + return LARDON3D_VISUAL_INDEX_INVALID_ARGUMENT; + } + Lardon3DCandidatePairComputation computed; + Lardon3DVisualIndexResult result = lardon3d_candidate_pair_compute( + project_path, database, visual_index_id, source_feature_set_id, query_options, &computed); + return result == LARDON3D_VISUAL_INDEX_OK + ? lardon3d_candidate_pair_publish(database, &computed, stats) + : result; +} + Lardon3DVisualIndexResult lardon3d_candidate_pair_generate_batch( const char *project_path, Lardon3DProjectDb *database, uint64_t visual_index_id, uint64_t after_feature_set_id, @@ -168,4 +205,4 @@ void lardon3d_candidate_pair_generation_fingerprint( SHA256_Update(&sha, &exclude, sizeof(exclude)); } SHA256_Final(fingerprint, &sha); -} \ No newline at end of file +} diff --git a/src/candidate_pair_gen_internal.h b/src/candidate_pair_gen_internal.h new file mode 100644 index 0000000..c347370 --- /dev/null +++ b/src/candidate_pair_gen_internal.h @@ -0,0 +1,43 @@ +#ifndef LARDON3D_CANDIDATE_PAIR_GEN_INTERNAL_H +#define LARDON3D_CANDIDATE_PAIR_GEN_INTERNAL_H + +#include +#include + +#include + +typedef struct { + uint64_t image_id_a; + uint64_t image_id_b; +} Lardon3DCandidatePairProposal; + +typedef struct { + uint64_t source_feature_set_id; + uint32_t queried_count; + size_t proposal_count; + Lardon3DCandidatePairProposal proposals[LARDON3D_VISUAL_INDEX_TOP_K_MAX]; +} Lardon3DCandidatePairComputation; + +/* Computation is read-only and owns no storage beyond `computed`. This split + * lets task-local workers use private DB handles while the task owner retains + * the only Candidate Pair publication path and its canonical source order. */ +Lardon3DVisualIndexResult lardon3d_candidate_pair_compute( + const char *project_path, Lardon3DProjectDb *database, uint64_t visual_index_id, + uint64_t source_feature_set_id, const Lardon3DVisualIndexQueryOptions *query_options, + Lardon3DCandidatePairComputation *computed); + +Lardon3DVisualIndexResult lardon3d_candidate_pair_publish( + Lardon3DProjectDb *database, const Lardon3DCandidatePairComputation *computed, + Lardon3DCandidatePairGenStats *stats); + +#ifdef LARDON3D_CANDIDATE_PAIR_TASK_TESTING +/* Test-only counters observe the production callback after Registry recovery. + * They prove that a multi-CPU execution contract distributes source work to + * more than the Queue owner; production state and scientific output are not + * instrumented or altered. */ +void lardon3d_candidate_pair_task_test_reset_parallel_counters(void); +size_t lardon3d_candidate_pair_task_test_started_participants(void); +size_t lardon3d_candidate_pair_task_test_computed_work_items(void); +#endif + +#endif diff --git a/src/candidate_pair_task.c b/src/candidate_pair_task.c index 010e5f7..b9043e7 100644 --- a/src/candidate_pair_task.c +++ b/src/candidate_pair_task.c @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -11,15 +13,22 @@ #include #include +#include "candidate_pair_gen_internal.h" + enum { CANDIDATE_PAIR_MINIMUM_BATCH = 1, CANDIDATE_PAIR_MAXIMUM_BATCH = 64, - CANDIDATE_PAIR_FIXED_MEMORY = 128 * 1024, + CANDIDATE_PAIR_CPU_THREADS = 12, + CANDIDATE_PAIR_WINDOW_PER_THREAD = 2, + CANDIDATE_PAIR_WINDOW_MAX = + CANDIDATE_PAIR_CPU_THREADS * CANDIDATE_PAIR_WINDOW_PER_THREAD, + CANDIDATE_PAIR_FIXED_MEMORY = 256 * 1024, CANDIDATE_PAIR_MEMORY_PER_ITEM = 256 * 256, }; typedef struct { char project_path[PATH_MAX]; + char database_path[LARDON3D_PROJECT_DB_PATH_CAPACITY]; Lardon3DProjectDb *database; Lardon3DResourceGovernor *governor; Lardon3DProjectDbCandidatePairGenerateTask parameters; @@ -61,7 +70,147 @@ static uint64_t elapsed_ns(struct timespec begin, struct timespec end) { : UINT64_MAX; } -enum { FEATURE_SET_PAGE = 64 }; +typedef struct { + const char *project_path; + Lardon3DProjectDb *database; + uint64_t visual_index_id; + const Lardon3DVisualIndexQueryOptions *query_options; + const uint64_t *source_ids; + Lardon3DCandidatePairComputation *computations; + Lardon3DVisualIndexResult *results; + size_t source_count; + size_t first; + size_t stride; +} CandidateComputeWorker; + +#ifdef LARDON3D_CANDIDATE_PAIR_TASK_TESTING +static atomic_size_t test_started_participants; +static atomic_size_t test_computed_work_items; + +void lardon3d_candidate_pair_task_test_reset_parallel_counters(void) { + atomic_store(&test_started_participants, 0); + atomic_store(&test_computed_work_items, 0); +} + +size_t lardon3d_candidate_pair_task_test_started_participants(void) { + return atomic_load(&test_started_participants); +} + +size_t lardon3d_candidate_pair_task_test_computed_work_items(void) { + return atomic_load(&test_computed_work_items); +} +#endif + +static void *compute_worker(void *userdata) { + CandidateComputeWorker *worker = userdata; +#ifdef LARDON3D_CANDIDATE_PAIR_TASK_TESTING + atomic_fetch_add(&test_started_participants, 1); +#endif + for (size_t i = worker->first; i < worker->source_count; i += worker->stride) { +#ifdef LARDON3D_CANDIDATE_PAIR_TASK_TESTING + atomic_fetch_add(&test_computed_work_items, 1); +#endif + worker->results[i] = lardon3d_candidate_pair_compute( + worker->project_path, worker->database, worker->visual_index_id, + worker->source_ids[i], worker->query_options, &worker->computations[i]); +#ifdef LARDON3D_CANDIDATE_PAIR_TASK_TESTING + const char *failed_id = getenv("LARDON3D_TEST_CANDIDATE_PAIR_FAIL_SOURCE_ID"); + if (failed_id && strtoull(failed_id, NULL, 10) == worker->source_ids[i]) { + worker->results[i] = LARDON3D_VISUAL_INDEX_IO_ERROR; + } +#endif + } + lardon3d_project_db_close(worker->database); + worker->database = NULL; + return NULL; +} + +static unsigned int membership_progress(uint64_t completed, uint64_t total) { + if (completed >= total && total != 0) return 99U; + if (total == 0) return 0U; + /* Find floor(completed * 99 / total) without overflowing uint64_t. + * Progress has only 99 incomplete buckets, so threshold comparison is + * both exact and independent of floating-point behavior. */ + unsigned int progress = 0; + uint64_t quotient = total / 99U; + uint64_t remainder = total % 99U; + for (unsigned int candidate = 1; candidate <= 99U; ++candidate) { + uint64_t threshold = quotient * candidate + + (remainder * candidate + 98U) / 99U; + if (completed < threshold) break; + progress = candidate; + } + return progress; +} + +static bool set_membership_progress(Lardon3DTask *task, uint64_t completed, + uint64_t total, uint64_t generated) { + char message[LARDON3D_TASK_MESSAGE_CAPACITY]; + (void)snprintf(message, sizeof(message), "FS:%lu/%lu générées:%lu", + (unsigned long)completed, (unsigned long)total, + (unsigned long)generated); + return lardon3d_task_set_progress( + task, membership_progress(completed, total), message); +} + +static bool compute_window(const Lardon3DCandidatePairTaskContext *context, + uint64_t visual_index_id, + const Lardon3DVisualIndexQueryOptions *query_options, + const uint64_t *source_ids, size_t source_count, + unsigned int admitted_threads, + Lardon3DCandidatePairComputation *computations, + Lardon3DVisualIndexResult *results) { + unsigned int worker_count = admitted_threads; + if (worker_count > source_count) worker_count = (unsigned int)source_count; + CandidateComputeWorker workers[CANDIDATE_PAIR_CPU_THREADS]; + pthread_t threads[CANDIDATE_PAIR_CPU_THREADS - 1]; + size_t created = 0; + bool started = true; + for (size_t i = 0; i < source_count; ++i) { + results[i] = LARDON3D_VISUAL_INDEX_IO_ERROR; + } + for (unsigned int i = 0; i < worker_count; ++i) { + workers[i] = (CandidateComputeWorker){ + .project_path = context->project_path, + .visual_index_id = visual_index_id, + .query_options = query_options, + .source_ids = source_ids, + .computations = computations, + .results = results, + .source_count = source_count, + .first = i, + .stride = worker_count, + }; + char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY]; + if (lardon3d_project_db_open(context->database_path, + &workers[i].database, error) != + LARDON3D_PROJECT_DB_OK) { + for (unsigned int opened = 0; opened < i; ++opened) { + lardon3d_project_db_close(workers[opened].database); + workers[opened].database = NULL; + } + return false; + } + } + for (unsigned int i = 1; i < worker_count; ++i) { + if (pthread_create(&threads[created], NULL, compute_worker, &workers[i]) != 0) { + started = false; + break; + } + ++created; + } + if (worker_count > 0) (void)compute_worker(&workers[0]); + for (size_t i = 0; i < created; ++i) { + if (pthread_join(threads[i], NULL) != 0) started = false; + } + for (unsigned int i = (unsigned int)created + 1; i < worker_count; ++i) { + if (workers[i].database) { + lardon3d_project_db_close(workers[i].database); + workers[i].database = NULL; + } + } + return started; +} static bool run(Lardon3DTask *task, void *userdata) { Lardon3DCandidatePairTaskContext *context = userdata; @@ -74,85 +223,119 @@ static bool run(Lardon3DTask *task, void *userdata) { }; uint64_t total_generated = 0; - uint64_t total_feature_sets = 0; + uint64_t completed_memberships = 0; + uint64_t total_memberships = 0; + if (lardon3d_project_db_visual_index_membership_progress( + context->database, context->parameters.visual_index_id, + context->parameters.after_feature_set_id, &completed_memberships, + &total_memberships) != LARDON3D_PROJECT_DB_OK) { + return lardon3d_task_fail(task, "Comptage des memberships impossible."); + } + if (!set_membership_progress(task, completed_memberships, total_memberships, 0)) { + return false; + } +#ifdef LARDON3D_CANDIDATE_PAIR_TASK_TESTING + bool test_pause_before_batch = false; +#endif for (;;) { Lardon3DTaskExecutionContract contract; if (!lardon3d_task_execution_contract(task, &contract) || contract.batch_size < CANDIDATE_PAIR_MINIMUM_BATCH || - contract.batch_size > CANDIDATE_PAIR_MAXIMUM_BATCH) { + contract.batch_size > CANDIDATE_PAIR_MAXIMUM_BATCH || + contract.cpu_threads == 0 || + contract.cpu_threads > CANDIDATE_PAIR_CPU_THREADS) { return lardon3d_task_fail(task, "Contrat de lot Candidate Pair invalide."); } size_t processed_in_sequence = 0; uint64_t cursor = context->parameters.after_feature_set_id; - Lardon3DProjectDbFeatureSet page[FEATURE_SET_PAGE]; - size_t count = 0; - Lardon3DProjectDbResult listed = - lardon3d_project_db_list_feature_sets(context->database, cursor, page, - FEATURE_SET_PAGE, &count); - if (listed != LARDON3D_PROJECT_DB_OK) { - return lardon3d_task_fail(task, - "Liste des Feature Sets impossible."); - } struct timespec begin, end; (void)clock_gettime(CLOCK_MONOTONIC, &begin); uint64_t seq_generated = 0; - while (count > 0 && - processed_in_sequence < contract.batch_size) { - for (size_t i = 0; i < count && - processed_in_sequence < contract.batch_size; - ++i) { - if (!lardon3d_task_checkpoint(task)) { - return false; + bool exhausted = false; + while (processed_in_sequence < contract.batch_size) { + if (!lardon3d_task_checkpoint(task)) return false; + size_t remaining = contract.batch_size - processed_in_sequence; + size_t window_capacity = (size_t)contract.cpu_threads * + CANDIDATE_PAIR_WINDOW_PER_THREAD; + if (window_capacity > CANDIDATE_PAIR_WINDOW_MAX) { + window_capacity = CANDIDATE_PAIR_WINDOW_MAX; + } + if (window_capacity > remaining) window_capacity = remaining; + uint64_t source_ids[CANDIDATE_PAIR_WINDOW_MAX]; + size_t source_count = 0; + Lardon3DProjectDbResult listed = + lardon3d_project_db_list_visual_index_memberships( + context->database, context->parameters.visual_index_id, + cursor, source_ids, window_capacity, &source_count); + if (listed != LARDON3D_PROJECT_DB_OK) { + return lardon3d_task_fail(task, "Liste des memberships impossible."); + } + if (source_count == 0) { + exhausted = true; + break; + } + +#ifdef LARDON3D_CANDIDATE_PAIR_TASK_TESTING + const char *pause_before = getenv( + "LARDON3D_TEST_CANDIDATE_PAIR_PAUSE_BEFORE_BATCH"); + if (pause_before && strcmp(pause_before, "1") == 0) { + /* Keep the typed cursor at the durable prefix before this + * window. This test-only interruption models a process death + * before participant dispatch, so recovery must exercise the + * normal Registry -> Queue -> Governor callback path. */ + if (!lardon3d_task_checkpoint(task)) return false; + (void)lardon3d_task_pause(task); + test_pause_before_batch = true; + exhausted = true; + break; + } +#endif + + Lardon3DCandidatePairComputation computations[CANDIDATE_PAIR_WINDOW_MAX]; + Lardon3DVisualIndexResult results[CANDIDATE_PAIR_WINDOW_MAX]; + /* The Queue callback is one admitted CPU participant. At most + * cpu_threads-1 child threads are created, every reader owns a + * private SQLite handle, and all are joined before reservation + * release or sequence_break. */ + if (!compute_window(context, context->parameters.visual_index_id, + &query_options, source_ids, source_count, + contract.cpu_threads, computations, results)) { + return lardon3d_task_fail(task, "Workers Candidate Pair impossibles."); + } + for (size_t i = 0; i < source_count; ++i) { + if (!lardon3d_task_checkpoint(task)) return false; + if (results[i] != LARDON3D_VISUAL_INDEX_OK) { + return lardon3d_task_fail(task, + "Calcul Candidate Pair impossible."); } Lardon3DCandidatePairGenStats stats; - Lardon3DVisualIndexResult result = - lardon3d_candidate_pair_generate( - context->project_path, context->database, - context->parameters.visual_index_id, - page[i].feature_set_id, &query_options, &stats); - if (result != LARDON3D_VISUAL_INDEX_OK) { - return lardon3d_task_fail( - task, "Génération Candidate Pair impossible."); + Lardon3DVisualIndexResult published = lardon3d_candidate_pair_publish( + context->database, &computations[i], &stats); + if (published != LARDON3D_VISUAL_INDEX_OK) { + return lardon3d_task_fail(task, + "Publication Candidate Pair impossible."); } seq_generated += stats.generated_count; - cursor = page[i].feature_set_id; ++processed_in_sequence; - } - if (processed_in_sequence < contract.batch_size) { - listed = lardon3d_project_db_list_feature_sets( - context->database, cursor, page, FEATURE_SET_PAGE, - &count); - if (listed != LARDON3D_PROJECT_DB_OK) { - return lardon3d_task_fail( - task, "Liste des Feature Sets impossible."); + ++completed_memberships; + cursor = source_ids[i]; + context->parameters.after_feature_set_id = cursor; + if (!set_membership_progress(task, completed_memberships, + total_memberships, + total_generated + seq_generated)) { + return false; } } } (void)clock_gettime(CLOCK_MONOTONIC, &end); total_generated += seq_generated; - total_feature_sets += processed_in_sequence; - context->parameters.after_feature_set_id = cursor; - bool exhausted = (count == 0); - unsigned int progress = exhausted ? 100U - : (total_feature_sets == 0 ? 0U - : (unsigned int)((total_feature_sets * 99U) / - (total_feature_sets + 1U))); - - char msg[LARDON3D_TASK_MESSAGE_CAPACITY]; - (void)snprintf(msg, sizeof(msg), "FS:%lu générées:%lu", - (unsigned long)total_feature_sets, - (unsigned long)total_generated); - if (!lardon3d_task_set_progress(task, progress, msg)) { - return false; - } - (void)lardon3d_resource_governor_record_batch( context->governor, LARDON3D_RESOURCE_TASK_CPU, (size_t)seq_generated, elapsed_ns(begin, end), 0); @@ -167,7 +350,7 @@ static bool run(Lardon3DTask *task, void *userdata) { } #ifdef LARDON3D_CANDIDATE_PAIR_TASK_TESTING - bool test_pause_requested = false; + bool test_pause_requested = test_pause_before_batch; const char *pause = getenv( "LARDON3D_TEST_CANDIDATE_PAIR_PAUSE_AFTER_BATCH"); if (pause && strcmp(pause, "1") == 0) { @@ -181,6 +364,8 @@ static bool run(Lardon3DTask *task, void *userdata) { && !test_pause_requested #endif ) { + /* 100 is reserved for an observed empty suffix, never merely for + * publishing the membership that had the highest initial rank. */ return lardon3d_task_set_progress( task, 100, "Génération Candidate Pair terminée."); } @@ -209,6 +394,14 @@ make_context(const Lardon3DTaskReconstructionContext *runtime, free(context); return NULL; } + /* Private compute readers must reopen the same durable database selected + * by the runtime. Project path identifies the asset root, not the database + * basename, and publication remains owned by the Queue callback handle. */ + if (!lardon3d_project_db_copy_path(runtime->project_db, + context->database_path)) { + free(context); + return NULL; + } context->database = runtime->project_db; context->governor = runtime->resource_governor; context->parameters = *parameters; @@ -276,12 +469,36 @@ Lardon3DTask *lardon3d_project_create_candidate_pair_generate_task( Lardon3DCandidatePairTaskContext *context = make_context(&runtime, ¶meters); if (!context) return NULL; + unsigned int desired_cpu_threads = CANDIDATE_PAIR_CPU_THREADS; +#ifdef LARDON3D_CANDIDATE_PAIR_TASK_TESTING + const char *test_threads = getenv("LARDON3D_TEST_CANDIDATE_PAIR_THREADS"); + if (test_threads) { + unsigned long parsed = strtoul(test_threads, NULL, 10); + if (parsed >= 1 && parsed <= CANDIDATE_PAIR_CPU_THREADS) { + desired_cpu_threads = (unsigned int)parsed; + } + } + /* Test-only sequence shaping proves that a healthy Governor re-admits + * parallel Candidate work after several durable boundaries. Production + * keeps its canonical 1..64 resource estimate. */ + size_t test_maximum_batch = CANDIDATE_PAIR_MAXIMUM_BATCH; + const char *test_batch = getenv("LARDON3D_TEST_CANDIDATE_PAIR_MAXIMUM_BATCH"); + if (test_batch) { + unsigned long parsed = strtoul(test_batch, NULL, 10); + if (parsed >= CANDIDATE_PAIR_MINIMUM_BATCH && + parsed <= CANDIDATE_PAIR_MAXIMUM_BATCH) { + test_maximum_batch = (size_t)parsed; + } + } +#else + const size_t test_maximum_batch = CANDIDATE_PAIR_MAXIMUM_BATCH; +#endif const Lardon3DResourceEstimate estimate = { .memory_fixed_bytes = CANDIDATE_PAIR_FIXED_MEMORY, .memory_bytes_per_item = CANDIDATE_PAIR_MEMORY_PER_ITEM, .minimum_batch_size = CANDIDATE_PAIR_MINIMUM_BATCH, - .maximum_batch_size = CANDIDATE_PAIR_MAXIMUM_BATCH, - .desired_cpu_threads = 1, + .maximum_batch_size = test_maximum_batch, + .desired_cpu_threads = desired_cpu_threads, .desired_io_slots = 1, .task_class = LARDON3D_RESOURCE_TASK_CPU, }; diff --git a/src/feature_extractor_opencv.cpp b/src/feature_extractor_opencv.cpp index 0b5a9ea..fa32427 100644 --- a/src/feature_extractor_opencv.cpp +++ b/src/feature_extractor_opencv.cpp @@ -290,13 +290,19 @@ lardon3d_feature_extract_orb(const char *path, const Lardon3DFeatureExtractorPar std::vector points; cv::Mat descriptors; extractor->detectAndCompute(image, cv::noArray(), points, descriptors); - if (points.size() > parameters->max_features || - points.size() > std::numeric_limits::max() || + if (points.size() > std::numeric_limits::max() || (!points.empty() && (descriptors.rows != static_cast(points.size()) || descriptors.cols != LARDON3D_FEATURE_DESCRIPTOR_DIMENSION || descriptors.type() != CV_8UC1))) { return LARDON3D_FEATURE_EXTRACT_ERROR; } + if (points.size() > parameters->max_features) { + /* OpenCV documents nfeatures as a requested maximum but may return one extra + * point while distributing per-level quotas. The Feature Set contract is an + * exact bound, so retain OpenCV's deterministic prefix and matching rows. */ + points.resize(parameters->max_features); + descriptors = descriptors.rowRange(0, static_cast(parameters->max_features)); + } size_t count = points.size(); Lardon3DFeatureKeypoint *keypoints = count == 0 ? nullptr diff --git a/src/feature_task.c b/src/feature_task.c index 06b4a73..8747ec4 100644 --- a/src/feature_task.c +++ b/src/feature_task.c @@ -293,8 +293,12 @@ lardon3d_project_create_feature_extract_task(Lardon3DAppState *state, uint64_t i .memory_bytes_per_item = 512ULL * 1024 * 1024, .minimum_batch_size = 1, .maximum_batch_size = 1, - .desired_cpu_threads = - lardon3d_feature_opencv_thread_count(), + /* Request the audited operational ceiling, not OpenCV's + * mutable current value: Matcher temporarily sets that + * process-wide value to one inside its Queue callback. The + * Governor reduces twelve to the host-reserved CPU budget, + * matching the stable startup configuration. */ + .desired_cpu_threads = 12, .desired_io_slots = 1, .task_class = LARDON3D_RESOURCE_TASK_CPU}; Lardon3DTask *task = lardon3d_task_create_typed("Extraction de features", &estimate, diff --git a/src/geometric_verifier.cpp b/src/geometric_verifier.cpp index 3770d7c..aa13d53 100644 --- a/src/geometric_verifier.cpp +++ b/src/geometric_verifier.cpp @@ -140,6 +140,33 @@ bool estimate_fundamental(const std::vector &points_a, return true; } +bool has_minimal_support(const Lardon3DMatchFileEntry *entries, size_t count, + uint32_t feature_count_a, uint32_t feature_count_b) { + // These per-pair maps are operational scratch owned until this bounded core + // call returns. Feature Store caps each side at 8192 entries, so v2 adds at + // most 16 KiB and does not impose a new scientific dataset-size limit. + std::vector observed_a(feature_count_a, 0); + std::vector observed_b(feature_count_b, 0); + uint32_t distinct_a = 0; + uint32_t distinct_b = 0; + for (size_t index = 0; index < count; ++index) { + const auto &entry = entries[index]; + if (entry.feature_index_a >= feature_count_a || + entry.feature_index_b >= feature_count_b) + return false; + if (!observed_a[entry.feature_index_a]) { + observed_a[entry.feature_index_a] = 1; + ++distinct_a; + } + if (!observed_b[entry.feature_index_b]) { + observed_b[entry.feature_index_b] = 1; + ++distinct_b; + } + } + return distinct_a >= LARDON3D_GEOMETRIC_VERIFIER_MINIMUM_MATCHES && + distinct_b >= LARDON3D_GEOMETRIC_VERIFIER_MINIMUM_MATCHES; +} + } // namespace #ifdef LARDON3D_GEOMETRIC_VERIFIER_TESTING @@ -150,6 +177,24 @@ extern "C" void lardon3d_geometric_verifier_test_reset_estimator_calls(void) { extern "C" uint32_t lardon3d_geometric_verifier_test_estimator_calls(void) { return estimator_calls; } + +extern "C" bool lardon3d_geometric_verifier_test_has_minimal_support( + const uint32_t *indices_a, const uint32_t *indices_b, size_t count, + uint32_t feature_count_a, uint32_t feature_count_b) { + try { + if (!indices_a || !indices_b) + return false; + std::vector entries(count); + for (size_t index = 0; index < count; ++index) { + entries[index].feature_index_a = indices_a[index]; + entries[index].feature_index_b = indices_b[index]; + } + return has_minimal_support(entries.data(), count, feature_count_a, + feature_count_b); + } catch (...) { + return false; + } +} #endif extern "C" Lardon3DGeometricVerifierParameters @@ -212,15 +257,25 @@ extern "C" bool lardon3d_geometric_verifier_fingerprint_bytes( extern "C" void lardon3d_geometric_verifier_fingerprint( const Lardon3DGeometricVerifierParameters *p, unsigned char output[32]) { + (void)lardon3d_geometric_verifier_fingerprint_for_version( + p, LARDON3D_GEOMETRIC_VERIFIER_VERSION, output); +} + +extern "C" bool lardon3d_geometric_verifier_fingerprint_for_version( + const Lardon3DGeometricVerifierParameters *p, uint32_t verifier_version, + unsigned char output[32]) { if (!output) - return; + return false; std::memset(output, 0, 32); unsigned char bytes[LARDON3D_GEOMETRIC_VERIFIER_FINGERPRINT_SIZE]; if (!lardon3d_geometric_verifier_fingerprint_bytes( p, LARDON3D_GEOMETRIC_ALGORITHM_USAC_MAGSAC, - LARDON3D_GEOMETRIC_VERIFIER_VERSION, bytes)) - return; - (void)digest(bytes, sizeof(bytes), output); + verifier_version, bytes) || + (verifier_version != LARDON3D_GEOMETRIC_VERIFIER_VERSION_V1 && + verifier_version != LARDON3D_GEOMETRIC_VERIFIER_VERSION_V2 && + verifier_version != LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3)) + return false; + return digest(bytes, sizeof(bytes), output); } extern "C" uint32_t @@ -267,16 +322,31 @@ lardon3d_geometric_verifier_verify_and_publish( const char *project_path, Lardon3DProjectDb *db, uint64_t match_id, const Lardon3DGeometricVerifierParameters *p, Lardon3DProjectDbGeometricVerificationResult *result, bool *reused) { + return lardon3d_geometric_verifier_verify_and_publish_version( + project_path, db, match_id, p, LARDON3D_GEOMETRIC_VERIFIER_VERSION, + result, reused); +} + +extern "C" Lardon3DGeometricVerifierResult +lardon3d_geometric_verifier_verify_and_publish_version( + const char *project_path, Lardon3DProjectDb *db, uint64_t match_id, + const Lardon3DGeometricVerifierParameters *p, uint32_t verifier_version, + Lardon3DProjectDbGeometricVerificationResult *result, bool *reused) { if (!project_path || !db || match_id == 0 || !p || !result || !reused || - !lardon3d_geometric_verifier_parameters_valid(p)) + !lardon3d_geometric_verifier_parameters_valid(p) || + (verifier_version != LARDON3D_GEOMETRIC_VERIFIER_VERSION_V1 && + verifier_version != LARDON3D_GEOMETRIC_VERIFIER_VERSION_V2 && + verifier_version != LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3)) return LARDON3D_GEOMETRIC_VERIFIER_INVALID_ARGUMENT; *reused = false; unsigned char fingerprint[32]; - lardon3d_geometric_verifier_fingerprint(p, fingerprint); + if (!lardon3d_geometric_verifier_fingerprint_for_version( + p, verifier_version, fingerprint)) + return LARDON3D_GEOMETRIC_VERIFIER_INVALID_ARGUMENT; Lardon3DProjectDbResult found = lardon3d_project_db_find_geometric_verification_result( db, match_id, LARDON3D_GEOMETRIC_VERIFIER_FUNDAMENTAL, - LARDON3D_GEOMETRIC_VERIFIER_VERSION, fingerprint, result); + verifier_version, fingerprint, result); if (found == LARDON3D_PROJECT_DB_OK) { *reused = true; return LARDON3D_GEOMETRIC_VERIFIER_OK; @@ -334,7 +404,23 @@ lardon3d_geometric_verifier_verify_and_publish( } cv::Mat mask; cv::Mat model; - if (count >= LARDON3D_GEOMETRIC_VERIFIER_MINIMUM_MATCHES) { + // Raw Matcher rows remain immutable evidence in their original order. V2 + // and v3 measure the minimal Fundamental sample from canonical observation + // identities on both sides; neither policy deduplicates estimator input. + const bool minimal_support = + verifier_version == LARDON3D_GEOMETRIC_VERIFIER_VERSION_V1 + ? count >= LARDON3D_GEOMETRIC_VERIFIER_MINIMUM_MATCHES + : has_minimal_support(entries.data(), entries.size(), + set_a.feature_count, set_b.feature_count); + // V3 may reject only when acceptance is mathematically impossible: no + // estimator mask over N raw rows can contain min_inlier_count>N inliers. + // This is parameter-derived acceptance feasibility, not a hardware bound, + // degeneracy heuristic, or recovery path for estimator exceptions. + const bool acceptance_feasible = + verifier_version != LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3 || + count >= p->min_inlier_count; + const bool sufficient_support = minimal_support && acceptance_feasible; + if (sufficient_support) { uint32_t seed = lardon3d_geometric_verifier_seed( parent.match_asset_sha256, fingerprint); if (!estimate_fundamental(points_a, points_b, p, seed, model, mask)) @@ -382,7 +468,7 @@ lardon3d_geometric_verifier_verify_and_publish( Lardon3DProjectDbResult created = lardon3d_project_db_create_geometric_verification_result( db, match_id, LARDON3D_GEOMETRIC_VERIFIER_FUNDAMENTAL, - LARDON3D_GEOMETRIC_VERIFIER_VERSION, fingerprint, + verifier_version, fingerprint, accepted ? LARDON3D_GEOMETRIC_VERIFIED : LARDON3D_GEOMETRIC_REJECTED, inliers, bitset.data(), bitset.size(), @@ -391,7 +477,7 @@ lardon3d_geometric_verifier_verify_and_publish( Lardon3DProjectDbResult concurrent = lardon3d_project_db_find_geometric_verification_result( db, match_id, LARDON3D_GEOMETRIC_VERIFIER_FUNDAMENTAL, - LARDON3D_GEOMETRIC_VERIFIER_VERSION, fingerprint, result); + verifier_version, fingerprint, result); if (concurrent == LARDON3D_PROJECT_DB_OK) { *reused = true; return LARDON3D_GEOMETRIC_VERIFIER_OK; @@ -401,6 +487,9 @@ lardon3d_geometric_verifier_verify_and_publish( return created == LARDON3D_PROJECT_DB_OK ? LARDON3D_GEOMETRIC_VERIFIER_OK : LARDON3D_GEOMETRIC_VERIFIER_DATABASE_ERROR; + // This catch set governs the complete public C entry point: unexpected + // OpenCV/malformed estimator behavior is execution failure and can never + // escape the ABI or be converted into a persisted scientific rejection. } catch (const std::bad_alloc &) { return LARDON3D_GEOMETRIC_VERIFIER_OUT_OF_MEMORY; } catch (const cv::Exception &) { diff --git a/src/geometric_verifier_task.c b/src/geometric_verifier_task.c index 54ab35a..7238411 100644 --- a/src/geometric_verifier_task.c +++ b/src/geometric_verifier_task.c @@ -21,6 +21,7 @@ typedef struct { Lardon3DProjectDb *database; Lardon3DResourceGovernor *governor; Lardon3DProjectDbGeometricVerifierTask durable; + uint32_t verifier_version; } Lardon3DGeometricVerifierTaskContext; static void destroy_context(void *userdata) { free(userdata); } @@ -87,10 +88,14 @@ static bool process_parent(Lardon3DTask *task, parameters_from_durable(&context->durable); Lardon3DProjectDbGeometricVerificationResult result; bool reused = false; - if (lardon3d_geometric_verifier_verify_and_publish( + if (lardon3d_geometric_verifier_verify_and_publish_version( context->project_path, context->database, parent->match_result_id, - ¶meters, &result, &reused) != LARDON3D_GEOMETRIC_VERIFIER_OK) { - return lardon3d_task_fail(task, "Vérification géométrique impossible."); + ¶meters, context->verifier_version, &result, + &reused) != LARDON3D_GEOMETRIC_VERIFIER_OK) { + // Failure transition success does not imply callback success: once an + // estimate fails for this parent, processing must stop before checkpointing. + (void)lardon3d_task_fail(task, "Vérification géométrique impossible."); + return false; } #ifdef LARDON3D_GEOMETRIC_VERIFIER_TASK_TESTING const char *pause = getenv("LARDON3D_TEST_GEOMETRIC_PAUSE_AFTER_PUBLICATION"); @@ -153,6 +158,9 @@ static bool run(Lardon3DTask *task, void *userdata) { if (!process_parent(task, context, &page[index])) { return false; } + // Publication/reuse must be durable before the business cursor advances. + // Restart may repeat an already published parent, but exact identity + // reuse makes that retry converge without duplicating or guessing GVRs. context->durable.after_match_result_id = page[index].match_result_id; ++processed; } @@ -187,10 +195,24 @@ make_context(const Lardon3DTaskReconstructionContext *runtime, Lardon3DGeometricVerifierParameters parameters = parameters_from_durable(durable); unsigned char fingerprint[32]; - lardon3d_geometric_verifier_fingerprint(¶meters, fingerprint); - if (!lardon3d_geometric_verifier_parameters_valid(¶meters) || - memcmp(fingerprint, durable->parameter_fingerprint, - sizeof(fingerprint)) != 0) { + uint32_t verifier_version = 0; + if (!lardon3d_geometric_verifier_parameters_valid(¶meters)) { + return NULL; + } + // Project DB v22 intentionally has no duplicate verifier-version column in + // this task payload. The exact versioned fingerprint unambiguously recovers + // historical v1/v2 or current v3 while keeping all identities immutable. + for (uint32_t candidate = LARDON3D_GEOMETRIC_VERIFIER_VERSION_V1; + candidate <= LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3; ++candidate) { + if (lardon3d_geometric_verifier_fingerprint_for_version( + ¶meters, candidate, fingerprint) && + memcmp(fingerprint, durable->parameter_fingerprint, + sizeof(fingerprint)) == 0) { + verifier_version = candidate; + break; + } + } + if (verifier_version == 0) { return NULL; } Lardon3DGeometricVerifierTaskContext *context = calloc(1, sizeof(*context)); @@ -206,6 +228,7 @@ make_context(const Lardon3DTaskReconstructionContext *runtime, context->database = runtime->project_db; context->governor = runtime->resource_governor; context->durable = *durable; + context->verifier_version = verifier_version; return context; } @@ -265,8 +288,11 @@ Lardon3DTask *lardon3d_project_create_geometric_verifier_task( .canonicalization_version = configuration->verifier.canonicalization_version, }; - lardon3d_geometric_verifier_fingerprint(&configuration->verifier, - durable.parameter_fingerprint); + if (!lardon3d_geometric_verifier_fingerprint_for_version( + &configuration->verifier, LARDON3D_GEOMETRIC_VERIFIER_VERSION, + durable.parameter_fingerprint)) { + return NULL; + } Lardon3DTaskReconstructionContext runtime = { .project_path = state->project_path, .project_db = state->project_db, diff --git a/src/incremental_reconstruction_task.cpp b/src/incremental_reconstruction_task.cpp index d743d9b..8bdf4e8 100644 --- a/src/incremental_reconstruction_task.cpp +++ b/src/incremental_reconstruction_task.cpp @@ -678,7 +678,12 @@ extern "C" bool lardon3d_incremental_reconstruction_task_reconstruct( delete context; return false; } - *binding = {run, context, destroy_context, finished, context}; + *binding = {}; + binding->callback = run; + binding->userdata = context; + binding->userdata_destroy = destroy_context; + binding->finished_callback = finished; + binding->finished_userdata = context; return true; } catch (...) { return false; diff --git a/src/matcher.cpp b/src/matcher.cpp index d6466c6..fba5e1f 100644 --- a/src/matcher.cpp +++ b/src/matcher.cpp @@ -18,6 +18,8 @@ extern "C" { #include #include #include + +#include "matcher_internal.h" } static const char matcher_orb_str[] = "orb_bf"; @@ -540,6 +542,55 @@ extern "C" Lardon3DMatcherResult lardon3d_matcher_match_and_publish_profiled( result, profile); } +/* Only the callback owner sets this hand-off while entering the established + * publication function. thread_local prevents concurrent Tasks or tests from + * confusing operation-owned temporary files; the stage itself remains + * unpublished until the normal atomic asset/DB path consumes it. */ +static thread_local Lardon3DMatcherStagedResult *matcher_publication_stage = nullptr; + +extern "C" void lardon3d_matcher_discard_staged(Lardon3DMatcherStagedResult *staged) { + if (!staged) return; + if (staged->temporary_path[0] != '\0') { + (void)unlink(staged->temporary_path); + } + std::memset(staged, 0, sizeof(*staged)); +} + +extern "C" Lardon3DMatcherResult lardon3d_matcher_stage( + const char *project_path, + const Lardon3DProjectDbFeatureSet *feature_set_a, + const Lardon3DProjectDbFeatureSet *feature_set_b, + const Lardon3DMatcherParams *params, Lardon3DOrbVulkanBackend *backend, + Lardon3DMatcherStagedResult *staged) { + if (!project_path || !feature_set_a || !feature_set_b || !params || !staged) { + return LARDON3D_MATCHER_INVALID_ARGUMENT; + } + std::memset(staged, 0, sizeof(*staged)); + char assets[4096], matches_dir[4096]; + if (!join_path(assets, project_path, "assets") || !ensure_directory(assets) || + !join_path(matches_dir, assets, "matches") || !ensure_directory(matches_dir)) { + return LARDON3D_MATCHER_IO_ERROR; + } + int written = std::snprintf(staged->temporary_path, sizeof(staged->temporary_path), + "%s/.match-XXXXXX", matches_dir); + if (written <= 0 || static_cast(written) >= sizeof(staged->temporary_path)) { + staged->temporary_path[0] = '\0'; + return LARDON3D_MATCHER_IO_ERROR; + } + int fd = mkstemp(staged->temporary_path); + if (fd < 0 || close(fd) != 0) { + lardon3d_matcher_discard_staged(staged); + return LARDON3D_MATCHER_IO_ERROR; + } + Lardon3DMatcherResult computed = lardon3d_matcher_run_with_backend( + project_path, feature_set_a, feature_set_b, params, staged->temporary_path, + backend, &staged->stats); + if (computed != LARDON3D_MATCHER_OK) { + lardon3d_matcher_discard_staged(staged); + } + return computed; +} + extern "C" Lardon3DMatcherResult lardon3d_matcher_match_and_publish_with_backend( const char *project_path, Lardon3DProjectDb *database, @@ -607,29 +658,40 @@ extern "C" Lardon3DMatcherResult lardon3d_matcher_match_and_publish_with_backend return LARDON3D_MATCHER_IO_ERROR; } - /* Write to temp file */ + /* Write to temp file, or consume a Task-private precomputed stage. */ char tmp_path[4096]; - int n = snprintf(tmp_path, sizeof(tmp_path), "%s/.match-XXXXXX", matches_dir); - if (n <= 0 || (size_t)n >= sizeof(tmp_path)) { - return LARDON3D_MATCHER_IO_ERROR; - } - int tmp_fd = mkstemp(tmp_path); - if (tmp_fd < 0) { - return LARDON3D_MATCHER_IO_ERROR; - } - - /* matcher_run reopens the named temp with O_TRUNC. */ - if (close(tmp_fd) != 0) { - (void)unlink(tmp_path); - return LARDON3D_MATCHER_IO_ERROR; - } - Lardon3DMatcherStats stats; - Lardon3DMatcherResult run_result = lardon3d_matcher_run_with_backend( - project_path, feature_set_a, feature_set_b, params, tmp_path, backend, &stats); - if (run_result != LARDON3D_MATCHER_OK) { - unlink(tmp_path); - return run_result; + int n = 0; + if (matcher_publication_stage && matcher_publication_stage->temporary_path[0] != '\0') { + n = snprintf(tmp_path, sizeof(tmp_path), "%s", + matcher_publication_stage->temporary_path); + if (n <= 0 || (size_t)n >= sizeof(tmp_path)) { + return LARDON3D_MATCHER_IO_ERROR; + } + stats = matcher_publication_stage->stats; + matcher_publication_stage->temporary_path[0] = '\0'; + } else { + n = snprintf(tmp_path, sizeof(tmp_path), "%s/.match-XXXXXX", matches_dir); + if (n <= 0 || (size_t)n >= sizeof(tmp_path)) { + return LARDON3D_MATCHER_IO_ERROR; + } + int tmp_fd = mkstemp(tmp_path); + if (tmp_fd < 0) { + return LARDON3D_MATCHER_IO_ERROR; + } + + /* matcher_run reopens the named temp with O_TRUNC. */ + if (close(tmp_fd) != 0) { + (void)unlink(tmp_path); + return LARDON3D_MATCHER_IO_ERROR; + } + + Lardon3DMatcherResult run_result = lardon3d_matcher_run_with_backend( + project_path, feature_set_a, feature_set_b, params, tmp_path, backend, &stats); + if (run_result != LARDON3D_MATCHER_OK) { + unlink(tmp_path); + return run_result; + } } if (profile) *profile = stats; @@ -760,6 +822,27 @@ extern "C" Lardon3DMatcherResult lardon3d_matcher_match_and_publish_with_backend return LARDON3D_MATCHER_OK; } +extern "C" Lardon3DMatcherResult lardon3d_matcher_publish_staged( + const char *project_path, Lardon3DProjectDb *database, + const Lardon3DProjectDbCandidatePair *pair, + const Lardon3DProjectDbFeatureSet *feature_set_a, + const Lardon3DProjectDbFeatureSet *feature_set_b, + const Lardon3DMatcherParams *params, Lardon3DMatcherStagedResult *staged, + Lardon3DProjectDbMatchResult *result) { + if (!staged || staged->temporary_path[0] == '\0' || matcher_publication_stage) { + return LARDON3D_MATCHER_INVALID_ARGUMENT; + } + matcher_publication_stage = staged; + Lardon3DMatcherResult published = lardon3d_matcher_match_and_publish_with_backend( + project_path, database, pair, feature_set_a, feature_set_b, params, nullptr, + result, nullptr); + matcher_publication_stage = nullptr; + /* Reuse and validation failures can return before the normal publication + * path takes ownership. In all cases the operation-owned temp ends here. */ + lardon3d_matcher_discard_staged(staged); + return published; +} + extern "C" Lardon3DMatcherResult lardon3d_matcher_match_and_publish( const char *project_path, Lardon3DProjectDb *database, diff --git a/src/matcher_internal.h b/src/matcher_internal.h new file mode 100644 index 0000000..e10ac0a --- /dev/null +++ b/src/matcher_internal.h @@ -0,0 +1,45 @@ +#ifndef LARDON3D_MATCHER_INTERNAL_H +#define LARDON3D_MATCHER_INTERNAL_H + +#include + +enum { LARDON3D_MATCHER_STAGED_PATH_CAPACITY = 4096 }; + +typedef struct { + char temporary_path[LARDON3D_MATCHER_STAGED_PATH_CAPACITY]; + Lardon3DMatcherStats stats; +} Lardon3DMatcherStagedResult; + +#ifdef __cplusplus +extern "C" { +#endif + +/* Computes one pair into an operation-owned temporary Match File without + * publishing an asset or mutating Project DB. The owner must either publish + * the stage or discard it; this split is what permits deterministic parallel + * compute followed by ordered, single-owner durable publication. */ +Lardon3DMatcherResult lardon3d_matcher_stage( + const char *project_path, + const Lardon3DProjectDbFeatureSet *feature_set_a, + const Lardon3DProjectDbFeatureSet *feature_set_b, + const Lardon3DMatcherParams *params, Lardon3DOrbVulkanBackend *backend, + Lardon3DMatcherStagedResult *staged); + +/* Publishes a successful stage using the existing Match Result identity, + * reuse, repair, and atomic asset rules. This function consumes the stage on + * every return path and must be called only by the Task callback owner. */ +Lardon3DMatcherResult lardon3d_matcher_publish_staged( + const char *project_path, Lardon3DProjectDb *database, + const Lardon3DProjectDbCandidatePair *pair, + const Lardon3DProjectDbFeatureSet *feature_set_a, + const Lardon3DProjectDbFeatureSet *feature_set_b, + const Lardon3DMatcherParams *params, Lardon3DMatcherStagedResult *staged, + Lardon3DProjectDbMatchResult *result); + +void lardon3d_matcher_discard_staged(Lardon3DMatcherStagedResult *staged); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/matcher_task.c b/src/matcher_task.c index 359b15e..39aa122 100644 --- a/src/matcher_task.c +++ b/src/matcher_task.c @@ -1,5 +1,9 @@ #include +#include #include +#ifdef LARDON3D_MATCHER_TASK_TESTING +#include +#endif #include #include #include @@ -7,13 +11,20 @@ #include #include +#include #include #include +#include "matcher_vulkan_config.h" +#include "matcher_internal.h" + enum { MATCHER_TASK_PAGE_CAPACITY = LARDON3D_MATCHER_TASK_MAXIMUM_BATCH + 1, MATCHER_TASK_MEMORY_BYTES = 10 * 1024 * 1024, - MATCHER_TASK_CPU_THREADS = 12, + MATCHER_TASK_CPU_THREADS = LARDON3D_MATCHER_TASK_MAXIMUM_BATCH, + MATCHER_TASK_WINDOW_PER_THREAD = 2, + MATCHER_TASK_WINDOW_MAX = LARDON3D_MATCHER_TASK_MAXIMUM_BATCH, + MATCHER_TASK_LEGACY_CPU_THREADS = 12, }; typedef struct { @@ -24,6 +35,42 @@ typedef struct { Lardon3DProjectDbMatcherTask parameters; } Lardon3DMatcherTaskContext; +typedef struct { + Lardon3DProjectDbCandidatePair pair; + Lardon3DProjectDbFeatureSet feature_set_a; + Lardon3DProjectDbFeatureSet feature_set_b; + Lardon3DMatcherStagedResult staged; + Lardon3DMatcherResult computed; +} Lardon3DMatcherPairStage; + +typedef struct { + const Lardon3DMatcherTaskContext *context; + const Lardon3DMatcherParams *matcher; + Lardon3DOrbVulkanBackend *backend; + Lardon3DMatcherPairStage *stages; + size_t count; + size_t participant; + size_t participants; +} Lardon3DMatcherWorker; + +#ifdef LARDON3D_MATCHER_TASK_TESTING +static atomic_size_t test_vulkan_uses; +static atomic_size_t test_forced_fallbacks; + +void lardon3d_matcher_task_test_reset_backend_counters(void) { + atomic_store(&test_vulkan_uses, 0); + atomic_store(&test_forced_fallbacks, 0); +} + +size_t lardon3d_matcher_task_test_vulkan_uses(void) { + return atomic_load(&test_vulkan_uses); +} + +size_t lardon3d_matcher_task_test_forced_fallbacks(void) { + return atomic_load(&test_forced_fallbacks); +} +#endif + static void destroy_context(void *userdata) { free(userdata); } static void runtime_state(const Lardon3DMatcherTaskContext *context, @@ -51,6 +98,51 @@ static void finished_callback(const Lardon3DTask *task, void *userdata) { &context->parameters); } +static Lardon3DResourceEstimate matcher_estimate(Lardon3DMatcherTaskMode mode) { + bool vulkan = mode == LARDON3D_MATCHER_TASK_MODE_ORB_VULKAN; + return (Lardon3DResourceEstimate){ + .memory_fixed_bytes = 0, + .gpu_memory_fixed_bytes = + vulkan ? LARDON3D_ORB_VULKAN_PERMANENT_BUFFER_BYTES : 0, + .memory_bytes_per_item = MATCHER_TASK_MEMORY_BYTES, + .gpu_memory_bytes_per_item = 0, + .minimum_batch_size = LARDON3D_MATCHER_TASK_MINIMUM_BATCH, + .maximum_batch_size = LARDON3D_MATCHER_TASK_MAXIMUM_BATCH, + .desired_cpu_threads = vulkan ? 1U : MATCHER_TASK_CPU_THREADS, + .desired_gpu_slots = vulkan ? 1U : 0U, + .desired_io_slots = 1, + .task_class = LARDON3D_RESOURCE_TASK_CPU, + }; +} + +static bool estimate_equals(const Lardon3DResourceEstimate *left, + const Lardon3DResourceEstimate *right) { + return left && right && + left->memory_fixed_bytes == right->memory_fixed_bytes && + left->gpu_memory_fixed_bytes == right->gpu_memory_fixed_bytes && + left->memory_bytes_per_item == right->memory_bytes_per_item && + left->gpu_memory_bytes_per_item == right->gpu_memory_bytes_per_item && + left->minimum_batch_size == right->minimum_batch_size && + left->maximum_batch_size == right->maximum_batch_size && + left->desired_cpu_threads == right->desired_cpu_threads && + left->desired_gpu_slots == right->desired_gpu_slots && + left->desired_io_slots == right->desired_io_slots && + left->task_class == right->task_class; +} + +static Lardon3DResourceEstimate legacy_matcher_estimate(bool vulkan) { + Lardon3DResourceEstimate estimate = matcher_estimate( + vulkan ? LARDON3D_MATCHER_TASK_MODE_ORB_VULKAN + : LARDON3D_MATCHER_TASK_MODE_CPU_PARALLEL); + /* Historical checkpoints reserved the Matcher working set once as fixed + * memory. Exact reconstruction must recognize that complete old admission + * shape before converting it to the current per-pair reservation. */ + estimate.memory_fixed_bytes = MATCHER_TASK_MEMORY_BYTES; + estimate.memory_bytes_per_item = 0; + estimate.desired_cpu_threads = MATCHER_TASK_LEGACY_CPU_THREADS; + return estimate; +} + static uint64_t elapsed_ns(struct timespec begin, struct timespec end) { uint64_t seconds = end.tv_sec >= begin.tv_sec ? (uint64_t)(end.tv_sec - begin.tv_sec) : 0; @@ -83,28 +175,126 @@ static bool load_feature_sets(Lardon3DMatcherTaskContext *context, feature_set_b) == LARDON3D_PROJECT_DB_OK; } -static bool process_pair(Lardon3DTask *task, - Lardon3DMatcherTaskContext *context, - const Lardon3DProjectDbCandidatePair *pair) { - if (!lardon3d_task_checkpoint(task)) { +static bool fail_task(Lardon3DTask *task, const char *message) { + (void)lardon3d_task_fail(task, message); + /* A successful state transition is not scientific callback success. */ + return false; +} + +static bool test_fail_pair(const char *name, uint64_t candidate_pair_id) { +#ifdef LARDON3D_MATCHER_TASK_TESTING + const char *value = getenv(name); + if (value) { + char *end = NULL; + unsigned long long parsed = strtoull(value, &end, 10); + return end && *end == '\0' && parsed == candidate_pair_id; + } +#else + (void)name; + (void)candidate_pair_id; +#endif + return false; +} + +static void *compute_worker(void *userdata) { + Lardon3DMatcherWorker *worker = userdata; + for (size_t index = worker->participant; index < worker->count; + index += worker->participants) { + Lardon3DMatcherPairStage *stage = &worker->stages[index]; + if (test_fail_pair("LARDON3D_TEST_MATCHER_FAIL_COMPUTE_PAIR_ID", + stage->pair.candidate_pair_id)) { + stage->computed = LARDON3D_MATCHER_FAILED; + continue; + } + Lardon3DOrbVulkanBackend *backend = worker->backend; +#ifdef LARDON3D_MATCHER_TASK_TESTING + const char *force_fallback = getenv("LARDON3D_TEST_MATCHER_FORCE_FALLBACK"); + if (backend && force_fallback && strcmp(force_fallback, "1") == 0) { + backend = NULL; + atomic_fetch_add(&test_forced_fallbacks, 1); + } +#endif + stage->computed = lardon3d_matcher_stage( + worker->context->project_path, &stage->feature_set_a, + &stage->feature_set_b, worker->matcher, backend, &stage->staged); +#ifdef LARDON3D_MATCHER_TASK_TESTING + if (stage->staged.stats.used_vulkan) { + atomic_fetch_add(&test_vulkan_uses, 1); + } +#endif + } + return NULL; +} + +static void discard_window(Lardon3DMatcherPairStage *stages, size_t count) { + for (size_t index = 0; index < count; ++index) { + lardon3d_matcher_discard_staged(&stages[index].staged); + } +} + +static bool compute_window(const Lardon3DMatcherTaskContext *context, + const Lardon3DMatcherParams *matcher, + Lardon3DMatcherPairStage *stages, size_t count, + unsigned int cpu_threads) { + size_t participants = count < cpu_threads ? count : cpu_threads; + pthread_t children[MATCHER_TASK_WINDOW_MAX - 1]; + Lardon3DMatcherWorker workers[MATCHER_TASK_WINDOW_MAX]; + size_t launched = 0; + for (size_t participant = 1; participant < participants; ++participant) { + workers[participant] = (Lardon3DMatcherWorker){ + .context = context, + .matcher = matcher, + .backend = cpu_threads == 1 ? context->orb_vulkan_backend : NULL, + .stages = stages, + .count = count, + .participant = participant, + .participants = participants, + }; + if (pthread_create(&children[launched], NULL, compute_worker, + &workers[participant]) != 0) { + break; + } + ++launched; + } + if (launched + 1 != participants) { + for (size_t index = 0; index < launched; ++index) { + (void)pthread_join(children[index], NULL); + } return false; } - Lardon3DProjectDbFeatureSet feature_set_a; - Lardon3DProjectDbFeatureSet feature_set_b; - if (!load_feature_sets(context, pair, &feature_set_a, &feature_set_b)) { - return lardon3d_task_fail(task, "Feature Sets du Matcher introuvables."); - } - Lardon3DMatcherParams matcher = { - .kind = (Lardon3DMatcherKind)context->parameters.matcher_kind, - .ratio_threshold = context->parameters.ratio_threshold, + workers[0] = (Lardon3DMatcherWorker){ + .context = context, + .matcher = matcher, + .backend = cpu_threads == 1 ? context->orb_vulkan_backend : NULL, + .stages = stages, + .count = count, + .participant = 0, + .participants = participants, }; + (void)compute_worker(&workers[0]); + bool joined = true; + for (size_t index = 0; index < launched; ++index) { + if (pthread_join(children[index], NULL) != 0) { + joined = false; + } + } + return joined; +} + +static bool publish_pair(Lardon3DTask *task, + Lardon3DMatcherTaskContext *context, + const Lardon3DMatcherParams *matcher, + Lardon3DMatcherPairStage *stage) { + if (test_fail_pair("LARDON3D_TEST_MATCHER_FAIL_PUBLISH_PAIR_ID", + stage->pair.candidate_pair_id)) { + return fail_task(task, "Publication Matcher injectée impossible."); + } Lardon3DProjectDbMatchResult result; - if (lardon3d_matcher_match_and_publish_with_backend( - context->project_path, context->database, pair, &feature_set_a, - &feature_set_b, &matcher, context->orb_vulkan_backend, &result, - NULL) != LARDON3D_MATCHER_OK) { - return lardon3d_task_fail(task, - "Matching de la Candidate Pair impossible."); + if (lardon3d_matcher_publish_staged( + context->project_path, context->database, &stage->pair, + &stage->feature_set_a, &stage->feature_set_b, matcher, &stage->staged, + &result) != LARDON3D_MATCHER_OK) { + return fail_task(task, "Matching de la Candidate Pair impossible."); } #ifdef LARDON3D_MATCHER_TASK_TESTING const char *pause = getenv("LARDON3D_TEST_MATCHER_PAUSE_AFTER_PUBLICATION"); @@ -144,7 +334,7 @@ static bool run(Lardon3DTask *task, void *userdata) { if (!lardon3d_task_execution_contract(task, &contract) || contract.batch_size < LARDON3D_MATCHER_TASK_MINIMUM_BATCH || contract.batch_size > LARDON3D_MATCHER_TASK_MAXIMUM_BATCH) { - return lardon3d_task_fail(task, "Contrat de lot Matcher invalide."); + return fail_task(task, "Contrat de lot Matcher invalide."); } Lardon3DProjectDbCandidatePair page[MATCHER_TASK_PAGE_CAPACITY]; @@ -153,7 +343,7 @@ static bool run(Lardon3DTask *task, void *userdata) { if (lardon3d_project_db_list_candidate_pairs( context->database, context->parameters.after_candidate_pair_id, page, page_capacity, &count) != LARDON3D_PROJECT_DB_OK) { - return lardon3d_task_fail(task, "Pagination Candidate Pair impossible."); + return fail_task(task, "Pagination Candidate Pair impossible."); } if (count == 0) { return lardon3d_task_set_progress(task, 100, "Matching terminé."); @@ -164,13 +354,63 @@ static bool run(Lardon3DTask *task, void *userdata) { struct timespec begin; struct timespec end; (void)clock_gettime(CLOCK_MONOTONIC, &begin); - for (size_t index = 0; index < batch_count; ++index) { - if (!process_pair(task, context, &page[index])) { - return false; + if (contract.cpu_threads == 0 || contract.cpu_threads > MATCHER_TASK_CPU_THREADS) { + return fail_task(task, "Contrat CPU Matcher invalide."); + } + unsigned int previous_opencv_threads = lardon3d_feature_opencv_thread_count(); + if (!lardon3d_feature_opencv_configure_threads(1)) { + return fail_task(task, "Configuration OpenCV Matcher impossible."); + } + size_t processed_in_batch = 0; + bool batch_ok = true; + Lardon3DMatcherParams matcher = { + .kind = (Lardon3DMatcherKind)context->parameters.matcher_kind, + .ratio_threshold = context->parameters.ratio_threshold, + }; + while (processed_in_batch < batch_count && batch_ok) { + size_t remaining = batch_count - processed_in_batch; + size_t window_count = (size_t)contract.cpu_threads * MATCHER_TASK_WINDOW_PER_THREAD; + if (window_count > MATCHER_TASK_WINDOW_MAX) window_count = MATCHER_TASK_WINDOW_MAX; + if (window_count > remaining) window_count = remaining; + Lardon3DMatcherPairStage stages[MATCHER_TASK_WINDOW_MAX] = {0}; + for (size_t index = 0; index < window_count; ++index) { + stages[index].pair = page[processed_in_batch + index]; + if (!load_feature_sets(context, &stages[index].pair, + &stages[index].feature_set_a, + &stages[index].feature_set_b)) { + batch_ok = false; + break; + } } - context->parameters.after_candidate_pair_id = - page[index].candidate_pair_id; - ++total_processed; + /* The Queue callback is one admitted compute participant. At most + * cpu_threads-1 children compute private stages, all are joined before + * ordered publication and before reservation release/sequence_break. */ + if (batch_ok && !compute_window(context, &matcher, stages, window_count, + contract.cpu_threads)) { + batch_ok = false; + } + for (size_t index = 0; index < window_count && batch_ok; ++index) { + if (!lardon3d_task_checkpoint(task) || + stages[index].computed != LARDON3D_MATCHER_OK || + !publish_pair(task, context, &matcher, &stages[index])) { + batch_ok = false; + break; + } + /* Cursor movement follows only the durable, ascending publication + * prefix. A failed stage and every later stage remain unpublished. */ + context->parameters.after_candidate_pair_id = stages[index].pair.candidate_pair_id; + ++processed_in_batch; + ++total_processed; + } + discard_window(stages, window_count); + } + (void)lardon3d_feature_opencv_configure_threads(previous_opencv_threads); + if (!batch_ok) { + Lardon3DTaskSnapshot snapshot; + if (lardon3d_task_snapshot(task, &snapshot) && snapshot.state != TASK_FAILED) { + return fail_task(task, "Calcul Matcher parallèle impossible."); + } + return false; } (void)clock_gettime(CLOCK_MONOTONIC, &end); (void)lardon3d_resource_governor_record_batch( @@ -180,7 +420,7 @@ static bool run(Lardon3DTask *task, void *userdata) { bool exhausted = count <= contract.batch_size; unsigned int progress = exhausted ? 100U : 99U; if (!checkpoint_batch(task, context, progress, total_processed)) { - return lardon3d_task_fail(task, "Checkpoint Matcher impossible."); + return fail_task(task, "Checkpoint Matcher impossible."); } if (exhausted) { return lardon3d_task_set_progress(task, 100, "Matching terminé."); @@ -275,10 +515,33 @@ bool lardon3d_matcher_task_reconstruct( if (!valid_configuration(&configuration)) { return false; } + const Lardon3DResourceEstimate cpu = matcher_estimate( + LARDON3D_MATCHER_TASK_MODE_CPU_PARALLEL); + const Lardon3DResourceEstimate vulkan = + matcher_estimate(LARDON3D_MATCHER_TASK_MODE_ORB_VULKAN); + const Lardon3DResourceEstimate legacy_cpu = legacy_matcher_estimate(false); + const Lardon3DResourceEstimate legacy_vulkan = legacy_matcher_estimate(true); + bool current_cpu = estimate_equals(&snapshot->estimate, &cpu); + bool current_vulkan = estimate_equals(&snapshot->estimate, &vulkan); + bool historical_cpu = estimate_equals(&snapshot->estimate, &legacy_cpu); + bool historical_vulkan = estimate_equals(&snapshot->estimate, &legacy_vulkan); + bool vulkan_mode = current_vulkan || historical_vulkan; + if ((!current_cpu && !current_vulkan && !historical_cpu && + !historical_vulkan) || + (vulkan_mode && configuration.matcher.kind != LARDON3D_MATCHER_ORB_BF)) { + return false; + } Lardon3DMatcherTaskContext *context = make_context(runtime, ¶meters); if (!context) { return false; } + /* Exact whole-estimate signatures select the operational mode. This avoids + * guessing backend identity from one field and rejects neighboring malformed + * snapshots. A missing backend after restart remains the already validated + * exact CPU fallback, while the immutable GPU reservation is retained. */ + if (!vulkan_mode) { + context->orb_vulkan_backend = NULL; + } *binding = (Lardon3DTaskKindBinding){ .callback = run, .userdata = context, @@ -289,15 +552,23 @@ bool lardon3d_matcher_task_reconstruct( return true; } -Lardon3DTask *lardon3d_project_create_matcher_task( +Lardon3DTask *lardon3d_project_create_matcher_task_with_mode( Lardon3DAppState *state, - const Lardon3DMatcherTaskConfiguration *configuration, uint64_t *task_id) { + const Lardon3DMatcherTaskConfiguration *configuration, + Lardon3DMatcherTaskMode mode, uint64_t *task_id) { if (task_id) { *task_id = 0; } if (!state || !state->project_loaded || !state->project_db || !state->resource_governor || !task_id || - !valid_configuration(configuration)) { + !valid_configuration(configuration) || + (mode != LARDON3D_MATCHER_TASK_MODE_CPU_PARALLEL && + mode != LARDON3D_MATCHER_TASK_MODE_ORB_VULKAN) || + (mode == LARDON3D_MATCHER_TASK_MODE_ORB_VULKAN && + (!LARDON3D_HAVE_VULKAN || + configuration->matcher.kind != LARDON3D_MATCHER_ORB_BF || + !state->hardware_profile.gpu_available || + !state->orb_vulkan_backend))) { return NULL; } uint64_t id = 0; @@ -323,25 +594,36 @@ Lardon3DTask *lardon3d_project_create_matcher_task( .resource_governor = state->resource_governor, .orb_vulkan_backend = state->orb_vulkan_backend, }; + bool vulkan_mode = mode == LARDON3D_MATCHER_TASK_MODE_ORB_VULKAN; Lardon3DMatcherTaskContext *context = make_context(&runtime, ¶meters); if (!context) { return NULL; } - bool may_use_vulkan = configuration->matcher.kind == LARDON3D_MATCHER_ORB_BF && - state->orb_vulkan_backend && - state->hardware_profile.gpu_available; - const Lardon3DResourceEstimate estimate = { - .memory_fixed_bytes = MATCHER_TASK_MEMORY_BYTES, - .gpu_memory_fixed_bytes = may_use_vulkan - ? LARDON3D_ORB_VULKAN_PERMANENT_BUFFER_BYTES - : 0, - .minimum_batch_size = LARDON3D_MATCHER_TASK_MINIMUM_BATCH, - .maximum_batch_size = LARDON3D_MATCHER_TASK_MAXIMUM_BATCH, - .desired_cpu_threads = MATCHER_TASK_CPU_THREADS, - .desired_gpu_slots = may_use_vulkan ? 1U : 0U, - .desired_io_slots = 1, - .task_class = LARDON3D_RESOURCE_TASK_CPU, - }; + /* Execution mode is fixed before admission. The Governor may reduce a + * parallel task to one CPU thread, but that CPU-only task still must not use + * Vulkan without the GPU resources declared by its immutable estimate. */ + if (!vulkan_mode) { + context->orb_vulkan_backend = NULL; + } + /* Each staged pair can retain the full bounded Matcher working set until + * ordered publication. The selected immutable estimate covers the entire + * window and never limits scientific dataset cardinality. */ + Lardon3DResourceEstimate estimate = matcher_estimate(mode); +#ifdef LARDON3D_MATCHER_TASK_TESTING + /* Tests may reduce CPU fan-out without selecting a backend. Vulkan remains + * reachable only through the explicit public mode selector above. */ + if (!vulkan_mode) { + const char *test_threads = getenv("LARDON3D_TEST_MATCHER_CPU_THREADS"); + if (test_threads) { + char *end = NULL; + unsigned long parsed = strtoul(test_threads, &end, 10); + if (end && *end == '\0' && parsed >= 1 && + parsed <= MATCHER_TASK_CPU_THREADS) { + estimate.desired_cpu_threads = (unsigned int)parsed; + } + } + } +#endif Lardon3DTask *task = lardon3d_task_create_typed( "Matching Candidate Pairs", &estimate, LARDON3D_MATCHER_TASK_KIND, LARDON3D_MATCHER_TASK_KIND_VERSION, run, context, destroy_context); @@ -356,14 +638,23 @@ Lardon3DTask *lardon3d_project_create_matcher_task( return task; } -bool lardon3d_project_enqueue_matcher_task( +Lardon3DTask *lardon3d_project_create_matcher_task( Lardon3DAppState *state, const Lardon3DMatcherTaskConfiguration *configuration, uint64_t *task_id) { + return lardon3d_project_create_matcher_task_with_mode( + state, configuration, LARDON3D_MATCHER_TASK_MODE_CPU_PARALLEL, task_id); +} + +bool lardon3d_project_enqueue_matcher_task_with_mode( + Lardon3DAppState *state, + const Lardon3DMatcherTaskConfiguration *configuration, + Lardon3DMatcherTaskMode mode, uint64_t *task_id) { if (!state || !state->task_queue) { return false; } Lardon3DTask *task = - lardon3d_project_create_matcher_task(state, configuration, task_id); + lardon3d_project_create_matcher_task_with_mode(state, configuration, mode, + task_id); if (!task) { return false; } @@ -373,3 +664,10 @@ bool lardon3d_project_enqueue_matcher_task( } return true; } + +bool lardon3d_project_enqueue_matcher_task( + Lardon3DAppState *state, + const Lardon3DMatcherTaskConfiguration *configuration, uint64_t *task_id) { + return lardon3d_project_enqueue_matcher_task_with_mode( + state, configuration, LARDON3D_MATCHER_TASK_MODE_CPU_PARALLEL, task_id); +} diff --git a/src/photo_quality.cpp b/src/photo_quality.cpp index b69adda..75de0e9 100644 --- a/src/photo_quality.cpp +++ b/src/photo_quality.cpp @@ -33,6 +33,7 @@ constexpr double kSobelLowTextureMagnitude = 0.035; // Low-gradient normalized * dimensions. This is parser resource admission, not a scientific image-size * limit: every byte read or skipped consumes the same finite work budget. */ constexpr uint64_t kJpegStructuralByteLimit = 64ULL * 1024ULL * 1024ULL; +constexpr size_t kJpegMaximumContainerImages = 8; bool jpeg_dimensions(const char *path, uint32_t &width, uint32_t &height) { std::FILE *file = std::fopen(path, "rb"); @@ -63,101 +64,146 @@ bool jpeg_dimensions(const char *path, uint32_t &width, uint32_t &height) { structural_bytes += count; return true; }; - if (read_byte() != 0xff || read_byte() != 0xd8) { - close(); - return false; - } bool have_dimensions = false; - bool in_entropy = false; - int marker = -1; - for (;;) { - if (marker < 0) { - int prefix = read_byte(); - if (in_entropy) { - while (prefix != EOF) { + bool primary_has_mpf = false; + bool soi_consumed = false; + for (size_t image_index = 0; image_index < kJpegMaximumContainerImages; ++image_index) { + if (!soi_consumed && (read_byte() != 0xff || read_byte() != 0xd8)) { + close(); + return false; + } + soi_consumed = false; + bool image_has_dimensions = false; + bool image_has_mpf = false; + bool in_entropy = false; + int marker = -1; + for (;;) { + if (marker < 0) { + int prefix = read_byte(); + if (in_entropy) { + while (prefix != EOF) { + if (prefix != 0xff) { + prefix = read_byte(); + continue; + } + marker = read_byte(); + while (marker == 0xff) + marker = read_byte(); + if (marker == 0x00 || (marker >= 0xd0 && marker <= 0xd7)) { + prefix = read_byte(); + continue; + } + break; + } + } else { if (prefix != 0xff) { - prefix = read_byte(); - continue; + close(); + return false; } marker = read_byte(); - while (marker == 0xff) - marker = read_byte(); - if (marker == 0x00 || (marker >= 0xd0 && marker <= 0xd7)) { - prefix = read_byte(); - continue; - } - break; } - } else { - if (prefix != 0xff) { + } + while (marker == 0xff) + marker = read_byte(); + if (marker == EOF || marker == 0x00 || marker == 0xd8 || + (!in_entropy && marker >= 0xd0 && marker <= 0xd7)) { + close(); + return false; + } + in_entropy = false; + if (marker == 0xd9) + break; + if (marker == 0x01) { + marker = -1; + continue; + } + const int high = read_byte(); + const int low = read_byte(); + if (high == EOF || low == EOF) { + close(); + return false; + } + const unsigned length = (static_cast(high) << 8) | + static_cast(low); + if (length < 2) { + close(); + return false; + } + if (marker == 0xe2 && length >= 6) { + unsigned char identifier[4]; + if (!read_exact(identifier, sizeof(identifier))) { + close(); + return false; + } + image_has_mpf = image_has_mpf || + std::memcmp(identifier, "MPF\0", sizeof(identifier)) == 0; + if (!skip(length - 6)) { + close(); + return false; + } + } else { + const bool sof = (marker >= 0xc0 && marker <= 0xcf && marker != 0xc4 && + marker != 0xc8 && marker != 0xcc); + if (sof) { + unsigned char header[6]; + if (length < 8 || !read_exact(header, sizeof(header))) { + close(); + return false; + } + const uint32_t image_height = (static_cast(header[1]) << 8) | header[2]; + const uint32_t image_width = (static_cast(header[3]) << 8) | header[4]; + if (image_width == 0 || image_height == 0 || image_has_dimensions || + !skip(length - 8)) { + close(); + return false; + } + if (image_index == 0) { + width = image_width; + height = image_height; + have_dimensions = true; + } + image_has_dimensions = true; + } else if (!skip(length - 2)) { close(); return false; } - marker = read_byte(); } - } - while (marker == 0xff) - marker = read_byte(); - if (marker == EOF || marker == 0x00 || marker == 0xd8 || - (!in_entropy && marker >= 0xd0 && marker <= 0xd7)) { - close(); - return false; - } - in_entropy = false; - if (marker == 0xd9) { - const int trailing = read_byte(); - close(); - return have_dimensions && trailing == EOF; - } - if (marker == 0x01) { + if (marker == 0xda) { + if (!image_has_dimensions) { + close(); + return false; + } + in_entropy = true; + } marker = -1; - continue; } - int high = read_byte(); - int low = read_byte(); - if (high == EOF || low == EOF) { + if (image_index == 0) + primary_has_mpf = image_has_mpf; + + /* The frozen MPF contract treats physical bytes after primary EOI as a + * bounded container, never as pixels: only zero gaps, structurally valid + * secondary JPEGs, and a zero-only trailer are accepted. */ + int trailing = read_byte(); + while (trailing == 0) + trailing = read_byte(); + if (trailing == EOF) { + const bool valid = have_dimensions && std::ferror(file) == 0; + close(); + return valid; + } + if (!primary_has_mpf || trailing != 0xff) { close(); return false; } - const unsigned length = (static_cast(high) << 8) | - static_cast(low); - if (length < 2) { - close(); - return false; - } - const bool sof = (marker >= 0xc0 && marker <= 0xcf && marker != 0xc4 && - marker != 0xc8 && marker != 0xcc); - if (sof) { - unsigned char header[6]; - if (length < 8 || !read_exact(header, sizeof(header))) { - close(); - return false; - } - height = (static_cast(header[1]) << 8) | header[2]; - width = (static_cast(header[3]) << 8) | header[4]; - if (width == 0 || height == 0 || have_dimensions || - !skip(length - 8)) { - close(); - return false; - } - have_dimensions = true; - } else if (!skip(length - 2)) { - close(); - return false; - } - if (marker == 0xda) { - if (!have_dimensions) { - close(); - return false; - } - in_entropy = true; - } - marker = -1; - if (std::ferror(file)) { + const int soi = read_byte(); + if (soi != 0xd8) { close(); return false; } + soi_consumed = true; } + close(); + return false; } int reduced_grayscale_flag(uint32_t maximum_dimension) { diff --git a/src/photo_quality_task.cpp b/src/photo_quality_task.cpp index 5b62259..89d6263 100644 --- a/src/photo_quality_task.cpp +++ b/src/photo_quality_task.cpp @@ -202,7 +202,12 @@ extern "C" bool lardon3d_photo_quality_task_reconstruct(const Lardon3DTaskDurabl auto *context = make_context(rt->project_path, rt->project_db, rt->resource_governor, persisted.scanset_id, request, blob.data(), persisted.request_size); if (!context) return false; - *binding = {run, context, destroy, finished, context}; + *binding = {}; + binding->callback = run; + binding->userdata = context; + binding->userdata_destroy = destroy; + binding->finished_callback = finished; + binding->finished_userdata = context; return true; } catch (...) { return false; } } diff --git a/src/project.c b/src/project.c index 64be368..291e6d0 100644 --- a/src/project.c +++ b/src/project.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -666,6 +667,41 @@ static bool checkpoint_paths(const Lardon3DAppState *state, uint64_t task_id, join_path(absolute, PATH_MAX, state->project_path, relative); } +static int checkpoint_lock_acquire(const char *checkpoint_path) { + char lock_path[PATH_MAX]; + int written = snprintf(lock_path, sizeof(lock_path), "%s.lock", checkpoint_path); + if (written < 0 || (size_t)written >= sizeof(lock_path)) { + return -1; + } + int lock = open(lock_path, O_CREAT | O_RDWR | O_CLOEXEC, 0600); + if (lock < 0 || flock(lock, LOCK_EX) != 0) { + if (lock >= 0) { + (void)close(lock); + } + return -1; + } + return lock; +} + +#ifdef LARDON3D_CHECKPOINT_TESTING +static void checkpoint_test_rendezvous(const char *variable) { + const char *value = getenv(variable); + if (!value || !value[0]) { + return; + } + char *end = NULL; + errno = 0; + long descriptor = strtol(value, &end, 10); + if (errno != 0 || !end || *end != '\0' || descriptor < 0 || descriptor > INT_MAX) { + return; + } + unsigned char token = 1; + if (write((int)descriptor, &token, sizeof(token)) == (ssize_t)sizeof(token)) { + (void)read((int)descriptor, &token, sizeof(token)); + } +} +#endif + static Lardon3DProjectTaskCheckpointResult checkpoint_task_internal(Lardon3DAppState *state, const Lardon3DTask *task, const char *image_import_source, uint64_t image_import_scanset_id, @@ -695,23 +731,40 @@ checkpoint_task_internal(Lardon3DAppState *state, const Lardon3DTask *task, if (!checkpoint_paths(state, snapshot.id, relative, absolute)) { return LARDON3D_PROJECT_TASK_CHECKPOINT_IO_ERROR; } - Lardon3DTaskCheckpointResult saved = lardon3d_task_checkpoint_save(absolute, &snapshot); - if (saved == LARDON3D_TASK_CHECKPOINT_INVALID) { - return LARDON3D_PROJECT_TASK_CHECKPOINT_INVALID_TASK; - } - if (saved != LARDON3D_TASK_CHECKPOINT_OK && - saved != LARDON3D_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE) { + /* Serialize the fixed .next slot per task through stage, DB commit, and + * promotion. The advisory lock is process-owned and is released by kernel + * close on crash, so it cannot become a durable recovery dependency. */ + int checkpoint_lock = checkpoint_lock_acquire(absolute); + if (checkpoint_lock < 0) { return LARDON3D_PROJECT_TASK_CHECKPOINT_IO_ERROR; } +#define CHECKPOINT_RETURN(value) do { (void)close(checkpoint_lock); return (value); } while (0) + /* DB and filesystem durability are independent: changing their write order + * cannot make S1 atomic. Keep .next durable across the DB commit window so + * recovery always has an explicitly published representation to compare. */ + Lardon3DTaskCheckpointResult saved = lardon3d_task_checkpoint_stage(absolute, &snapshot); + if (saved == LARDON3D_TASK_CHECKPOINT_INVALID) { + CHECKPOINT_RETURN(LARDON3D_PROJECT_TASK_CHECKPOINT_INVALID_TASK); + } + /* S1 must be directory-durable before SQLite may publish S1. A staged + * namespace with an uncertain parent-directory sync is not a safe recovery + * representation, even when its file descriptor itself was synced. */ + if (saved == LARDON3D_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE) { + CHECKPOINT_RETURN(LARDON3D_PROJECT_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE); + } + if (saved != LARDON3D_TASK_CHECKPOINT_OK) { + CHECKPOINT_RETURN(LARDON3D_PROJECT_TASK_CHECKPOINT_IO_ERROR); + } struct timespec now; if (clock_gettime(CLOCK_REALTIME, &now) != 0) { - return LARDON3D_PROJECT_TASK_CHECKPOINT_DB_ERROR; + CHECKPOINT_RETURN(LARDON3D_PROJECT_TASK_CHECKPOINT_DB_ERROR); } Lardon3DProjectDbCheckpoint checkpoint = { .format_version = LARDON3D_TASK_CHECKPOINT_VERSION, - .durability = saved == LARDON3D_TASK_CHECKPOINT_OK - ? LARDON3D_DB_CHECKPOINT_DURABLE - : LARDON3D_DB_CHECKPOINT_PUBLISHED_NOT_DURABLE, + /* A fully synced .next is the durable checkpoint representation during + * the DB-to-canonical handoff, even though checkpoint.path names the + * legacy canonical location for v22 compatibility. */ + .durability = LARDON3D_DB_CHECKPOINT_DURABLE, .updated_at = now.tv_sec, }; (void)snprintf(checkpoint.path, sizeof(checkpoint.path), "%s", relative); @@ -759,14 +812,26 @@ checkpoint_task_internal(Lardon3DAppState *state, const Lardon3DTask *task, : lardon3d_project_db_record_task(state->project_db, &snapshot, task_kind, task_kind_version, &checkpoint, now.tv_sec); if (recorded == LARDON3D_PROJECT_DB_BUSY) { - return LARDON3D_PROJECT_TASK_CHECKPOINT_DB_BUSY; + CHECKPOINT_RETURN(LARDON3D_PROJECT_TASK_CHECKPOINT_DB_BUSY); } if (recorded != LARDON3D_PROJECT_DB_OK) { - return LARDON3D_PROJECT_TASK_CHECKPOINT_DB_ERROR; + CHECKPOINT_RETURN(LARDON3D_PROJECT_TASK_CHECKPOINT_DB_ERROR); } - return saved == LARDON3D_TASK_CHECKPOINT_OK +#ifdef LARDON3D_CHECKPOINT_TESTING + /* Deterministic crash boundary: production recovery must select .next after + * the SQLite FULL transaction is durable but before canonical promotion. */ + const char *stop_after_db_commit = getenv("LARDON3D_TEST_PROJECT_CHECKPOINT_AFTER_DB_COMMIT"); + if (stop_after_db_commit && strcmp(stop_after_db_commit, "1") == 0) { + CHECKPOINT_RETURN(LARDON3D_PROJECT_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE); + } +#endif + Lardon3DTaskCheckpointResult promoted = lardon3d_task_checkpoint_promote_staged(absolute); + Lardon3DProjectTaskCheckpointResult result = promoted == LARDON3D_TASK_CHECKPOINT_OK ? LARDON3D_PROJECT_TASK_CHECKPOINT_OK : LARDON3D_PROJECT_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE; +#undef CHECKPOINT_RETURN + (void)close(checkpoint_lock); + return result; } Lardon3DProjectTaskCheckpointResult lardon3d_project_checkpoint_task(Lardon3DAppState *state, @@ -933,26 +998,96 @@ Lardon3DProjectDbResult lardon3d_project_list_recoverable(Lardon3DAppState *stat strcmp(relative, tasks[index].checkpoint.path) != 0) { entry->status = LARDON3D_PROJECT_RECOVERY_INVALID_CHECKPOINT; } else { + /* Writer and recovery take this lock before touching SQLite. Recovery + * must hold it across staged validation and promotion: otherwise a + * writer could replace .next after it matched the DB-stored task + * summary fields. + * Reload the DB row only after acquiring the lock, because the paged + * row above may have become stale while recovery waited for a writer. */ +#ifdef LARDON3D_CHECKPOINT_TESTING + /* Test-only rendezvous proves that the paged row is never used after + * another writer advances it before recovery obtains .chk.lock. */ + checkpoint_test_rendezvous("LARDON3D_TEST_RECOVERY_BEFORE_CHECKPOINT_LOCK_FD"); +#endif + int checkpoint_lock = checkpoint_lock_acquire(absolute); + if (checkpoint_lock < 0) { + entry->status = LARDON3D_PROJECT_RECOVERY_CHECKPOINT_IO_ERROR; + cursor = entry->task_id; + ++*count; + continue; + } + Lardon3DProjectDbTask durable_task; + Lardon3DProjectDbResult loaded_task = + lardon3d_project_db_load_task(state->project_db, entry->task_id, &durable_task); + if (loaded_task != LARDON3D_PROJECT_DB_OK) { + (void)close(checkpoint_lock); + return loaded_task; + } + /* Pagination is only a candidate discovery mechanism. A writer may + * have made this task terminal while recovery waited, so do not emit a + * recoverable entry unless the locked, authoritative row is pending. */ + if (durable_task.recovery_state != TASK_PENDING || !durable_task.has_checkpoint || + !durable_task.has_task_kind || strcmp(relative, durable_task.checkpoint.path) != 0) { + (void)close(checkpoint_lock); + cursor = entry->task_id; + continue; + } + entry->durability = durable_task.checkpoint.durability; + (void)snprintf(entry->name, sizeof(entry->name), "%s", durable_task.name); + if (durable_task.has_task_kind) { + (void)snprintf(entry->task_kind, sizeof(entry->task_kind), "%s", durable_task.task_kind); + entry->task_kind_version = durable_task.task_kind_version; + } uint32_t version = 0; + Lardon3DTaskCheckpointResult promotion = LARDON3D_TASK_CHECKPOINT_OK; + /* A codec-valid canonical checkpoint whose DB-stored task-summary + * fields match wins. A stale or corrupt .next is ignored in that + * case; the DB deliberately does not duplicate the full snapshot. */ Lardon3DTaskCheckpointResult loaded = lardon3d_task_checkpoint_load(absolute, &entry->snapshot, &version); - if (loaded == LARDON3D_TASK_CHECKPOINT_NOT_FOUND) { + bool canonical_match = loaded == LARDON3D_TASK_CHECKPOINT_OK + && version == durable_task.checkpoint.format_version + && coherent_recovery(&durable_task, &entry->snapshot); + if (!canonical_match) { + Lardon3DTaskDurableSnapshot staged_snapshot; + uint32_t staged_version = 0; + Lardon3DTaskCheckpointResult staged = lardon3d_task_checkpoint_load_staged( + absolute, &staged_snapshot, &staged_version); + if (staged == LARDON3D_TASK_CHECKPOINT_OK + && staged_version == durable_task.checkpoint.format_version + && coherent_recovery(&durable_task, &staged_snapshot)) { + entry->snapshot = staged_snapshot; +#ifdef LARDON3D_CHECKPOINT_TESTING + checkpoint_test_rendezvous("LARDON3D_TEST_RECOVERY_AFTER_STAGED_MATCH_FD"); +#endif + /* The same lock protects this re-read/promote helper from a + * replacement of .next between DB-summary validation and rename. */ + promotion = lardon3d_task_checkpoint_promote_staged(absolute); + if (promotion == LARDON3D_TASK_CHECKPOINT_OK || + promotion == LARDON3D_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE) { + loaded = LARDON3D_TASK_CHECKPOINT_OK; + version = staged_version; + canonical_match = true; + } + } + } + if (!canonical_match && loaded == LARDON3D_TASK_CHECKPOINT_NOT_FOUND) { entry->status = LARDON3D_PROJECT_RECOVERY_MISSING_CHECKPOINT; - } else if (loaded == LARDON3D_TASK_CHECKPOINT_UNSUPPORTED_VERSION) { + } else if (!canonical_match && loaded == LARDON3D_TASK_CHECKPOINT_UNSUPPORTED_VERSION) { entry->status = LARDON3D_PROJECT_RECOVERY_UNSUPPORTED_CHECKPOINT; - } else if (loaded == LARDON3D_TASK_CHECKPOINT_INVALID) { + } else if (!canonical_match && loaded == LARDON3D_TASK_CHECKPOINT_INVALID) { entry->status = LARDON3D_PROJECT_RECOVERY_INVALID_CHECKPOINT; - } else if (loaded != LARDON3D_TASK_CHECKPOINT_OK || - version != tasks[index].checkpoint.format_version || - !coherent_recovery(&tasks[index], &entry->snapshot)) { + } else if (!canonical_match) { entry->status = loaded == LARDON3D_TASK_CHECKPOINT_IO_ERROR ? LARDON3D_PROJECT_RECOVERY_CHECKPOINT_IO_ERROR : LARDON3D_PROJECT_RECOVERY_INVALID_CHECKPOINT; } else { - entry->status = entry->durability == LARDON3D_DB_CHECKPOINT_PUBLISHED_NOT_DURABLE + entry->status = promotion == LARDON3D_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE || + entry->durability == LARDON3D_DB_CHECKPOINT_PUBLISHED_NOT_DURABLE ? LARDON3D_PROJECT_RECOVERABLE_PUBLISHED_NOT_DURABLE : LARDON3D_PROJECT_RECOVERABLE; } + (void)close(checkpoint_lock); } cursor = entry->task_id; ++*count; diff --git a/src/project_db.c b/src/project_db.c index 9d2cee3..443267c 100644 --- a/src/project_db.c +++ b/src/project_db.c @@ -1404,6 +1404,22 @@ Lardon3DProjectDbResult lardon3d_project_db_open(const char *path, Lardon3DProje return LARDON3D_PROJECT_DB_OK; } +bool lardon3d_project_db_copy_path( + Lardon3DProjectDb *database, + char path[LARDON3D_PROJECT_DB_PATH_CAPACITY]) { + if (!database || !path) return false; + path[0] = '\0'; + if (pthread_mutex_lock(&database->mutex) != 0) return false; + const char *opened_path = sqlite3_db_filename(database->connection, "main"); + size_t length = opened_path + ? strnlen(opened_path, LARDON3D_PROJECT_DB_PATH_CAPACITY) + : 0; + bool valid = length > 0 && length < LARDON3D_PROJECT_DB_PATH_CAPACITY; + if (valid) memcpy(path, opened_path, length + 1); + (void)pthread_mutex_unlock(&database->mutex); + return valid; +} + void lardon3d_project_db_close(Lardon3DProjectDb *database) { if (!database) { return; @@ -5975,6 +5991,99 @@ Lardon3DProjectDbResult lardon3d_project_db_list_visual_index_pending( return result; } +Lardon3DProjectDbResult lardon3d_project_db_list_visual_index_memberships( + Lardon3DProjectDb *db, uint64_t index_id, uint64_t after, uint64_t *ids, + size_t capacity, size_t *count) { + if (count) { + *count = 0; + } + if (!db || !valid_catalog_id(index_id) || after > INT64_MAX || !ids || !count || + capacity == 0 || capacity > LARDON3D_PROJECT_DB_CATALOG_PAGE_MAX) { + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + } + (void)pthread_mutex_lock(&db->mutex); + sqlite3_stmt *statement = NULL; + Lardon3DProjectDbResult result = prepare( + db, + "SELECT m.feature_set_id FROM visual_indexes v LEFT JOIN " + "visual_index_memberships m ON m.visual_index_id=v.visual_index_id AND " + "m.feature_set_id>?2 WHERE v.visual_index_id=?1 ORDER BY m.feature_set_id LIMIT ?3", + &statement); + bool index_found = false; + if (result == LARDON3D_PROJECT_DB_OK) { + sqlite3_bind_int64(statement, 1, (sqlite3_int64)index_id); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)after); + sqlite3_bind_int64(statement, 3, (sqlite3_int64)capacity); + int code = SQLITE_DONE; + while (*count < capacity && (code = sqlite3_step(statement)) == SQLITE_ROW) { + index_found = true; + if (sqlite3_column_type(statement, 0) == SQLITE_NULL) { + continue; + } + sqlite3_int64 id = sqlite3_column_int64(statement, 0); + if (id <= 0) { + result = LARDON3D_PROJECT_DB_CORRUPT; + break; + } + ids[(*count)++] = (uint64_t)id; + } + if (result == LARDON3D_PROJECT_DB_OK && code != SQLITE_DONE && *count < capacity) { + result = sqlite_result(db, code, "list visual index memberships"); + } else if (result == LARDON3D_PROJECT_DB_OK && !index_found) { + result = LARDON3D_PROJECT_DB_NOT_FOUND; + } + sqlite3_finalize(statement); + } + (void)pthread_mutex_unlock(&db->mutex); + return result; +} + +Lardon3DProjectDbResult lardon3d_project_db_visual_index_membership_progress( + Lardon3DProjectDb *db, uint64_t index_id, uint64_t after, + uint64_t *completed_count, uint64_t *total_count) { + if (completed_count) { + *completed_count = 0; + } + if (total_count) { + *total_count = 0; + } + if (!db || !valid_catalog_id(index_id) || after > INT64_MAX || !completed_count || + !total_count) { + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + } + (void)pthread_mutex_lock(&db->mutex); + sqlite3_stmt *statement = NULL; + Lardon3DProjectDbResult result = prepare( + db, + "SELECT (SELECT count(*) FROM visual_index_memberships m WHERE " + "m.visual_index_id=v.visual_index_id AND m.feature_set_id<=?2)," + "(SELECT count(*) FROM visual_index_memberships m WHERE " + "m.visual_index_id=v.visual_index_id) FROM visual_indexes v WHERE v.visual_index_id=?1", + &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + sqlite3_bind_int64(statement, 1, (sqlite3_int64)index_id); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)after); + int code = sqlite3_step(statement); + if (code == SQLITE_ROW) { + sqlite3_int64 completed = sqlite3_column_int64(statement, 0); + sqlite3_int64 total = sqlite3_column_int64(statement, 1); + if (completed < 0 || total < 0 || completed > total) { + result = LARDON3D_PROJECT_DB_CORRUPT; + } else { + *completed_count = (uint64_t)completed; + *total_count = (uint64_t)total; + } + } else if (code == SQLITE_DONE) { + result = LARDON3D_PROJECT_DB_NOT_FOUND; + } else { + result = sqlite_result(db, code, "count visual index memberships"); + } + sqlite3_finalize(statement); + } + (void)pthread_mutex_unlock(&db->mutex); + return result; +} + Lardon3DProjectDbResult lardon3d_project_db_publish_visual_index_segment( Lardon3DProjectDb *db, const Lardon3DProjectDbVisualIndexSegment *segment, const uint64_t *ids, size_t count, Lardon3DProjectDbVisualIndexSegment *published) { diff --git a/src/raw_development_task.cpp b/src/raw_development_task.cpp index 935a534..f9dd1e0 100644 --- a/src/raw_development_task.cpp +++ b/src/raw_development_task.cpp @@ -136,7 +136,12 @@ extern "C" bool lardon3d_raw_development_task_reconstruct( Context *context = make_context(runtime_context, persisted.capture_id, persisted.source_asset_id); if (!context) return false; - *binding = {run, context, destroy, finished, context}; + *binding = {}; + binding->callback = run; + binding->userdata = context; + binding->userdata_destroy = destroy; + binding->finished_callback = finished; + binding->finished_userdata = context; return true; } catch (...) { return false; diff --git a/src/sift_task.c b/src/sift_task.c index de57ca0..31d1dd9 100644 --- a/src/sift_task.c +++ b/src/sift_task.c @@ -200,7 +200,13 @@ bool lardon3d_sift_extract_reconstruct(const Lardon3DTaskDurableSnapshot *snapsh if (memcmp(fingerprint, parameters.parameter_fingerprint, 32) != 0) return false; SiftTaskContext *context = make_context(runtime, ¶meters); if (!context) return false; - *binding = (Lardon3DTaskKindBinding){run, context, destroy_context, finished, context}; + *binding = (Lardon3DTaskKindBinding){ + .callback = run, + .userdata = context, + .userdata_destroy = destroy_context, + .finished_callback = finished, + .finished_userdata = context, + }; return true; } @@ -246,7 +252,10 @@ Lardon3DTask *lardon3d_project_create_sift_extract_task( .memory_bytes_per_item = 1024ULL * 1024 * 1024, .minimum_batch_size = 1, .maximum_batch_size = 1, - .desired_cpu_threads = 1, + /* Keep the immutable estimate independent of Matcher's + * temporary process-wide single-thread setting. Governor + * reduction makes this ceiling equal the startup pool. */ + .desired_cpu_threads = 12, .desired_io_slots = 1, .task_class = LARDON3D_RESOURCE_TASK_CPU}; const char *kind = parameters->rootsift ? LARDON3D_ROOTSIFT_EXTRACT_TASK_KIND diff --git a/src/sparse_sfm_task.cpp b/src/sparse_sfm_task.cpp index 42a8329..f3fd234 100644 --- a/src/sparse_sfm_task.cpp +++ b/src/sparse_sfm_task.cpp @@ -590,7 +590,12 @@ extern "C" bool lardon3d_sparse_sfm_task_reconstruct( delete context; return false; } - *binding = {run, context, destroy_context, finished, context}; + *binding = {}; + binding->callback = run; + binding->userdata = context; + binding->userdata_destroy = destroy_context; + binding->finished_callback = finished; + binding->finished_userdata = context; return true; } catch (...) { return false; diff --git a/src/task_checkpoint.c b/src/task_checkpoint.c index b2e9f59..e9f5061 100644 --- a/src/task_checkpoint.c +++ b/src/task_checkpoint.c @@ -286,6 +286,79 @@ sync_parent_directory(const char *path) return close(descriptor) == 0 && synced; } +static bool +staged_path(const char *path, char output[4096]) +{ + int length = snprintf(output, 4096, "%s.next", path); + return length >= 0 && length < 4096; +} + +Lardon3DTaskCheckpointResult +lardon3d_task_checkpoint_stage( + const char *path, + const Lardon3DTaskDurableSnapshot *snapshot +) +{ + char staged[4096]; + if (!path || !path[0] || !staged_path(path, staged)) { + return LARDON3D_TASK_CHECKPOINT_INVALID; + } + return lardon3d_task_checkpoint_save(staged, snapshot); +} + +Lardon3DTaskCheckpointResult +lardon3d_task_checkpoint_load_staged( + const char *path, + Lardon3DTaskDurableSnapshot *snapshot, + uint32_t *format_version +) +{ + char staged[4096]; + if (!path || !path[0] || !staged_path(path, staged)) { + return LARDON3D_TASK_CHECKPOINT_INVALID; + } + return lardon3d_task_checkpoint_load(staged, snapshot, format_version); +} + +Lardon3DTaskCheckpointResult +lardon3d_task_checkpoint_discard_staged(const char *path) +{ + char staged[4096]; + if (!path || !path[0] || !staged_path(path, staged)) { + return LARDON3D_TASK_CHECKPOINT_INVALID; + } + if (unlink(staged) != 0) { + return errno == ENOENT ? LARDON3D_TASK_CHECKPOINT_OK + : LARDON3D_TASK_CHECKPOINT_IO_ERROR; + } + return sync_parent_directory(staged) ? LARDON3D_TASK_CHECKPOINT_OK + : LARDON3D_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE; +} + +Lardon3DTaskCheckpointResult +lardon3d_task_checkpoint_promote_staged(const char *path) +{ + Lardon3DTaskDurableSnapshot snapshot; + uint32_t version = 0; + Lardon3DTaskCheckpointResult loaded = lardon3d_task_checkpoint_load_staged( + path, &snapshot, &version + ); + if (loaded != LARDON3D_TASK_CHECKPOINT_OK) { + return loaded; + } + if (version != LARDON3D_TASK_CHECKPOINT_VERSION) { + return LARDON3D_TASK_CHECKPOINT_UNSUPPORTED_VERSION; + } + /* Do not rename .next over canonical: a crash after that namespace change + * could lose the only S1 bytes. Saving from the validated staged snapshot + * keeps .next as the exact-recovery representation through directory fsync. */ + Lardon3DTaskCheckpointResult saved = lardon3d_task_checkpoint_save(path, &snapshot); + if (saved != LARDON3D_TASK_CHECKPOINT_OK) { + return saved; + } + return lardon3d_task_checkpoint_discard_staged(path); +} + Lardon3DTaskCheckpointResult lardon3d_task_checkpoint_save( const char *path, diff --git a/src/task_kind_registry.c b/src/task_kind_registry.c index 559ccec..6702983 100644 --- a/src/task_kind_registry.c +++ b/src/task_kind_registry.c @@ -4,6 +4,100 @@ #include +enum { + CANDIDATE_LEGACY_FIXED_BYTES = 128 * 1024, + CANDIDATE_CURRENT_FIXED_BYTES = 256 * 1024, + CANDIDATE_PER_ITEM_BYTES = 64 * 1024, + MATCHER_LEGACY_FIXED_BYTES = 10 * 1024 * 1024, + MATCHER_CURRENT_PER_ITEM_BYTES = 10 * 1024 * 1024, + MATCHER_GPU_FIXED_BYTES = 640 * 1024, + SIFT_FIXED_BYTES = 64 * 1024 * 1024, +}; + +static bool +estimate_equals(const Lardon3DResourceEstimate *left, + const Lardon3DResourceEstimate *right) +{ + return left->memory_fixed_bytes == right->memory_fixed_bytes + && left->gpu_memory_fixed_bytes == right->gpu_memory_fixed_bytes + && left->memory_bytes_per_item == right->memory_bytes_per_item + && left->gpu_memory_bytes_per_item == right->gpu_memory_bytes_per_item + && left->minimum_batch_size == right->minimum_batch_size + && left->maximum_batch_size == right->maximum_batch_size + && left->desired_cpu_threads == right->desired_cpu_threads + && left->desired_gpu_slots == right->desired_gpu_slots + && left->desired_io_slots == right->desired_io_slots + && left->task_class == right->task_class; +} + +static bool +normalize_known_legacy_estimate(const char *kind, + const Lardon3DResourceEstimate *durable, + Lardon3DResourceEstimate *effective) +{ + Lardon3DResourceEstimate current = {0}; + Lardon3DResourceEstimate historical = {0}; + if (strcmp(kind, "candidate_pair.generate") == 0) { + current = (Lardon3DResourceEstimate) { + .memory_fixed_bytes = CANDIDATE_CURRENT_FIXED_BYTES, + .memory_bytes_per_item = CANDIDATE_PER_ITEM_BYTES, + .minimum_batch_size = 1, + .maximum_batch_size = 64, + .desired_cpu_threads = 12, + .desired_io_slots = 1, + .task_class = LARDON3D_RESOURCE_TASK_CPU, + }; + historical = current; + historical.memory_fixed_bytes = CANDIDATE_LEGACY_FIXED_BYTES; + historical.desired_cpu_threads = 1; + } else if (strcmp(kind, "matcher.run") == 0) { + const bool gpu = durable->desired_gpu_slots == 1; + current = (Lardon3DResourceEstimate) { + .gpu_memory_fixed_bytes = gpu ? MATCHER_GPU_FIXED_BYTES : 0, + .memory_bytes_per_item = MATCHER_CURRENT_PER_ITEM_BYTES, + .minimum_batch_size = 1, + .maximum_batch_size = 8, + .desired_cpu_threads = gpu ? 1U : 8U, + .desired_gpu_slots = gpu ? 1U : 0U, + .desired_io_slots = 1, + .task_class = LARDON3D_RESOURCE_TASK_CPU, + }; + historical = current; + historical.memory_fixed_bytes = MATCHER_LEGACY_FIXED_BYTES; + historical.memory_bytes_per_item = 0; + historical.desired_cpu_threads = 12; + } else if (strcmp(kind, "features.extract.sift") == 0 + || strcmp(kind, "features.extract.rootsift") == 0) { + current = (Lardon3DResourceEstimate) { + .memory_fixed_bytes = SIFT_FIXED_BYTES, + .memory_bytes_per_item = UINT64_C(1024) * 1024 * 1024, + .minimum_batch_size = 1, + .maximum_batch_size = 1, + .desired_cpu_threads = 12, + .desired_io_slots = 1, + .task_class = LARDON3D_RESOURCE_TASK_CPU, + }; + historical = current; + historical.desired_cpu_threads = 1; + } else { + *effective = *durable; + return true; + } + if (estimate_equals(durable, ¤t)) { + *effective = *durable; + return true; + } + if (!estimate_equals(durable, &historical)) { + return false; + } + /* These exact signatures are historical operational policy, not scientific + * identity. Normalization is deliberately ephemeral: a pre-terminal crash + * may repeat it, while persistence retains one unambiguous checkpoint + * summary until ordinary Task progress advances it. */ + *effective = current; + return true; +} + bool lardon3d_task_kind_registry_init( Lardon3DTaskKindRegistry *registry, @@ -98,8 +192,16 @@ lardon3d_task_kind_registry_restore( } return LARDON3D_TASK_KIND_RECONSTRUCTION_FAILED; } + Lardon3DTaskDurableSnapshot effective_snapshot = *snapshot; + if (!normalize_known_legacy_estimate(kind, &snapshot->estimate, + &effective_snapshot.estimate)) { + if (binding.userdata_destroy) { + binding.userdata_destroy(binding.userdata); + } + return LARDON3D_TASK_KIND_RECONSTRUCTION_FAILED; + } *task = lardon3d_task_restore_typed( - snapshot, + &effective_snapshot, kind, kind_version, binding.callback, diff --git a/src/track_builder_task.cpp b/src/track_builder_task.cpp index b7b63cf..19b7f2e 100644 --- a/src/track_builder_task.cpp +++ b/src/track_builder_task.cpp @@ -259,7 +259,12 @@ static bool reconstruct_impl( return false; Context *context = nullptr; if (!make_context(runtime, durable, &context)) return false; - *binding = {run, context, destroy_context, finished, context}; + *binding = {}; + binding->callback = run; + binding->userdata = context; + binding->userdata_destroy = destroy_context; + binding->finished_callback = finished; + binding->finished_userdata = context; return true; } diff --git a/src/visual_index.c b/src/visual_index.c index dbf2a52..6b5abf9 100644 --- a/src/visual_index.c +++ b/src/visual_index.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -14,12 +15,15 @@ #include #include +#include "visual_index_internal.h" + enum { SEGMENT_HEADER_SIZE = 128, POSTING_SIZE = 24, DESCRIPTOR_SIZE = 32, SEGMENT_POSTING_MAX = LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX * 1024 * LARDON3D_VISUAL_INDEX_TABLE_COUNT, + FEATURE_SET_POSTING_MAX = 1024 * LARDON3D_VISUAL_INDEX_TABLE_COUNT, }; typedef struct { @@ -208,6 +212,92 @@ static Lardon3DVisualIndexResult append_feature_postings( return LARDON3D_VISUAL_INDEX_OK; } +typedef struct { + const char *project_path; + const Lardon3DProjectDbFeatureSet *sets; + uint32_t maximum_features; + Posting *postings; + size_t set_count; + size_t first_set; + size_t stride; + size_t *posting_counts; + Lardon3DVisualIndexResult *results; +} PostingBuildJob; + +static void *build_posting_slices(void *userdata) { + PostingBuildJob *job = userdata; + for (size_t i = job->first_set; i < job->set_count; i += job->stride) { + size_t count = 0; + job->results[i] = append_feature_postings( + job->project_path, &job->sets[i], job->maximum_features, + job->postings + i * FEATURE_SET_POSTING_MAX, FEATURE_SET_POSTING_MAX, &count); + job->posting_counts[i] = count; + } + return NULL; +} + +static Lardon3DVisualIndexResult build_postings( + const char *project_path, const Lardon3DProjectDbFeatureSet *sets, size_t set_count, + uint32_t maximum_features, unsigned int cpu_threads, Posting *postings, + size_t *posting_count) { + size_t counts[LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX] = {0}; + Lardon3DVisualIndexResult results[LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX]; + for (size_t i = 0; i < set_count; ++i) { + results[i] = LARDON3D_VISUAL_INDEX_IO_ERROR; + } + size_t participants = cpu_threads < set_count ? cpu_threads : set_count; + PostingBuildJob jobs[LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX]; + pthread_t children[LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX - 1]; + bool created[LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX - 1] = {false}; + for (size_t participant = 0; participant < participants; ++participant) { + jobs[participant] = (PostingBuildJob){.project_path = project_path, + .sets = sets, + .maximum_features = maximum_features, + .postings = postings, + .set_count = set_count, + .first_set = participant, + .stride = participants, + .posting_counts = counts, + .results = results}; + } + for (size_t participant = 1; participant < participants; ++participant) { + created[participant - 1] = + pthread_create(&children[participant - 1], NULL, build_posting_slices, + &jobs[participant]) == 0; + } + (void)build_posting_slices(&jobs[0]); + bool joined = true; + for (size_t participant = 1; participant < participants; ++participant) { + if (created[participant - 1] && pthread_join(children[participant - 1], NULL) != 0) { + joined = false; + } + } + if (!joined) { + return LARDON3D_VISUAL_INDEX_IO_ERROR; + } + /* A failed pthread_create does not change results: the owner computes only + * the missing disjoint slices after all successfully created children join. */ + for (size_t participant = 1; participant < participants; ++participant) { + if (!created[participant - 1]) { + (void)build_posting_slices(&jobs[participant]); + } + } + size_t compact_count = 0; + for (size_t i = 0; i < set_count; ++i) { + if (results[i] != LARDON3D_VISUAL_INDEX_OK) { + return results[i]; + } + /* Completion order is deliberately discarded. Each source owns a fixed + * private slice, then the owner compacts in feature_set_id selection order; + * create_segment_file applies the persisted total posting order. */ + memmove(postings + compact_count, postings + i * FEATURE_SET_POSTING_MAX, + counts[i] * sizeof(*postings)); + compact_count += counts[i]; + } + *posting_count = compact_count; + return LARDON3D_VISUAL_INDEX_OK; +} + static bool write_exact(int descriptor, const void *data, size_t size) { const unsigned char *bytes = data; while (size > 0) { @@ -372,7 +462,8 @@ static Lardon3DVisualIndexResult create_segment_file( static Lardon3DVisualIndexResult update_once_internal( const char *project_path, Lardon3DProjectDb *database, uint64_t index_id, uint64_t producer_task_id, uint64_t after, size_t maximum_feature_sets, uint64_t *last, - size_t *indexed_count, void (*after_select)(void *), void *hook_userdata) { + size_t *indexed_count, unsigned int cpu_threads, void (*after_select)(void *), + void *hook_userdata) { if (last) { *last = after; } @@ -381,7 +472,8 @@ static Lardon3DVisualIndexResult update_once_internal( } if (!project_path || !database || index_id == 0 || maximum_feature_sets == 0 || maximum_feature_sets > LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX || !last || - !indexed_count) { + !indexed_count || cpu_threads == 0 || + cpu_threads > LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX) { return LARDON3D_VISUAL_INDEX_INVALID_ARGUMENT; } Lardon3DProjectDbVisualIndex index; @@ -420,11 +512,13 @@ static Lardon3DVisualIndexResult update_once_internal( size_t posting_count = 0; Lardon3DVisualIndexResult result = LARDON3D_VISUAL_INDEX_OK; uint64_t ids[LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX]; - for (size_t i = 0; i < set_count && result == LARDON3D_VISUAL_INDEX_OK; ++i) { + for (size_t i = 0; i < set_count; ++i) { ids[i] = sets[i].feature_set_id; - result = append_feature_postings(project_path, &sets[i], index.max_features_per_set, postings, - SEGMENT_POSTING_MAX, &posting_count); } + /* All parallel work ends before the deterministic sort, asset publication, + * and membership transaction. A failed slice therefore publishes nothing. */ + result = build_postings(project_path, sets, set_count, index.max_features_per_set, + cpu_threads, postings, &posting_count); unsigned char hash[32]; char path[4096]; uint64_t size = 0; @@ -472,7 +566,15 @@ Lardon3DVisualIndexResult lardon3d_visual_index_update_once( uint64_t producer_task_id, uint64_t after, size_t maximum_feature_sets, uint64_t *last, size_t *indexed_count) { return update_once_internal(project_path, database, index_id, producer_task_id, after, - maximum_feature_sets, last, indexed_count, NULL, NULL); + maximum_feature_sets, last, indexed_count, 1, NULL, NULL); +} + +Lardon3DVisualIndexResult lardon3d_visual_index_update_once_parallel( + const char *project_path, Lardon3DProjectDb *database, uint64_t index_id, + uint64_t producer_task_id, uint64_t after, size_t maximum_feature_sets, + unsigned int cpu_threads, uint64_t *last, size_t *indexed_count) { + return update_once_internal(project_path, database, index_id, producer_task_id, after, + maximum_feature_sets, last, indexed_count, cpu_threads, NULL, NULL); } #ifdef LARDON3D_VISUAL_INDEX_TESTING @@ -482,7 +584,7 @@ Lardon3DVisualIndexResult lardon3d_visual_index_test_update_once( size_t *indexed_count, Lardon3DVisualIndexAfterSelectHook after_select, void *hook_userdata) { return update_once_internal(project_path, database, index_id, producer_task_id, after, - maximum_feature_sets, last, indexed_count, after_select, + maximum_feature_sets, last, indexed_count, 1, after_select, hook_userdata); } #endif diff --git a/src/visual_index_internal.h b/src/visual_index_internal.h new file mode 100644 index 0000000..e63f505 --- /dev/null +++ b/src/visual_index_internal.h @@ -0,0 +1,18 @@ +#ifndef LARDON3D_VISUAL_INDEX_INTERNAL_H +#define LARDON3D_VISUAL_INDEX_INTERNAL_H + +#include +#include + +#include +#include + +/* Task-internal entry point: cpu_threads includes the Queue callback. Child + * threads only read immutable Feature Files into disjoint staging slices; the + * callback retains sole ownership of canonical reduction and publication. */ +Lardon3DVisualIndexResult lardon3d_visual_index_update_once_parallel( + const char *project_path, Lardon3DProjectDb *database, uint64_t visual_index_id, + uint64_t producer_task_id, uint64_t after_feature_set_id, size_t maximum_feature_sets, + unsigned int cpu_threads, uint64_t *last_feature_set_id, size_t *indexed_count); + +#endif diff --git a/src/visual_index_task.c b/src/visual_index_task.c index 3a7eefb..d9c34af 100644 --- a/src/visual_index_task.c +++ b/src/visual_index_task.c @@ -11,6 +11,10 @@ #include #include +#include "visual_index_internal.h" + +enum { VISUAL_INDEX_TASK_CPU_THREADS = 12 }; + typedef struct { char project_path[PATH_MAX]; Lardon3DProjectDb *database; @@ -64,7 +68,8 @@ static bool run(Lardon3DTask *task, void *userdata) { } Lardon3DTaskExecutionContract contract; if (!lardon3d_task_execution_contract(task, &contract) || contract.batch_size == 0 || - contract.batch_size > LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX) { + contract.batch_size > LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX || + contract.cpu_threads == 0 || contract.cpu_threads > VISUAL_INDEX_TASK_CPU_THREADS) { return lardon3d_task_fail(task, "Contrat Visual Index invalide."); } struct timespec begin; @@ -72,10 +77,10 @@ static bool run(Lardon3DTask *task, void *userdata) { clock_gettime(CLOCK_MONOTONIC, &begin); uint64_t last = context->parameters.after_feature_set_id; size_t indexed = 0; - Lardon3DVisualIndexResult result = lardon3d_visual_index_update_once( + Lardon3DVisualIndexResult result = lardon3d_visual_index_update_once_parallel( context->project_path, context->database, context->parameters.visual_index_id, lardon3d_task_id(task), context->parameters.after_feature_set_id, contract.batch_size, - &last, &indexed); + contract.cpu_threads, &last, &indexed); clock_gettime(CLOCK_MONOTONIC, &end); if (result == LARDON3D_VISUAL_INDEX_NO_CHANGE) { return lardon3d_task_set_progress(task, 100, "Visual Index à jour."); @@ -190,7 +195,7 @@ Lardon3DTask *lardon3d_project_create_visual_index_update_task( .memory_bytes_per_item = 2ULL * 1024 * 1024, .minimum_batch_size = 1, .maximum_batch_size = 16, - .desired_cpu_threads = 1, + .desired_cpu_threads = VISUAL_INDEX_TASK_CPU_THREADS, .desired_io_slots = 1, .task_class = LARDON3D_RESOURCE_TASK_CPU}; Lardon3DTask *task = lardon3d_task_create_typed( diff --git a/tests/pre_sfm_real_execution.cpp b/tests/pre_sfm_real_execution.cpp new file mode 100644 index 0000000..bb44dbb --- /dev/null +++ b/tests/pre_sfm_real_execution.cpp @@ -0,0 +1,1318 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +} + +namespace { + +enum class Mode { kA6000, kS21 }; +enum class RestartBoundary { kNone, kRepresentations, kFeatures, kGeometry }; + +struct Options { + Mode mode{}; + bool has_mode{}; + bool resume_geometry_existing{}; + bool resume_pre_gv_existing{}; + bool resume_candidate_existing{}; + unsigned int cpu_budget{}; + std::filesystem::path project_dir; + std::vector roots; + size_t limit{LARDON3D_VISUAL_INDEX_CANDIDATE_MAX}; + RestartBoundary restart{RestartBoundary::kNone}; +}; + +struct Campaign { + std::vector sources; + std::vector confirmations; +}; + +struct Runtime { + Lardon3DAppState state{}; + std::string project_path; + std::string database_path; + unsigned int cpu_budget{}; +}; + +struct Evidence { + size_t selected{}; + size_t features{}; + size_t pairs{}; + size_t matches{}; + size_t verified{}; + size_t rejected{}; + size_t tracks{}; +}; + +void usage(const char *program) { + std::fprintf( + stderr, + "Usage: %s --mode a6000|s21 --project-dir ABSOLUTE_EMPTY_DIR " + "--root ABSOLUTE_DIR [--root ABSOLUTE_DIR ...] [--limit 1..4096] " + "[--restart-boundary representations|features|geometry]\n" + " %s --resume-geometry-existing --project-dir " + "ABSOLUTE_EXISTING_DIR\n" + " %s --resume-pre-gv-existing --project-dir " + "ABSOLUTE_EXISTING_DIR\n" + " %s --resume-candidate-existing --project-dir " + "ABSOLUTE_EXISTING_DIR [--cpu-budget 1..12]\n", + program, + program, + program, + program); +} + +bool parse_size(const char *text, size_t &value) { + char *end = nullptr; + errno = 0; + const unsigned long long parsed = std::strtoull(text, &end, 10); + if (errno != 0 || end == text || *end != '\0' || parsed == 0 || + parsed > LARDON3D_VISUAL_INDEX_CANDIDATE_MAX) + return false; + value = static_cast(parsed); + return true; +} + +bool parse_cpu_budget(const char *text, unsigned int &value) { + size_t parsed = 0; + if (!parse_size(text, parsed) || parsed > 12) return false; + value = static_cast(parsed); + return true; +} + +bool parse_options(int argc, char **argv, Options &options) { + for (int index = 1; index < argc; ++index) { + const std::string argument(argv[index]); + if (argument == "--help") { + usage(argv[0]); + std::exit(0); + } + if (argument == "--resume-geometry-existing") { + options.resume_geometry_existing = true; + continue; + } + if (argument == "--resume-pre-gv-existing") { + options.resume_pre_gv_existing = true; + continue; + } + if (argument == "--resume-candidate-existing") { + options.resume_candidate_existing = true; + continue; + } + if ((argument == "--mode" || argument == "--project-dir" || argument == "--root" || + argument == "--limit" || argument == "--restart-boundary" || + argument == "--cpu-budget") && + index + 1 >= argc) + return false; + if (argument == "--mode") { + const std::string value(argv[++index]); + if (value == "a6000") options.mode = Mode::kA6000; + else if (value == "s21") options.mode = Mode::kS21; + else return false; + options.has_mode = true; + } else if (argument == "--project-dir") { + options.project_dir = argv[++index]; + } else if (argument == "--root") { + options.roots.emplace_back(argv[++index]); + } else if (argument == "--limit") { + if (!parse_size(argv[++index], options.limit)) return false; + } else if (argument == "--restart-boundary") { + const std::string value(argv[++index]); + if (value == "representations") options.restart = RestartBoundary::kRepresentations; + else if (value == "features") options.restart = RestartBoundary::kFeatures; + else if (value == "geometry") options.restart = RestartBoundary::kGeometry; + else return false; + } else if (argument == "--cpu-budget") { + if (!parse_cpu_budget(argv[++index], options.cpu_budget)) return false; + } else { + return false; + } + } + const unsigned int resume_mode_count = options.resume_geometry_existing + + options.resume_pre_gv_existing + + options.resume_candidate_existing; + if (resume_mode_count != 0) + return !options.has_mode && !options.project_dir.empty() && options.roots.empty() && + options.restart == RestartBoundary::kNone && + resume_mode_count == 1 && + (options.resume_candidate_existing || options.cpu_budget == 0); + if (options.cpu_budget != 0) return false; + return options.has_mode && !options.project_dir.empty() && !options.roots.empty() && + options.roots.size() <= LARDON3D_ACQUISITION_CAMPAIGN_MAX_ROOTS; +} + +bool prepare_existing_project(const std::filesystem::path &input, Runtime &runtime) { + if (!input.is_absolute() || input.lexically_normal() != input) return false; + std::error_code error; + if (!std::filesystem::is_directory(input, error) || error || + !std::filesystem::is_regular_file(input / "project.lardon3d", error) || error || + !std::filesystem::is_directory(input / ".lardon3d" / "checkpoints", error) || error) + return false; + runtime.project_path = input.string(); + runtime.database_path = (input / "project.lardon3d").string(); + return runtime.project_path.size() < LARDON3D_APP_STATE_PATH_CAPACITY; +} + +bool prepare_empty_project(const std::filesystem::path &input, std::string &path) { + if (!input.is_absolute() || input.lexically_normal() != input) return false; + std::error_code error; + if (std::filesystem::exists(input, error)) { + if (error || !std::filesystem::is_directory(input, error) || error || + !std::filesystem::is_empty(input, error) || error) + return false; + } else if (!std::filesystem::create_directories(input, error) || error) { + return false; + } + path = input.string(); + if (path.size() >= LARDON3D_APP_STATE_PATH_CAPACITY) return false; + return std::filesystem::create_directories(input / ".lardon3d" / "checkpoints", error) && + !error; +} + +bool canonical_root(const std::filesystem::path &input, Lardon3DAcquisitionCampaignRoot &root) { + if (!input.is_absolute()) return false; + std::error_code error; + const auto canonical = std::filesystem::canonical(input, error); + if (error || !std::filesystem::is_directory(canonical, error) || error) return false; + const std::string path = canonical.string(); + if (path.size() >= sizeof(root.path)) return false; + std::memcpy(root.path, path.c_str(), path.size() + 1u); + return true; +} + +std::string source_stem(const char *path) { + const char *name = std::strrchr(path, '/'); + name = name == nullptr ? path : name + 1; + const char *dot = std::strrchr(name, '.'); + return std::string(name, dot == nullptr ? std::strlen(name) + : static_cast(dot - name)); +} + +bool discover_campaign(const Options &options, Campaign &campaign) { + std::vector roots(options.roots.size()); + for (size_t index = 0; index < roots.size(); ++index) + if (!canonical_root(options.roots[index], roots[index])) { + std::fprintf(stderr, "campaign root %zu is not an absolute readable directory: %s\n", + index, options.roots[index].c_str()); + return false; + } + auto discovery = std::make_unique(); + const auto discovery_result = + lardon3d_acquisition_campaign_discover(roots.data(), roots.size(), discovery.get()); + if (discovery_result != LARDON3D_ACQUISITION_CAMPAIGN_OK) { + std::fprintf(stderr, "campaign discovery failed with result %d\n", + static_cast(discovery_result)); + return false; + } + + std::vector> groups; + if (options.mode == Mode::kS21) { + for (size_t index = 0; index < discovery->source_count && groups.size() < options.limit; + ++index) { + if (discovery->sources[index].source_kind == LARDON3D_ACQUISITION_SOURCE_JPEG) + groups.push_back({index}); + } + } else { + struct Pair { size_t raw{SIZE_MAX}; size_t jpeg{SIZE_MAX}; }; + std::map pairs; + for (size_t index = 0; index < discovery->source_count; ++index) { + Pair &pair = pairs[source_stem(discovery->sources[index].path)]; + if (discovery->sources[index].source_kind == LARDON3D_ACQUISITION_SOURCE_RAW) { + if (pair.raw != SIZE_MAX) { + std::fprintf(stderr, "duplicate RAW stem in discovery: %s\n", + source_stem(discovery->sources[index].path).c_str()); + return false; + } + pair.raw = index; + } else if (discovery->sources[index].source_kind == LARDON3D_ACQUISITION_SOURCE_JPEG) { + if (pair.jpeg != SIZE_MAX) { + std::fprintf(stderr, "duplicate JPEG stem in discovery: %s\n", + source_stem(discovery->sources[index].path).c_str()); + return false; + } + pair.jpeg = index; + } + } + for (const auto &entry : pairs) { + if (entry.second.raw == SIZE_MAX || entry.second.jpeg == SIZE_MAX) { + std::fprintf(stderr, "unpaired A6000 stem: %s (%s missing)\n", entry.first.c_str(), + entry.second.raw == SIZE_MAX ? "RAW" : "JPEG"); + return false; + } + groups.push_back({entry.second.raw, entry.second.jpeg}); + if (groups.size() == options.limit) break; + } + } + if (groups.empty()) { + std::fprintf(stderr, "campaign discovery found no usable groups among %zu sources\n", + discovery->source_count); + return false; + } + + std::vector selected_indices; + for (const auto &group : groups) + selected_indices.insert(selected_indices.end(), group.begin(), group.end()); + std::sort(selected_indices.begin(), selected_indices.end()); + std::vector compact_index(discovery->source_count, SIZE_MAX); + for (const size_t discovery_index : selected_indices) { + compact_index[discovery_index] = campaign.sources.size(); + campaign.sources.push_back(discovery->sources[discovery_index]); + } + + /* Filename stems are used only to form explicit CALLER_EXPLICIT input before + * Captures exist. The compact source array retains discovery's global canonical + * path order, while confirmations map each pair independently of that order. + * Once S3-E returns, every phase below follows durable group, Capture, Asset and + * image IDs and never treats this convenience key as identity. */ + for (const auto &group : groups) { + Lardon3DAcquisitionCampaignConfirmation confirmation{}; + confirmation.source_count = group.size(); + for (size_t member = 0; member < group.size(); ++member) + confirmation.source_indices[member] = compact_index[group[member]]; + campaign.confirmations.push_back(confirmation); + } + auto plan = std::make_unique(); + const auto plan_result = lardon3d_acquisition_campaign_plan( + campaign.sources.data(), campaign.sources.size(), campaign.confirmations.data(), + campaign.confirmations.size(), plan.get()); + if (plan_result != LARDON3D_ACQUISITION_CAMPAIGN_OK || + plan->group_count != campaign.confirmations.size()) { + std::fprintf(stderr, + "explicit campaign plan rejected: result=%d sources=%zu confirmations=%zu " + "groups=%zu\n", + static_cast(plan_result), campaign.sources.size(), + campaign.confirmations.size(), plan->group_count); + return false; + } + return true; +} + +void stop_runtime(Runtime &runtime) { + if (runtime.state.task_queue) lardon3d_task_queue_destroy(runtime.state.task_queue); + if (runtime.state.resource_governor) + lardon3d_resource_governor_destroy(runtime.state.resource_governor); + if (runtime.state.project_db) lardon3d_project_db_close(runtime.state.project_db); + runtime.state.task_queue = nullptr; + runtime.state.resource_governor = nullptr; + runtime.state.project_db = nullptr; +} + +bool start_runtime(Runtime &runtime) { + char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY]{}; + lardon3d_app_state_init(&runtime.state); + if (lardon3d_project_db_open(runtime.database_path.c_str(), &runtime.state.project_db, error) != + LARDON3D_PROJECT_DB_OK) { + std::fprintf(stderr, "project DB open failed: %s\n", error); + return false; + } + char hardware_error[256]{}; + Lardon3DResourcePolicy policy{}; + if (!lardon3d_hardware_profile_detect(&runtime.state.hardware_profile, hardware_error, + sizeof(hardware_error)) || + !lardon3d_resource_policy_default(&runtime.state.hardware_profile, &policy)) { + std::fprintf(stderr, "hardware policy failed: %s\n", hardware_error); + stop_runtime(runtime); + return false; + } + if (runtime.cpu_budget != 0) { + if (runtime.cpu_budget > runtime.state.hardware_profile.logical_cpu_count) { + std::fprintf(stderr, "requested CPU budget %u exceeds logical CPU count %u\n", + runtime.cpu_budget, runtime.state.hardware_profile.logical_cpu_count); + stop_runtime(runtime); + return false; + } + // This runner-only override is applied before Governor creation and is never + // persisted. The Governor remains the sole admission owner: reserving N + // logical CPUs constrains the recovered Candidate Task's execution contract. + policy.system_cpu_reserve = runtime.state.hardware_profile.logical_cpu_count - + runtime.cpu_budget; + } + unsigned int feature_threads = runtime.state.hardware_profile.logical_cpu_count - + policy.system_cpu_reserve; + if (feature_threads > 12) feature_threads = 12; + /* Match the production startup contract: OpenCV's process-wide pool is + * configured from the same detected profile and Governor policy before any + * Queue worker exists. The runner-only CPU override therefore constrains + * both admission and nested feature extraction consistently. */ + if (!lardon3d_feature_opencv_configure_threads(feature_threads)) { + std::fprintf(stderr, "OpenCV thread configuration failed for %u threads\n", + feature_threads); + stop_runtime(runtime); + return false; + } + runtime.state.resource_governor = + lardon3d_resource_governor_create(&runtime.state.hardware_profile, &policy); + runtime.state.task_queue = runtime.state.resource_governor + ? lardon3d_task_queue_create(runtime.state.resource_governor, 2) + : nullptr; + if (!runtime.state.task_queue) { + stop_runtime(runtime); + return false; + } + runtime.state.project_loaded = true; + std::memcpy(runtime.state.project_path, runtime.project_path.c_str(), + runtime.project_path.size() + 1u); + /* Task owns execution state, Queue owns bounded dispatch/backpressure, and + * Governor owns each task's admission reservation. Sequential enqueue/wait + * below releases that reservation at the task boundary and never reserves an + * entire campaign or creates a parallel worker/scheduler subsystem. */ + return true; +} + +bool wait_completed(Runtime &runtime, uint64_t task_id, const char *phase) { + for (;;) { + Lardon3DTaskSnapshot snapshot{}; + if (!lardon3d_task_queue_get(runtime.state.task_queue, task_id, &snapshot)) return false; + if (snapshot.state == TASK_COMPLETED || snapshot.state == TASK_FAILED || + snapshot.state == TASK_CANCELLED) { + Lardon3DProjectDbTask durable{}; + if (lardon3d_project_db_load_task(runtime.state.project_db, task_id, &durable) == + LARDON3D_PROJECT_DB_OK && + durable.saved_state == snapshot.state) { + if (snapshot.state != TASK_COMPLETED) { + std::fprintf(stderr, "%s task %llu ended in %s: %s\n", phase, + static_cast(task_id), + lardon3d_task_state_name(snapshot.state), snapshot.message); + return false; + } + return lardon3d_task_queue_remove(runtime.state.task_queue, task_id); + } + } + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } +} + +bool restart_runtime(Runtime &runtime, uint64_t execution_id, const char *boundary) { + stop_runtime(runtime); + if (!start_runtime(runtime)) return false; + Lardon3DProjectDbSelectedExecution execution{}; + if (lardon3d_project_db_load_selected_execution(runtime.state.project_db, execution_id, + &execution) != LARDON3D_PROJECT_DB_OK) { + std::fprintf(stderr, "restart at %s lost retained execution %llu\n", boundary, + static_cast(execution_id)); + return false; + } + /* This boundary deliberately has no sidecar or durable top-level phase. The + * caller retains only execution_id; the next phase reconstructs exact image, + * FeatureSet or result IDs from v22 durable records and their cursors. */ + std::printf("{\"record\":\"restart\",\"kind\":\"REOPEN_ONLY\"," + "\"boundary\":\"%s\",\"execution_id\":%llu}\n", + boundary, static_cast(execution_id)); + return true; +} + +bool recover_quality_task(Runtime &runtime, uint64_t task_id) { + /* Durable Task creation publishes its typed request and initial cursor before + * Queue transfer. The original unqueued object has been discarded to model + * loss before first dispatch. Recovery must use the existing registry to + * reconstruct callback state, then transfer the Task through Queue and + * Governor admission; the runner adds no top-level durable state, test hook, + * or alternate worker path. */ + stop_runtime(runtime); + if (!start_runtime(runtime)) return false; + Lardon3DProjectRecoverySummary recovery{}; + if (lardon3d_project_resume_recoverable_tasks( + &runtime.state, lardon3d_task_kind_registry_production(), &recovery) != + LARDON3D_PROJECT_DB_OK || + recovery.resumed != 1 || recovery.failed != 0) + return false; + std::printf("{\"record\":\"restart\"," + "\"kind\":\"REGISTRY_QUEUE_GOVERNOR_RESTART\"," + "\"task_id\":%llu,\"inspected\":%zu,\"resumed\":%zu}\n", + static_cast(task_id), recovery.inspected, recovery.resumed); + return wait_completed(runtime, task_id, "photo_quality.triage recovered"); +} + +bool run_task_pair(Runtime &runtime, uint64_t scanset_id, const Campaign &campaign, + bool prove_restart, uint64_t &quality_task_id, uint64_t &campaign_task_id) { + Lardon3DPhotoQualityTaskRequest quality_request{ + campaign.sources.data(), campaign.sources.size(), campaign.confirmations.data(), + campaign.confirmations.size()}; + if (prove_restart) { + Lardon3DTask *pending = lardon3d_project_create_photo_quality_task( + &runtime.state, scanset_id, &quality_request, &quality_task_id); + if (!pending) return false; + lardon3d_task_destroy(pending); + if (!recover_quality_task(runtime, quality_task_id)) { + std::fprintf(stderr, "quality Task could not be restored at its pending boundary\n"); + return false; + } + } else if (!lardon3d_project_enqueue_photo_quality(&runtime.state, scanset_id, &quality_request, + &quality_task_id) || + !wait_completed(runtime, quality_task_id, "photo_quality.triage")) { + return false; + } + Lardon3DAcquisitionCampaignTaskRequest campaign_request{}; + campaign_request.sources = campaign.sources.data(); + campaign_request.source_count = campaign.sources.size(); + campaign_request.confirmations = campaign.confirmations.data(); + campaign_request.confirmation_count = campaign.confirmations.size(); + campaign_request.ingest_options.representation = LARDON3D_ACQUISITION_SELECT_JPEG_SOURCE; + campaign_request.ingest_options.select_representation = 1; + campaign_request.ingest_options.imported_at = std::time(nullptr); + campaign_request.ingest_options.max_source_bytes = UINT64_MAX; + return lardon3d_project_enqueue_acquisition_campaign( + &runtime.state, scanset_id, &campaign_request, &campaign_task_id) && + wait_completed(runtime, campaign_task_id, "acquisition_campaign.run"); +} + +bool create_selection(Runtime &runtime, Mode mode, uint64_t quality_task_id, + uint64_t campaign_task_id, size_t group_count, + Lardon3DProjectDbSelectedExecution &execution) { + std::vector items; + for (size_t index = 0; index < group_count; ++index) { + const uint32_t group_id = static_cast(index + 1u); + Lardon3DProjectDbPhotoQualityResult quality{}; + Lardon3DProjectDbAcquisitionCampaignCapture campaign{}; + if (lardon3d_project_db_load_photo_quality_result(runtime.state.project_db, quality_task_id, + group_id, &quality) != + LARDON3D_PROJECT_DB_OK || + lardon3d_project_db_load_acquisition_campaign_capture( + runtime.state.project_db, campaign_task_id, group_id, &campaign) != + LARDON3D_PROJECT_DB_OK) + return false; + if (quality.metrics.recommendation != LARDON3D_PHOTO_QUALITY_GOOD || + quality.override_value != LARDON3D_PHOTO_QUALITY_OVERRIDE_NONE) + continue; + Lardon3DProjectDbSelectedExecutionItem item{}; + item.item_index = static_cast(items.size()); + item.quality_group_id = group_id; + item.campaign_group_id = group_id; + item.capture_id = campaign.capture_id; + if (mode == Mode::kA6000) { + Lardon3DProjectDbCaptureSourceAsset sources[4]{}; + size_t source_count = 0; + if (lardon3d_project_db_list_capture_source_assets(runtime.state.project_db, + campaign.capture_id, 0, sources, 4, + &source_count) != + LARDON3D_PROJECT_DB_OK) + return false; + size_t raw_count = 0; + for (size_t source = 0; source < source_count; ++source) { + if (sources[source].source_kind == LARDON3D_DB_CAPTURE_SOURCE_RAW) { + item.source_asset_id = sources[source].asset_id; + ++raw_count; + } + } + if (raw_count != 1) return false; + item.representation_source = LARDON3D_SELECTED_REPRESENTATION_RAW_ASSET; + } else { + item.representation_source = LARDON3D_SELECTED_REPRESENTATION_SOURCE_IMAGE; + } + items.push_back(item); + } + if (items.empty()) { + std::printf("{\"record\":\"selection\",\"selected\":0," + "\"reason\":\"no GOOD+NONE groups\"}\n"); + return true; + } + /* These item records are the only bridge between independent quality and + * campaign group namespaces. Equal numeric IDs are supplied explicitly and + * never used to infer Capture, Asset, or image identity. */ + return lardon3d_project_db_create_selected_execution( + runtime.state.project_db, quality_task_id, campaign_task_id, items.data(), + items.size(), std::time(nullptr), &execution) == LARDON3D_PROJECT_DB_OK; +} + +bool publish_representations(Runtime &runtime, uint64_t execution_id, + std::vector &image_ids) { + Lardon3DProjectDbSelectedExecution execution{}; + if (lardon3d_project_db_load_selected_execution(runtime.state.project_db, execution_id, + &execution) != LARDON3D_PROJECT_DB_OK) + return false; + for (uint32_t index = execution.next_item_index; index < execution.item_count; ++index) { + Lardon3DProjectDbSelectedExecutionItem item{}; + if (lardon3d_project_db_load_selected_execution_item(runtime.state.project_db, execution_id, + index, &item) != LARDON3D_PROJECT_DB_OK) + return false; + uint64_t image_id = 0; + if (item.representation_source == LARDON3D_SELECTED_REPRESENTATION_RAW_ASSET) { + uint64_t task_id = 0; + if (!lardon3d_project_enqueue_raw_development(&runtime.state, item.capture_id, + item.source_asset_id, &task_id) || + !wait_completed(runtime, task_id, "raw.develop")) + return false; + Lardon3DProjectDbRawDevelopmentTask raw{}; + if (lardon3d_project_db_load_raw_development_task(runtime.state.project_db, task_id, &raw) != + LARDON3D_PROJECT_DB_OK || + raw.phase != LARDON3D_RAW_DEVELOPMENT_TASK_PUBLISHED || !raw.has_image || + raw.capture_id != item.capture_id || raw.source_asset_id != item.source_asset_id) + return false; + image_id = raw.image_id; + } else { + if (lardon3d_project_db_get_selected_capture_image(runtime.state.project_db, item.capture_id, + &image_id) != LARDON3D_PROJECT_DB_OK) + return false; + } + if (lardon3d_project_db_record_selected_representation(runtime.state.project_db, execution_id, + index, image_id, index + 1u) != + LARDON3D_PROJECT_DB_OK) + return false; + } + if (lardon3d_project_db_load_selected_execution(runtime.state.project_db, execution_id, + &execution) != LARDON3D_PROJECT_DB_OK || + execution.stage != LARDON3D_SELECTED_EXECUTION_CALIBRATION) + return false; + image_ids.clear(); + for (uint32_t index = 0; index < execution.item_count; ++index) { + Lardon3DProjectDbSelectedExecutionItem item{}; + if (lardon3d_project_db_load_selected_execution_item(runtime.state.project_db, execution_id, + index, &item) != LARDON3D_PROJECT_DB_OK || + !item.has_image) + return false; + image_ids.push_back(item.image_id); + } + return true; +} + +bool extract_features(Runtime &runtime, const std::vector &image_ids, + const Lardon3DFeatureExtractorParameters ¶meters, + std::vector &feature_sets) { + unsigned char fingerprint[32]{}; + lardon3d_feature_extractor_parameter_fingerprint(¶meters, fingerprint); + feature_sets.clear(); + for (const uint64_t image_id : image_ids) { + Lardon3DProjectDbFeatureSet feature{}; + if (lardon3d_project_db_find_feature_set(runtime.state.project_db, image_id, + LARDON3D_FEATURE_EXTRACTOR_KIND, + LARDON3D_FEATURE_EXTRACTOR_VERSION, fingerprint, + &feature) != LARDON3D_PROJECT_DB_OK) { + uint64_t task_id = 0; + if (!lardon3d_project_enqueue_feature_extract(&runtime.state, image_id, ¶meters, + &task_id) || + !wait_completed(runtime, task_id, "features.extract") || + lardon3d_project_db_find_feature_set( + runtime.state.project_db, image_id, LARDON3D_FEATURE_EXTRACTOR_KIND, + LARDON3D_FEATURE_EXTRACTOR_VERSION, fingerprint, &feature) != + LARDON3D_PROJECT_DB_OK) + return false; + } + feature_sets.push_back(feature); + } + return true; +} + +bool collect_verified_ids(Runtime &runtime, const unsigned char verifier_fingerprint[32], + std::vector &verified_ids, + size_t *applicable = nullptr, size_t *verified = nullptr, + size_t *rejected = nullptr) { + verified_ids.clear(); + if (applicable) *applicable = 0; + if (verified) *verified = 0; + if (rejected) *rejected = 0; + uint64_t match_cursor = 0; + for (;;) { + Lardon3DProjectDbMatchResult matches[64]{}; + size_t match_count = 0; + if (lardon3d_project_db_list_match_results(runtime.state.project_db, match_cursor, matches, 64, + &match_count) != LARDON3D_PROJECT_DB_OK) + return false; + for (size_t index = 0; index < match_count; ++index) { + match_cursor = matches[index].match_result_id; + if (matches[index].result_status != LARDON3D_MATCH_RESULT_STATUS_MATCHED || + matches[index].match_count == 0) + continue; + if (applicable) ++*applicable; + Lardon3DProjectDbGeometricVerificationResult result{}; + if (lardon3d_project_db_find_geometric_verification_result( + runtime.state.project_db, matches[index].match_result_id, + LARDON3D_GEOMETRIC_VERIFIER_FUNDAMENTAL, LARDON3D_GEOMETRIC_VERIFIER_VERSION, + verifier_fingerprint, &result) != LARDON3D_PROJECT_DB_OK) + return false; + if (result.status == LARDON3D_GEOMETRIC_VERIFIED) { + if (verified) ++*verified; + verified_ids.push_back(result.geometric_verification_result_id); + } else if (result.status == LARDON3D_GEOMETRIC_REJECTED) { + if (rejected) ++*rejected; + } else { + return false; + } + } + if (match_count < 64) break; + } + std::sort(verified_ids.begin(), verified_ids.end()); + return std::adjacent_find(verified_ids.begin(), verified_ids.end()) == verified_ids.end(); +} + +bool downstream(Runtime &runtime, const std::vector &feature_sets, + const Lardon3DFeatureExtractorParameters &orb, uint64_t &geometry_task_id, + std::vector &verified_ids) { + Lardon3DVisualIndexConfiguration index_configuration{ + LARDON3D_VISUAL_INDEX_VERSION, 1024, 256}; + uint64_t visual_index_id = 0; + if (feature_sets.empty() || + lardon3d_visual_index_create(runtime.state.project_db, &feature_sets.front(), + &index_configuration, &visual_index_id) != + LARDON3D_VISUAL_INDEX_OK) + return false; + uint64_t task_id = 0; + if (!lardon3d_project_enqueue_visual_index_update(&runtime.state, visual_index_id, &task_id) || + !wait_completed(runtime, task_id, "visual_index.update")) + return false; + + Lardon3DVisualIndexQueryOptions query{ + std::min(64, static_cast(feature_sets.size())), 1, + LARDON3D_VISUAL_INDEX_SAME_SCANSET, true}; + if (!lardon3d_project_enqueue_candidate_pair_generate(&runtime.state, visual_index_id, &query, + &task_id) || + !wait_completed(runtime, task_id, "candidate_pair.generate")) + return false; + Lardon3DMatcherTaskConfiguration matcher{}; + std::snprintf(matcher.feature_extractor_kind, sizeof(matcher.feature_extractor_kind), "%s", + LARDON3D_FEATURE_EXTRACTOR_KIND); + matcher.feature_extractor_version = LARDON3D_FEATURE_EXTRACTOR_VERSION; + lardon3d_feature_extractor_parameter_fingerprint(&orb, matcher.feature_parameter_fingerprint); + matcher.matcher.kind = LARDON3D_MATCHER_ORB_BF; + matcher.matcher.ratio_threshold = lardon3d_matcher_default_ratio(LARDON3D_MATCHER_ORB_BF); + if (!lardon3d_project_enqueue_matcher_task(&runtime.state, &matcher, &task_id) || + !wait_completed(runtime, task_id, "matcher.run")) + return false; + Lardon3DGeometricVerifierTaskConfiguration verifier{ + lardon3d_geometric_verifier_default_parameters()}; + if (!lardon3d_project_enqueue_geometric_verifier_task(&runtime.state, &verifier, + &geometry_task_id) || + !wait_completed(runtime, geometry_task_id, "geometric_verifier.run")) + return false; + + unsigned char verifier_fingerprint[32]{}; + lardon3d_geometric_verifier_fingerprint(&verifier.verifier, verifier_fingerprint); + return collect_verified_ids(runtime, verifier_fingerprint, verified_ids); +} + +bool build_tracks(Runtime &runtime, const std::vector &verified_ids, + const unsigned char verifier_fingerprint[32], + uint64_t *task_id_output = nullptr, + Lardon3DProjectDbTrackSet *track_set_output = nullptr) { + if (task_id_output) *task_id_output = 0; + if (track_set_output) *track_set_output = {}; + if (verified_ids.empty()) return true; + if (verified_ids.size() > (std::numeric_limits::max() - 8U) / 8U) return false; + std::vector scope_bytes(8U + verified_ids.size() * 8U); + std::memcpy(scope_bytes.data(), "L3DTSIS1", 8); + for (size_t index = 0; index < verified_ids.size(); ++index) { + uint64_t value = verified_ids[index]; + for (size_t byte = 0; byte < 8; ++byte) { + scope_bytes[8U + index * 8U + byte] = static_cast(value & 0xffU); + value >>= 8U; + } + } + Lardon3DProjectDbTrackSet exact{}; + std::snprintf(exact.builder_kind, sizeof(exact.builder_kind), "track_builder"); + exact.builder_version = LARDON3D_TRACK_BUILDER_VERSION; + unsigned int digest_size = 0; + if (!lardon3d_track_builder_fingerprint(exact.parameter_fingerprint) || + EVP_Digest(scope_bytes.data(), scope_bytes.size(), exact.input_scope_hash, &digest_size, + EVP_sha256(), nullptr) != 1 || + digest_size != 32) + return false; + exact.verifier_kind = LARDON3D_GEOMETRIC_VERIFIER_FUNDAMENTAL; + exact.verifier_version = LARDON3D_GEOMETRIC_VERIFIER_VERSION; + std::memcpy(exact.verifier_fingerprint, verifier_fingerprint, 32); + exact.gvr_count = verified_ids.size(); + Lardon3DProjectDbTrackSet existing{}; + const Lardon3DProjectDbResult found = + lardon3d_project_db_find_track_set(runtime.state.project_db, &exact, &existing); + if (found == LARDON3D_PROJECT_DB_OK) { + if (track_set_output) *track_set_output = existing; + return true; + } + if (found != LARDON3D_PROJECT_DB_NOT_FOUND) return false; + Lardon3DTrackBuilderTaskConfiguration configuration{ + runtime.project_path.c_str(), runtime.state.project_db, + LARDON3D_GEOMETRIC_VERIFIER_FUNDAMENTAL, LARDON3D_GEOMETRIC_VERIFIER_VERSION, + verifier_fingerprint, verified_ids.data(), verified_ids.size()}; + uint64_t task_id = 0; + if (!lardon3d_project_enqueue_track_builder_task(&runtime.state, &configuration, &task_id) || + !wait_completed(runtime, task_id, "track_builder.run")) + return false; + Lardon3DProjectDbTrackBuilderTask durable{}; + Lardon3DProjectDbTrackSet identity{}; + if (lardon3d_project_db_load_track_builder_task(runtime.state.project_db, task_id, &durable) != + LARDON3D_PROJECT_DB_OK) + return false; + std::memcpy(identity.builder_kind, durable.builder_kind, sizeof(identity.builder_kind)); + identity.builder_version = durable.builder_version; + std::memcpy(identity.parameter_fingerprint, durable.builder_fingerprint, + sizeof(identity.parameter_fingerprint)); + identity.verifier_kind = durable.verifier_kind; + identity.verifier_version = durable.verifier_version; + std::memcpy(identity.verifier_fingerprint, durable.verifier_fingerprint, + sizeof(identity.verifier_fingerprint)); + std::memcpy(identity.input_scope_hash, durable.input_scope_hash, + sizeof(identity.input_scope_hash)); + identity.gvr_count = durable.gvr_count; + Lardon3DProjectDbTrackSet published{}; + if (lardon3d_project_db_find_track_set(runtime.state.project_db, &identity, &published) != + LARDON3D_PROJECT_DB_OK) + return false; + if (task_id_output) *task_id_output = task_id; + if (track_set_output) *track_set_output = published; + return true; +} + +bool collect_evidence(Runtime &runtime, uint64_t execution_id, Evidence &evidence) { + Lardon3DProjectDbSelectedExecution execution{}; + if (execution_id != 0 && + lardon3d_project_db_load_selected_execution(runtime.state.project_db, execution_id, + &execution) == LARDON3D_PROJECT_DB_OK) + evidence.selected = execution.item_count; + uint64_t cursor = 0; + for (;;) { + Lardon3DProjectDbFeatureSet page[64]{}; + size_t count = 0; + if (lardon3d_project_db_list_feature_sets(runtime.state.project_db, cursor, page, 64, &count) != + LARDON3D_PROJECT_DB_OK) + return false; + evidence.features += count; + if (count) cursor = page[count - 1].feature_set_id; + if (count < 64) break; + } + cursor = 0; + for (;;) { + Lardon3DProjectDbCandidatePair page[64]{}; + size_t count = 0; + if (lardon3d_project_db_list_candidate_pairs(runtime.state.project_db, cursor, page, 64, + &count) != LARDON3D_PROJECT_DB_OK) + return false; + evidence.pairs += count; + if (count) cursor = page[count - 1].candidate_pair_id; + if (count < 64) break; + } + cursor = 0; + for (;;) { + Lardon3DProjectDbMatchResult page[64]{}; + size_t count = 0; + if (lardon3d_project_db_list_match_results(runtime.state.project_db, cursor, page, 64, + &count) != LARDON3D_PROJECT_DB_OK) + return false; + evidence.matches += count; + for (size_t index = 0; index < count; ++index) { + Lardon3DProjectDbGeometricVerificationResult geometries[4]{}; + size_t geometry_count = 0; + if (lardon3d_project_db_list_geometric_verification_results( + runtime.state.project_db, page[index].match_result_id, 0, geometries, 4, + &geometry_count) != LARDON3D_PROJECT_DB_OK) + return false; + for (size_t geometry = 0; geometry < geometry_count; ++geometry) { + if (geometries[geometry].status == LARDON3D_GEOMETRIC_VERIFIED) ++evidence.verified; + else ++evidence.rejected; + } + } + if (count) cursor = page[count - 1].match_result_id; + if (count < 64) break; + } + Lardon3DProjectDbTrackSet tracks[8]{}; + size_t track_set_count = 0; + if (lardon3d_project_db_list_track_sets(runtime.state.project_db, 0, tracks, 8, + &track_set_count) != LARDON3D_PROJECT_DB_OK) + return false; + for (size_t index = 0; index < track_set_count; ++index) evidence.tracks += tracks[index].track_count; + return true; +} + +bool count_exact_gvrs(Runtime &runtime, const unsigned char fingerprint[32], + size_t &count) { + count = 0; + uint64_t match_cursor = 0; + for (;;) { + Lardon3DProjectDbMatchResult matches[64]{}; + size_t match_count = 0; + if (lardon3d_project_db_list_match_results(runtime.state.project_db, match_cursor, matches, 64, + &match_count) != LARDON3D_PROJECT_DB_OK) + return false; + for (size_t index = 0; index < match_count; ++index) { + match_cursor = matches[index].match_result_id; + uint64_t geometry_cursor = 0; + for (;;) { + Lardon3DProjectDbGeometricVerificationResult page[8]{}; + size_t geometry_count = 0; + if (lardon3d_project_db_list_geometric_verification_results( + runtime.state.project_db, matches[index].match_result_id, geometry_cursor, page, 8, + &geometry_count) != LARDON3D_PROJECT_DB_OK) + return false; + for (size_t geometry = 0; geometry < geometry_count; ++geometry) { + if (page[geometry].verifier_kind == LARDON3D_GEOMETRIC_VERIFIER_FUNDAMENTAL && + page[geometry].verifier_version == LARDON3D_GEOMETRIC_VERIFIER_VERSION && + std::memcmp(page[geometry].parameter_fingerprint, fingerprint, 32) == 0) + ++count; + geometry_cursor = page[geometry].geometric_verification_result_id; + } + if (geometry_count < 8) break; + } + } + if (match_count < 64) break; + } + return true; +} + +bool find_pending_pre_gv_task(Runtime &runtime, uint64_t &candidate_task_id, + uint64_t &matcher_task_id) { + candidate_task_id = 0; + matcher_task_id = 0; + uint64_t cursor = 0; + for (;;) { + Lardon3DProjectRecoveryEntry entries[8]{}; + size_t count = 0; + if (lardon3d_project_list_recoverable( + &runtime.state, lardon3d_task_kind_registry_production(), cursor, + entries, 8, &count) != LARDON3D_PROJECT_DB_OK) + return false; + for (size_t index = 0; index < count; ++index) { + const auto &entry = entries[index]; + cursor = entry.task_id; + if ((entry.status != LARDON3D_PROJECT_RECOVERABLE && + entry.status != + LARDON3D_PROJECT_RECOVERABLE_PUBLISHED_NOT_DURABLE) || + entry.snapshot.recovery_state != TASK_PENDING) { + std::fprintf(stderr, + "pending task %llu is not safely recoverable (status=%d)\n", + static_cast(entry.task_id), + static_cast(entry.status)); + return false; + } + if (std::strcmp(entry.task_kind, + LARDON3D_CANDIDATE_PAIR_GENERATE_TASK_KIND) == 0) { + if (candidate_task_id != 0) return false; + candidate_task_id = entry.task_id; + } else if (std::strcmp(entry.task_kind, LARDON3D_MATCHER_TASK_KIND) == 0) { + if (matcher_task_id != 0) return false; + matcher_task_id = entry.task_id; + } else { + // The production recovery API resumes every pending task. Refuse an + // unrelated task rather than replaying an earlier pipeline stage. + std::fprintf(stderr, + "PRE-GV recovery refuses pending task %llu of kind %s\n", + static_cast(entry.task_id), + entry.task_kind); + return false; + } + } + if (count < 8) break; + } + return !(candidate_task_id != 0 && matcher_task_id != 0); +} + +bool recover_one_pre_gv_task(Runtime &runtime, uint64_t task_id, + const char *phase, + Lardon3DProjectRecoverySummary &recovery) { + recovery = {}; + if (lardon3d_project_resume_recoverable_tasks( + &runtime.state, lardon3d_task_kind_registry_production(), + &recovery) != LARDON3D_PROJECT_DB_OK || + recovery.resumed != 1 || recovery.failed != 0 || recovery.queue_full) + return false; + return wait_completed(runtime, task_id, phase); +} + +bool find_pending_candidate_only_task(Runtime &runtime, uint64_t &candidate_task_id) { + candidate_task_id = 0; + uint64_t cursor = 0; + for (;;) { + Lardon3DProjectRecoveryEntry entries[8]{}; + size_t count = 0; + if (lardon3d_project_list_recoverable( + &runtime.state, lardon3d_task_kind_registry_production(), cursor, + entries, 8, &count) != LARDON3D_PROJECT_DB_OK) + return false; + for (size_t index = 0; index < count; ++index) { + const auto &entry = entries[index]; + cursor = entry.task_id; + if ((entry.status != LARDON3D_PROJECT_RECOVERABLE && + entry.status != LARDON3D_PROJECT_RECOVERABLE_PUBLISHED_NOT_DURABLE) || + entry.snapshot.recovery_state != TASK_PENDING || + std::strcmp(entry.task_kind, + LARDON3D_CANDIDATE_PAIR_GENERATE_TASK_KIND) != 0 || + candidate_task_id != 0) { + std::fprintf(stderr, + "candidate-only recovery refuses unsafe pending task %llu of kind %s\n", + static_cast(entry.task_id), entry.task_kind); + return false; + } + candidate_task_id = entry.task_id; + } + if (count < 8) break; + } + // An absent entry is intentionally rejected: it means the requested pending + // Candidate boundary is missing or already complete, neither of which this + // repair mode may reinterpret or replace. + return candidate_task_id != 0; +} + +// This repair route owns exactly one durable Candidate Task. Registry recovery +// enqueues through the production Queue and Governor, but the preflight makes +// that all-task API safe by admitting no Matcher, GV, Track, SfM, or prior work. +bool run_existing_candidate(Runtime &runtime) { + if (!start_runtime(runtime)) return false; + Evidence before{}; + uint64_t candidate_task_id = 0; + if (!collect_evidence(runtime, 0, before) || + !find_pending_candidate_only_task(runtime, candidate_task_id)) { + stop_runtime(runtime); + return false; + } + Lardon3DProjectRecoverySummary recovery{}; + const bool recovered = recover_one_pre_gv_task( + runtime, candidate_task_id, "candidate_pair.generate recovered", recovery); + Evidence after{}; + const bool ok = recovered && collect_evidence(runtime, 0, after) && + after.features == before.features && after.pairs >= before.pairs && + after.matches == before.matches && after.verified == before.verified && + after.rejected == before.rejected && after.tracks == before.tracks; + std::printf( + "{\"record\":\"existing_candidate_summary\",\"ok\":%s," + "\"candidate_task_id\":%llu,\"candidate_action\":\"recovered\"," + "\"candidate_recovery_inspected\":%zu,\"candidate_recovery_resumed\":%zu," + "\"cpu_budget\":%u,\"feature_sets_before\":%zu,\"feature_sets_after\":%zu," + "\"candidate_pairs_before\":%zu,\"candidate_pairs_after\":%zu," + "\"match_results_before\":%zu,\"match_results_after\":%zu," + "\"gvrs_before\":%zu,\"gvrs_after\":%zu,\"tracks_before\":%zu," + "\"tracks_after\":%zu,\"prior_stages_replayed\":false," + "\"matcher_enqueued\":false,\"gv_enqueued\":false," + "\"track_builder_enqueued\":false,\"sparse_sfm_run\":false}\n", + ok ? "true" : "false", static_cast(candidate_task_id), + recovery.inspected, recovery.resumed, runtime.cpu_budget, before.features, + after.features, before.pairs, after.pairs, before.matches, after.matches, + before.verified + before.rejected, after.verified + after.rejected, + before.tracks, after.tracks); + stop_runtime(runtime); + return ok; +} + +// This repair route owns only the already-durable Candidate/Matcher boundary. +// It may recover those production Tasks or enqueue the missing Matcher, but it +// must never reconstruct prior acquisition stages or cross into GV/Tracks/SfM. +bool run_existing_pre_gv(Runtime &runtime) { + if (!start_runtime(runtime)) return false; + Evidence before{}; + if (!collect_evidence(runtime, 0, before) || before.verified != 0 || + before.rejected != 0 || before.tracks != 0) { + std::fprintf(stderr, + "PRE-GV mode requires a project with no GV or Track evidence\n"); + stop_runtime(runtime); + return false; + } + + uint64_t candidate_task_id = 0; + uint64_t pending_matcher_task_id = 0; + if (!find_pending_pre_gv_task(runtime, candidate_task_id, + pending_matcher_task_id)) { + stop_runtime(runtime); + return false; + } + Lardon3DProjectRecoverySummary candidate_recovery{}; + bool candidate_recovered = false; + if (candidate_task_id != 0) { + if (!recover_one_pre_gv_task(runtime, candidate_task_id, + "candidate_pair.generate recovered", + candidate_recovery)) { + stop_runtime(runtime); + return false; + } + candidate_recovered = true; + } + + Evidence after_candidate{}; + if (!collect_evidence(runtime, 0, after_candidate) || + after_candidate.matches > after_candidate.pairs || + after_candidate.verified != before.verified || + after_candidate.rejected != before.rejected || + after_candidate.tracks != before.tracks) { + stop_runtime(runtime); + return false; + } + + uint64_t newly_pending_candidate = 0; + uint64_t matcher_task_id = 0; + if (!find_pending_pre_gv_task(runtime, newly_pending_candidate, + matcher_task_id) || + newly_pending_candidate != 0) { + stop_runtime(runtime); + return false; + } + if (pending_matcher_task_id != 0) matcher_task_id = pending_matcher_task_id; + + Lardon3DProjectRecoverySummary matcher_recovery{}; + const char *matcher_action = "already_complete"; + if (after_candidate.matches < after_candidate.pairs) { + if (matcher_task_id != 0) { + if (!recover_one_pre_gv_task(runtime, matcher_task_id, + "matcher.run recovered", + matcher_recovery)) { + stop_runtime(runtime); + return false; + } + matcher_action = "recovered"; + } else { + const Lardon3DFeatureExtractorParameters orb{8192, 8, 20}; + Lardon3DMatcherTaskConfiguration matcher{}; + std::snprintf(matcher.feature_extractor_kind, + sizeof(matcher.feature_extractor_kind), "%s", + LARDON3D_FEATURE_EXTRACTOR_KIND); + matcher.feature_extractor_version = LARDON3D_FEATURE_EXTRACTOR_VERSION; + lardon3d_feature_extractor_parameter_fingerprint( + &orb, matcher.feature_parameter_fingerprint); + matcher.matcher.kind = LARDON3D_MATCHER_ORB_BF; + matcher.matcher.ratio_threshold = + lardon3d_matcher_default_ratio(LARDON3D_MATCHER_ORB_BF); + if (!lardon3d_project_enqueue_matcher_task(&runtime.state, &matcher, + &matcher_task_id) || + !wait_completed(runtime, matcher_task_id, "matcher.run existing")) { + stop_runtime(runtime); + return false; + } + matcher_action = "enqueued"; + } + } else if (matcher_task_id != 0) { + std::fprintf(stderr, + "complete Match Results coexist with a pending Matcher task\n"); + stop_runtime(runtime); + return false; + } + + Evidence after{}; + const bool ok = collect_evidence(runtime, 0, after) && + after.features == before.features && + after.pairs >= before.pairs && after.matches == after.pairs && + after.verified == before.verified && + after.rejected == before.rejected && + after.tracks == before.tracks; + std::printf( + "{\"record\":\"existing_pre_gv_summary\",\"ok\":%s," + "\"candidate_task_id\":%llu,\"candidate_action\":\"%s\"," + "\"candidate_recovery_inspected\":%zu," + "\"candidate_recovery_resumed\":%zu,\"matcher_task_id\":%llu," + "\"matcher_action\":\"%s\",\"matcher_recovery_resumed\":%zu," + "\"feature_sets_before\":%zu,\"feature_sets_after\":%zu," + "\"candidate_pairs_before\":%zu,\"candidate_pairs_after\":%zu," + "\"match_results_before\":%zu,\"match_results_after\":%zu," + "\"gvrs_before\":%zu,\"gvrs_after\":%zu," + "\"tracks_before\":%zu,\"tracks_after\":%zu," + "\"prior_stages_replayed\":false,\"gv_enqueued\":false," + "\"track_builder_enqueued\":false,\"sparse_sfm_run\":false}\n", + ok ? "true" : "false", + static_cast(candidate_task_id), + candidate_recovered ? "recovered" : "already_complete", + candidate_recovery.inspected, candidate_recovery.resumed, + static_cast(matcher_task_id), matcher_action, + matcher_recovery.resumed, before.features, after.features, before.pairs, + after.pairs, before.matches, after.matches, + before.verified + before.rejected, after.verified + after.rejected, + before.tracks, after.tracks); + stop_runtime(runtime); + return ok; +} + +bool run_existing_geometry(Runtime &runtime) { + if (!start_runtime(runtime)) return false; + Lardon3DGeometricVerifierTaskConfiguration configuration{ + lardon3d_geometric_verifier_default_parameters()}; + unsigned char fingerprint[32]{}; + lardon3d_geometric_verifier_fingerprint(&configuration.verifier, fingerprint); + size_t exact_before = 0; + if (!count_exact_gvrs(runtime, fingerprint, exact_before)) { + stop_runtime(runtime); + return false; + } + + /* This route deliberately starts at the immutable Match Result boundary. + * It enqueues no quality, campaign, representation, feature, index, + * candidate, or Matcher work. Task/Queue/Governor retain their production + * ownership, and exact current identity reuse makes a full rerun convergent. */ + uint64_t geometry_task_id = 0; + if (!lardon3d_project_enqueue_geometric_verifier_task( + &runtime.state, &configuration, &geometry_task_id) || + !wait_completed(runtime, geometry_task_id, + "geometric_verifier.run current existing")) { + std::printf("{\"record\":\"existing_geometry_summary\",\"ok\":false," + "\"stage\":\"geometric_verifier.run\",\"task_errors\":1}\n"); + stop_runtime(runtime); + return false; + } + + std::vector verified_ids; + size_t applicable = 0; + size_t verified = 0; + size_t rejected = 0; + size_t exact_after = 0; + if (!collect_verified_ids(runtime, fingerprint, verified_ids, &applicable, &verified, + &rejected) || + !count_exact_gvrs(runtime, fingerprint, exact_after) || exact_after != applicable) { + std::printf("{\"record\":\"existing_geometry_summary\",\"ok\":false," + "\"stage\":\"current_evidence_audit\",\"task_errors\":1}\n"); + stop_runtime(runtime); + return false; + } + + // Track Builder receives only exact current-policy VERIFIED IDs. Historical + // v1/v2 evidence is neither replayed nor mixed, and the exact scope identity + // reuses an existing Track Set instead of publishing duplicate lineage. + uint64_t track_task_id = 0; + Lardon3DProjectDbTrackSet track_set{}; + if (!build_tracks(runtime, verified_ids, fingerprint, &track_task_id, &track_set)) { + std::printf("{\"record\":\"existing_geometry_summary\",\"ok\":false," + "\"stage\":\"track_builder.run\",\"task_errors\":1}\n"); + stop_runtime(runtime); + return false; + } + + const size_t created = exact_after >= exact_before ? exact_after - exact_before : 0; + std::printf( + "{\"record\":\"existing_geometry_summary\",\"ok\":true," + "\"verifier_version\":%u,\"geometry_task_id\":%llu," + "\"track_task_id\":%llu,\"applicable_matched_parents\":%zu," + "\"verified\":%zu,\"rejected\":%zu,\"gvrs_before\":%zu," + "\"gvrs_created\":%zu,\"gvrs_after\":%zu," + "\"track_set_id\":%llu,\"track_count\":%llu," + "\"track_set_reused\":%s," + "\"task_errors\":0,\"upstream_replayed\":false," + "\"mixed_verifier_lineage\":false,\"sparse_sfm_run\":false}\n", + LARDON3D_GEOMETRIC_VERIFIER_VERSION, + static_cast(geometry_task_id), + static_cast(track_task_id), applicable, verified, rejected, exact_before, + created, exact_after, static_cast(track_set.track_set_id), + static_cast(track_set.track_count), + track_task_id == 0 && track_set.track_set_id != 0 ? "true" : "false"); + stop_runtime(runtime); + return true; +} + +} // namespace + +int main(int argc, char **argv) { + Options options; + if (!parse_options(argc, argv, options)) { + usage(argv[0]); + return 2; + } + Runtime runtime; + if (options.resume_geometry_existing || options.resume_pre_gv_existing || + options.resume_candidate_existing) { + if (!prepare_existing_project(options.project_dir, runtime)) { + std::fprintf(stderr, "existing project must be absolute, normalized, and durable\n"); + return 2; + } + runtime.cpu_budget = options.cpu_budget; + if (options.resume_candidate_existing) + return run_existing_candidate(runtime) ? 0 : 1; + return (options.resume_pre_gv_existing ? run_existing_pre_gv(runtime) + : run_existing_geometry(runtime)) ? 0 : 1; + } + if (!prepare_empty_project(options.project_dir, runtime.project_path)) { + std::fprintf(stderr, "project directory must be absolute, normalized and empty\n"); + return 2; + } + runtime.database_path = (options.project_dir / "project.lardon3d").string(); + Campaign campaign; + if (!discover_campaign(options, campaign) || !start_runtime(runtime)) { + std::fprintf(stderr, "campaign discovery or runtime setup failed\n"); + return 1; + } + + const char *mode_name = options.mode == Mode::kA6000 ? "a6000" : "s21"; + Lardon3DProjectDbScanSet scanset{}; + if (lardon3d_project_db_create_scanset(runtime.state.project_db, mode_name, &scanset) != + LARDON3D_PROJECT_DB_OK) { + stop_runtime(runtime); + return 1; + } + uint64_t quality_task_id = 0; + uint64_t campaign_task_id = 0; + Lardon3DProjectDbSelectedExecution execution{}; + bool ok = run_task_pair(runtime, scanset.scanset_id, campaign, + options.restart != RestartBoundary::kNone, quality_task_id, + campaign_task_id) && + create_selection(runtime, options.mode, quality_task_id, campaign_task_id, + campaign.confirmations.size(), execution); + if (!ok || execution.execution_id == 0) { + stop_runtime(runtime); + return ok ? 0 : 1; + } + + std::vector image_ids; + ok = publish_representations(runtime, execution.execution_id, image_ids); + if (ok && options.restart == RestartBoundary::kRepresentations) + ok = restart_runtime(runtime, execution.execution_id, "representations") && + publish_representations(runtime, execution.execution_id, image_ids); + + /* ORB is run only for exact images retained by the selected execution. The + * 4096 option limit is an operational Visual Index v1 bound, not a scientific + * campaign-size rule; callers use one empty project per campaign. */ + const Lardon3DFeatureExtractorParameters orb{8192, 8, 20}; + std::vector features; + if (ok) ok = extract_features(runtime, image_ids, orb, features); + if (ok && options.restart == RestartBoundary::kFeatures) { + ok = restart_runtime(runtime, execution.execution_id, "features") && + publish_representations(runtime, execution.execution_id, image_ids) && + extract_features(runtime, image_ids, orb, features); + } + + uint64_t geometry_task_id = 0; + std::vector verified_ids; + Lardon3DGeometricVerifierParameters verifier = lardon3d_geometric_verifier_default_parameters(); + unsigned char verifier_fingerprint[32]{}; + lardon3d_geometric_verifier_fingerprint(&verifier, verifier_fingerprint); + if (ok) ok = downstream(runtime, features, orb, geometry_task_id, verified_ids); + if (ok && options.restart == RestartBoundary::kGeometry) { + ok = restart_runtime(runtime, execution.execution_id, "geometry"); + if (ok) ok = collect_verified_ids(runtime, verifier_fingerprint, verified_ids); + } + if (ok) ok = build_tracks(runtime, verified_ids, verifier_fingerprint); + if (ok && verified_ids.empty()) + std::printf("{\"record\":\"tracks\",\"created\":false," + "\"reason\":\"empty verified GVR set\"}\n"); + + /* Real A6000/S21 calibration is unavailable. The selected execution remains + * truthfully at CALIBRATION, and this runner never fabricates a scope or runs + * sparse SfM; verified geometry and tracks are the terminal evidence. */ + if (ok) { + stop_runtime(runtime); + ok = start_runtime(runtime); + } + Evidence evidence{}; + if (ok) ok = collect_evidence(runtime, execution.execution_id, evidence); + if (ok) { + std::printf("{\"record\":\"summary\",\"mode\":\"%s\"," + "\"execution_id\":%llu,\"selected\":%zu,\"feature_sets\":%zu," + "\"candidate_pairs\":%zu,\"matches\":%zu,\"verified\":%zu," + "\"rejected\":%zu,\"tracks\":%zu," + "\"orb_max_features_per_set\":8192," + "\"visual_index_max_features_per_set\":1024," + "\"visual_index_feature_set_limit\":4096," + "\"candidate_top_k_limit\":64," + "\"calibration_attached\":false,\"sparse_sfm_run\":false}\n", + mode_name, static_cast(execution.execution_id), + evidence.selected, evidence.features, evidence.pairs, evidence.matches, + evidence.verified, evidence.rejected, evidence.tracks); + } + stop_runtime(runtime); + return ok ? 0 : 1; +} diff --git a/tests/test_candidate_pair_gen.c b/tests/test_candidate_pair_gen.c index d4ac91b..3eaeac7 100644 --- a/tests/test_candidate_pair_gen.c +++ b/tests/test_candidate_pair_gen.c @@ -9,12 +9,16 @@ #include #include +#include + #include #include #include #include #include +#include "../src/candidate_pair_gen_internal.h" + #define CHECK(condition) \ do { \ if (!(condition)) { \ @@ -106,6 +110,25 @@ static bool pair_exists(const Lardon3DProjectDbCandidatePair *pairs, size_t coun return false; } +static bool retain_sparse_memberships(const char *database_path, uint64_t index_id, + uint64_t first_id, uint64_t last_id) { + sqlite3 *database = NULL; + if (sqlite3_open_v2(database_path, &database, SQLITE_OPEN_READWRITE, NULL) != SQLITE_OK) { + sqlite3_close(database); + return false; + } + char sql[256]; + int n = snprintf(sql, sizeof(sql), + "DELETE FROM visual_index_memberships WHERE visual_index_id=%lu " + "AND feature_set_id NOT IN(%lu,%lu)", + (unsigned long)index_id, (unsigned long)first_id, + (unsigned long)last_id); + bool ok = n > 0 && (size_t)n < sizeof(sql) && + sqlite3_exec(database, sql, NULL, NULL, NULL) == SQLITE_OK; + sqlite3_close(database); + return ok; +} + static bool run_test(void) { char root[] = "/tmp/lardon3d-candidate-pair-gen-XXXXXX"; CHECK(mkdtemp(root)); @@ -298,6 +321,23 @@ static bool run_test(void) { CHECK(lardon3d_candidate_pair_generate(root, database, index_id, 999999, &options, &stats) == LARDON3D_VISUAL_INDEX_NOT_FOUND); + /* A staged duplicate retains the established find-before-create semantics: + * the first proposal creates and the second is accounted as skipped. */ + uint64_t duplicate_a = images[0].image_id < images[4].image_id + ? images[0].image_id : images[4].image_id; + uint64_t duplicate_b = images[0].image_id < images[4].image_id + ? images[4].image_id : images[0].image_id; + Lardon3DCandidatePairComputation duplicate = { + .source_feature_set_id = feature_sets[0].feature_set_id, + .queried_count = 2, + .proposal_count = 2, + .proposals = {{duplicate_a, duplicate_b}, {duplicate_a, duplicate_b}}, + }; + CHECK(lardon3d_candidate_pair_publish(database, &duplicate, &stats) == + LARDON3D_VISUAL_INDEX_OK && + stats.queried_count == 2 && stats.generated_count == 1 && + stats.skipped_count == 1); + /* Génération batch : tous les FeatureSets. */ Lardon3DCandidatePairGenStats batch_stats; uint64_t last_feature_set_id = 0; @@ -324,8 +364,34 @@ static bool run_test(void) { &found) == LARDON3D_PROJECT_DB_OK && found.image_id_a == images[0].image_id && found.image_id_b == images[1].image_id); lardon3d_project_db_close(database); + + /* Membership progress is a row rank, not arithmetic on sparse IDs. This + * fixture intentionally retains only the first and last indexed source. */ + CHECK(retain_sparse_memberships(database_path, index_id, + feature_sets[0].feature_set_id, + feature_sets[5].feature_set_id) && + lardon3d_project_db_open(database_path, &database, error) == + LARDON3D_PROJECT_DB_OK); + uint64_t membership_ids[4] = {0}; + size_t membership_count = 0; + uint64_t completed = 0, total = 0; + CHECK(lardon3d_project_db_list_visual_index_memberships( + database, index_id, 0, membership_ids, 4, &membership_count) == + LARDON3D_PROJECT_DB_OK && + membership_count == 2 && + membership_ids[0] == feature_sets[0].feature_set_id && + membership_ids[1] == feature_sets[5].feature_set_id); + CHECK(lardon3d_project_db_visual_index_membership_progress( + database, index_id, feature_sets[0].feature_set_id, &completed, &total) == + LARDON3D_PROJECT_DB_OK && + completed == 1 && total == 2); + CHECK(lardon3d_project_db_visual_index_membership_progress( + database, index_id, feature_sets[5].feature_set_id, &completed, &total) == + LARDON3D_PROJECT_DB_OK && + completed == 2 && total == 2); + lardon3d_project_db_close(database); CHECK(remove_tree(root)); return true; } -int main(void) { return run_test() ? 0 : 1; } \ No newline at end of file +int main(void) { return run_test() ? 0 : 1; } diff --git a/tests/test_candidate_pair_task.c b/tests/test_candidate_pair_task.c index 012bd19..3f3aaa5 100644 --- a/tests/test_candidate_pair_task.c +++ b/tests/test_candidate_pair_task.c @@ -12,6 +12,8 @@ #include #include +#include + #include #include #include @@ -19,10 +21,14 @@ #include #include #include +#include #include #include #include +#include "src/candidate_pair_gen_internal.h" +#include "resource_snapshot_test_utils.h" + #define CHECK(x) \ do { \ if (!(x)) { \ @@ -115,16 +121,95 @@ static bool wait_state(Lardon3DTaskQueue *q, uint64_t id, } } +static bool reset_candidate_pairs(Lardon3DProjectDb *project_database) { + char database_path[LARDON3D_PROJECT_DB_PATH_CAPACITY]; + if (!lardon3d_project_db_copy_path(project_database, database_path)) return false; + sqlite3 *database = NULL; + if (sqlite3_open_v2(database_path, &database, SQLITE_OPEN_READWRITE, NULL) != SQLITE_OK) { + sqlite3_close(database); + return false; + } + (void)sqlite3_busy_timeout(database, 5000); + char *error = NULL; + bool ok = sqlite3_exec( + database, + "BEGIN; DELETE FROM candidate_pairs; " + "DELETE FROM sqlite_sequence WHERE name='candidate_pairs'; COMMIT;", + NULL, NULL, &error) == SQLITE_OK; + if (!ok) fprintf(stderr, "reset Candidate Pair: %s\n", error ? error : "unknown"); + sqlite3_free(error); + sqlite3_close(database); + return ok; +} + +static bool load_pairs(Lardon3DProjectDb *database, + Lardon3DProjectDbCandidatePair pairs[32], size_t *count) { + return lardon3d_project_db_list_candidate_pairs(database, 0, pairs, 32, count) == + LARDON3D_PROJECT_DB_OK; +} + +static bool same_pair_order(const Lardon3DProjectDbCandidatePair *a, size_t a_count, + const Lardon3DProjectDbCandidatePair *b, size_t b_count) { + if (a_count != b_count) return false; + for (size_t i = 0; i < a_count; ++i) { + if (a[i].image_id_a != b[i].image_id_a || a[i].image_id_b != b[i].image_id_b) { + return false; + } + } + return true; +} + +static bool wait_saved_cursor(Lardon3DProjectDb *database, uint64_t task_id, + uint64_t expected) { + struct timespec deadline; + if (clock_gettime(CLOCK_MONOTONIC, &deadline) != 0) return false; + deadline.tv_sec += 5; + for (;;) { + Lardon3DProjectDbCandidatePairGenerateTask saved; + if (lardon3d_project_db_load_candidate_pair_generate_task( + database, task_id, &saved) == LARDON3D_PROJECT_DB_OK && + saved.after_feature_set_id == expected) { + return true; + } + struct timespec now; + if (clock_gettime(CLOCK_MONOTONIC, &now) != 0 || + now.tv_sec > deadline.tv_sec || + (now.tv_sec == deadline.tv_sec && now.tv_nsec >= deadline.tv_nsec)) { + return false; + } + sched_yield(); + } +} + +static bool same_estimate(const Lardon3DResourceEstimate *left, + const Lardon3DResourceEstimate *right) { + return left->memory_fixed_bytes == right->memory_fixed_bytes && + left->gpu_memory_fixed_bytes == right->gpu_memory_fixed_bytes && + left->memory_bytes_per_item == right->memory_bytes_per_item && + left->gpu_memory_bytes_per_item == right->gpu_memory_bytes_per_item && + left->minimum_batch_size == right->minimum_batch_size && + left->maximum_batch_size == right->maximum_batch_size && + left->desired_cpu_threads == right->desired_cpu_threads && + left->desired_gpu_slots == right->desired_gpu_slots && + left->desired_io_slots == right->desired_io_slots && + left->task_class == right->task_class; +} + static bool run_test(void) { char root[] = "/tmp/lardon3d-candidate-pair-task-XXXXXX"; CHECK(mkdtemp(root) && setenv("LARDON3D_PROJECTS_ROOT", root, 1) == 0); - /* --- Créer 3 images distinctes --- */ + /* Six FeatureSets let the test cross three real Task/Queue/Governor + * sequence boundaries when its test-only admitted maximum is two. */ char source_a[PATH_MAX], source_b[PATH_MAX], source_c[PATH_MAX]; + char source_d[PATH_MAX], source_e[PATH_MAX], source_f[PATH_MAX]; CHECK(join_path(source_a, root, "a.pgm") && write_pgm(source_a, 1)); CHECK(join_path(source_b, root, "b.pgm") && write_pgm(source_b, 2)); CHECK(join_path(source_c, root, "c.pgm") && write_pgm(source_c, 3)); + CHECK(join_path(source_d, root, "d.pgm") && write_pgm(source_d, 4)); + CHECK(join_path(source_e, root, "e.pgm") && write_pgm(source_e, 5)); + CHECK(join_path(source_f, root, "f.pgm") && write_pgm(source_f, 6)); Lardon3DAppState state; lardon3d_app_state_init(&state); @@ -133,8 +218,8 @@ static bool run_test(void) { Lardon3DProjectDbScanSet scanset; CHECK(lardon3d_image_catalog_create_scanset(&state, "S", &scanset)); - Lardon3DProjectDbImage img_a, img_b, img_c; - Lardon3DProjectDbImageAsset asset_a, asset_b, asset_c; + Lardon3DProjectDbImage img_a, img_b, img_c, img_d, img_e, img_f; + Lardon3DProjectDbImageAsset asset_a, asset_b, asset_c, asset_d, asset_e, asset_f; CHECK(lardon3d_image_catalog_import_file(&state, scanset.scanset_id, source_a, 0, &img_a, &asset_a) == @@ -146,21 +231,42 @@ static bool run_test(void) { lardon3d_image_catalog_import_file(&state, scanset.scanset_id, source_c, 0, &img_c, &asset_c) == + LARDON3D_IMAGE_CATALOG_IMPORTED && + lardon3d_image_catalog_import_file(&state, scanset.scanset_id, + source_d, 0, &img_d, + &asset_d) == + LARDON3D_IMAGE_CATALOG_IMPORTED && + lardon3d_image_catalog_import_file(&state, scanset.scanset_id, + source_e, 0, &img_e, + &asset_e) == + LARDON3D_IMAGE_CATALOG_IMPORTED && + lardon3d_image_catalog_import_file(&state, scanset.scanset_id, + source_f, 0, &img_f, + &asset_f) == LARDON3D_IMAGE_CATALOG_IMPORTED); /* Extraction feature ORB */ Lardon3DFeatureExtractorParameters orb_params = {512, 4, 10}; - uint64_t orb_a_id = 0, orb_b_id = 0, orb_c_id = 0; + uint64_t orb_a_id = 0, orb_b_id = 0, orb_c_id = 0, orb_d_id = 0, orb_e_id = 0, orb_f_id = 0; CHECK(lardon3d_project_enqueue_feature_extract( &state, img_a.image_id, &orb_params, &orb_a_id) && lardon3d_project_enqueue_feature_extract( &state, img_b.image_id, &orb_params, &orb_b_id) && lardon3d_project_enqueue_feature_extract( - &state, img_c.image_id, &orb_params, &orb_c_id)); + &state, img_c.image_id, &orb_params, &orb_c_id) && + lardon3d_project_enqueue_feature_extract( + &state, img_d.image_id, &orb_params, &orb_d_id) && + lardon3d_project_enqueue_feature_extract( + &state, img_e.image_id, &orb_params, &orb_e_id) && + lardon3d_project_enqueue_feature_extract( + &state, img_f.image_id, &orb_params, &orb_f_id)); Lardon3DTaskSnapshot snap; CHECK(wait_state(state.task_queue, orb_a_id, TASK_COMPLETED, &snap)); CHECK(wait_state(state.task_queue, orb_b_id, TASK_COMPLETED, &snap)); CHECK(wait_state(state.task_queue, orb_c_id, TASK_COMPLETED, &snap)); + CHECK(wait_state(state.task_queue, orb_d_id, TASK_COMPLETED, &snap)); + CHECK(wait_state(state.task_queue, orb_e_id, TASK_COMPLETED, &snap)); + CHECK(wait_state(state.task_queue, orb_f_id, TASK_COMPLETED, &snap)); unsigned char fp[32]; lardon3d_feature_extractor_parameter_fingerprint(&orb_params, fp); @@ -189,21 +295,108 @@ static bool run_test(void) { &state, visual_index_id, &vi_task_id)); CHECK(wait_state(state.task_queue, vi_task_id, TASK_COMPLETED, &snap)); - /* --- Test 1: Création task Candidate Pair --- */ + /* Candidate workers must follow the runtime-selected durable DB filename; + * project_path alone intentionally cannot identify that database. */ + char project_path[PATH_MAX], default_database_path[PATH_MAX]; + char selected_database_path[PATH_MAX]; + CHECK(snprintf(project_path, sizeof(project_path), "%s", state.project_path) > 0 && + join_path(default_database_path, project_path, "project.db") && + join_path(selected_database_path, project_path, "selected.lardon3d")); + lardon3d_task_queue_destroy(state.task_queue); + state.task_queue = NULL; + lardon3d_project_close(&state); + CHECK(rename(default_database_path, selected_database_path) == 0); + char database_error[LARDON3D_PROJECT_DB_ERROR_CAPACITY]; + CHECK(lardon3d_project_db_open(selected_database_path, &state.project_db, + database_error) == LARDON3D_PROJECT_DB_OK); + state.project_loaded = true; + CHECK(snprintf(state.project_path, sizeof(state.project_path), "%s", + project_path) > 0); + state.task_queue = lardon3d_task_queue_create(state.resource_governor, 4); + CHECK(state.task_queue != NULL); + + /* --- Test 1: déterminisme exact à 1/2/4 threads admis --- */ Lardon3DVisualIndexQueryOptions qopts = { .top_k = 16, .minimum_evidence_count = 10, .scanset_filter = LARDON3D_VISUAL_INDEX_ANY_SCANSET, .exclude_same_asset = false, }; - uint64_t cp_task_id = 0; - CHECK(lardon3d_project_enqueue_candidate_pair_generate( - &state, visual_index_id, &qopts, &cp_task_id)); - CHECK(cp_task_id != 0); - CHECK(wait_state(state.task_queue, cp_task_id, TASK_COMPLETED, &snap)); - CHECK(snap.progress == 100); - /* Vérification : des paires ont été générées */ + /* A healthy Governor admits two bounded sources per sequence. The six + * durable memberships create three productive windows: initial admission + * plus two productive re-admissions after sequence_break. Completion then + * re-admits once more only to observe the empty suffix. This proves that + * parallel fan-out is not a one-window startup effect. The production + * estimate remains 1..64; this test-only cap exercises the boundary. */ + CHECK(reset_candidate_pairs(state.project_db) && + setenv("LARDON3D_TEST_CANDIDATE_PAIR_THREADS", "2", 1) == 0 && + setenv("LARDON3D_TEST_CANDIDATE_PAIR_MAXIMUM_BATCH", "2", 1) == 0); + lardon3d_candidate_pair_task_test_reset_parallel_counters(); + uint64_t multi_sequence_task_id = 0; + CHECK(lardon3d_project_enqueue_candidate_pair_generate( + &state, visual_index_id, &qopts, &multi_sequence_task_id) && + wait_state(state.task_queue, multi_sequence_task_id, TASK_COMPLETED, &snap) && + snap.progress == 100 && + lardon3d_candidate_pair_task_test_started_participants() >= 6 && + lardon3d_candidate_pair_task_test_computed_work_items() >= 6); + Lardon3DProjectDbTask multi_sequence_task; + CHECK(lardon3d_project_db_load_task(state.project_db, multi_sequence_task_id, + &multi_sequence_task) == LARDON3D_PROJECT_DB_OK && + multi_sequence_task.sequence_count >= 3); + Lardon3DProjectDbCandidatePairGenerateTask multi_sequence_cursor; + CHECK(lardon3d_project_db_load_candidate_pair_generate_task( + state.project_db, multi_sequence_task_id, &multi_sequence_cursor) == + LARDON3D_PROJECT_DB_OK && + multi_sequence_cursor.after_feature_set_id != 0); + CHECK(unsetenv("LARDON3D_TEST_CANDIDATE_PAIR_MAXIMUM_BATCH") == 0 && + unsetenv("LARDON3D_TEST_CANDIDATE_PAIR_THREADS") == 0); + + /* CPU admission does not authorize extra work. With twelve available + * CPUs but an admitted one-item batch, each of the six sequence breaks + * has exactly one independent source and therefore one useful worker. */ + CHECK(reset_candidate_pairs(state.project_db) && + setenv("LARDON3D_TEST_CANDIDATE_PAIR_THREADS", "12", 1) == 0 && + setenv("LARDON3D_TEST_CANDIDATE_PAIR_MAXIMUM_BATCH", "1", 1) == 0); + lardon3d_candidate_pair_task_test_reset_parallel_counters(); + uint64_t pressure_task_id = 0; + CHECK(lardon3d_project_enqueue_candidate_pair_generate( + &state, visual_index_id, &qopts, &pressure_task_id) && + wait_state(state.task_queue, pressure_task_id, TASK_COMPLETED, &snap) && + lardon3d_candidate_pair_task_test_started_participants() == 6 && + lardon3d_candidate_pair_task_test_computed_work_items() == 6); + Lardon3DProjectDbTask pressure_task; + CHECK(lardon3d_project_db_load_task(state.project_db, pressure_task_id, + &pressure_task) == LARDON3D_PROJECT_DB_OK && + pressure_task.sequence_count >= 6); + CHECK(unsetenv("LARDON3D_TEST_CANDIDATE_PAIR_MAXIMUM_BATCH") == 0 && + unsetenv("LARDON3D_TEST_CANDIDATE_PAIR_THREADS") == 0); + + const char *thread_counts[] = {"1", "2", "4"}; + uint64_t cp_task_id = 0; + Lardon3DProjectDbCandidatePair baseline[32]; + size_t baseline_count = 0; + for (size_t run = 0; run < 3; ++run) { + CHECK(reset_candidate_pairs(state.project_db) && + setenv("LARDON3D_TEST_CANDIDATE_PAIR_THREADS", thread_counts[run], 1) == 0); + uint64_t current_task_id = 0; + CHECK(lardon3d_project_enqueue_candidate_pair_generate( + &state, visual_index_id, &qopts, ¤t_task_id)); + if (run == 0) cp_task_id = current_task_id; + CHECK(wait_state(state.task_queue, current_task_id, TASK_COMPLETED, &snap) && + snap.progress == 100); + Lardon3DProjectDbCandidatePair current[32]; + size_t current_count = 0; + CHECK(load_pairs(state.project_db, current, ¤t_count) && current_count > 0); + if (run == 0) { + memcpy(baseline, current, current_count * sizeof(*current)); + baseline_count = current_count; + } else { + CHECK(same_pair_order(baseline, baseline_count, current, current_count)); + } + } + CHECK(unsetenv("LARDON3D_TEST_CANDIDATE_PAIR_THREADS") == 0); + Lardon3DProjectDbCandidatePair pairs[16]; size_t pair_count = 0; CHECK(lardon3d_project_db_list_candidate_pairs( @@ -225,12 +418,51 @@ static bool run_test(void) { LARDON3D_PROJECT_DB_OK && pair_count2 == pair_count); - /* --- Test 3: Checkpoint / reprise --- */ - /* Pause après premier batch */ - CHECK(setenv("LARDON3D_TEST_CANDIDATE_PAIR_PAUSE_AFTER_BATCH", "1", + /* --- Test 3: échec first/middle/last et frontière contiguë --- */ + uint64_t ordered_sources[] = {fs_a.feature_set_id, fs_b.feature_set_id, + fs_c.feature_set_id}; + for (size_t failed = 0; failed < 3; ++failed) { + CHECK(reset_candidate_pairs(state.project_db)); + for (size_t prefix = 0; prefix < failed; ++prefix) { + Lardon3DCandidatePairGenStats prefix_stats; + CHECK(lardon3d_candidate_pair_generate( + state.project_path, state.project_db, visual_index_id, + ordered_sources[prefix], &qopts, &prefix_stats) == + LARDON3D_VISUAL_INDEX_OK); + } + Lardon3DProjectDbCandidatePair expected[32]; + size_t expected_count = 0; + CHECK(load_pairs(state.project_db, expected, &expected_count) && + reset_candidate_pairs(state.project_db)); + + char failed_id[32]; + (void)snprintf(failed_id, sizeof(failed_id), "%lu", + (unsigned long)ordered_sources[failed]); + CHECK(setenv("LARDON3D_TEST_CANDIDATE_PAIR_FAIL_SOURCE_ID", failed_id, 1) == 0 && + setenv("LARDON3D_TEST_CANDIDATE_PAIR_THREADS", "4", 1) == 0); + uint64_t failed_task_id = 0; + CHECK(lardon3d_project_enqueue_candidate_pair_generate( + &state, visual_index_id, &qopts, &failed_task_id) && + wait_state(state.task_queue, failed_task_id, TASK_FAILED, &snap)); + uint64_t expected_cursor = failed == 0 ? 0 : ordered_sources[failed - 1]; + CHECK(wait_saved_cursor(state.project_db, failed_task_id, expected_cursor)); + Lardon3DProjectDbCandidatePair actual[32]; + size_t actual_count = 0; + CHECK(load_pairs(state.project_db, actual, &actual_count) && + same_pair_order(expected, expected_count, actual, actual_count)); + } + CHECK(unsetenv("LARDON3D_TEST_CANDIDATE_PAIR_FAIL_SOURCE_ID") == 0 && + unsetenv("LARDON3D_TEST_CANDIDATE_PAIR_THREADS") == 0 && + reset_candidate_pairs(state.project_db)); + + /* --- Test 4: Checkpoint / reprise --- */ + /* Pause before the first window so recovery has enough independent + * sources to prove that its production callback actually fans out. */ + CHECK(setenv("LARDON3D_TEST_CANDIDATE_PAIR_PAUSE_BEFORE_BATCH", "1", 1) == 0 && setenv("LARDON3D_TEST_CANDIDATE_PAIR_SKIP_FINISHED_CHECKPOINT", - "1", 1) == 0); + "1", 1) == 0 && + setenv("LARDON3D_TEST_CANDIDATE_PAIR_THREADS", "1", 1) == 0); Lardon3DVisualIndexQueryOptions qopts_b = { .top_k = 8, .minimum_evidence_count = 0, @@ -247,10 +479,215 @@ static bool run_test(void) { lardon3d_project_close(&state); lardon3d_resource_governor_destroy(state.resource_governor); state.resource_governor = NULL; - CHECK(unsetenv("LARDON3D_TEST_CANDIDATE_PAIR_PAUSE_AFTER_BATCH") == 0 && + CHECK(rename(selected_database_path, default_database_path) == 0); + CHECK(unsetenv("LARDON3D_TEST_CANDIDATE_PAIR_PAUSE_BEFORE_BATCH") == 0 && unsetenv("LARDON3D_TEST_CANDIDATE_PAIR_SKIP_FINISHED_CHECKPOINT") == - 0); + 0 && + unsetenv("LARDON3D_TEST_CANDIDATE_PAIR_THREADS") == 0); + /* Persist the real Candidate v1 resource shape. A current 256 KiB task + * forced to one thread is a near miss, not a historical checkpoint. */ + char candidate_checkpoint_path[PATH_MAX]; + CHECK(snprintf(candidate_checkpoint_path, sizeof(candidate_checkpoint_path), + "%s/.lardon3d/checkpoints/%lu.chk", project_path, + (unsigned long)cp_task3_id) > 0); + Lardon3DTaskDurableSnapshot legacy_snapshot; + uint32_t checkpoint_version = 0; + CHECK(lardon3d_task_checkpoint_load(candidate_checkpoint_path, &legacy_snapshot, + &checkpoint_version) == + LARDON3D_TASK_CHECKPOINT_OK && + checkpoint_version == LARDON3D_TASK_CHECKPOINT_VERSION); + legacy_snapshot.estimate = (Lardon3DResourceEstimate){ + .memory_fixed_bytes = 128 * 1024, + .gpu_memory_fixed_bytes = 0, + .memory_bytes_per_item = 64 * 1024, + .gpu_memory_bytes_per_item = 0, + .minimum_batch_size = 1, + .maximum_batch_size = 64, + .desired_cpu_threads = 1, + .desired_gpu_slots = 0, + .desired_io_slots = 1, + .task_class = LARDON3D_RESOURCE_TASK_CPU, + }; + CHECK(lardon3d_task_checkpoint_save(candidate_checkpoint_path, &legacy_snapshot) == + LARDON3D_TASK_CHECKPOINT_OK); + + /* Candidate v1 recovery recognizes the exact historical serial estimate, + * then normalizes it privately before Queue admission. This operational + * estimate is not checkpointed, so Task identity and the typed cursor stay + * durable and unchanged. */ + lardon3d_app_state_init(&state); + CHECK(runtime(&state) && + lardon3d_project_db_open(default_database_path, &state.project_db, + database_error) == LARDON3D_PROJECT_DB_OK); + state.project_loaded = true; + CHECK(snprintf(state.project_path, sizeof(state.project_path), "%s", + project_path) > 0); + Lardon3DProjectRecoveryEntry recovery_entries[8]; + size_t recovery_count = 0; + CHECK(lardon3d_project_list_recoverable( + &state, lardon3d_task_kind_registry_production(), 0, + recovery_entries, 8, &recovery_count) == LARDON3D_PROJECT_DB_OK); + Lardon3DProjectRecoveryEntry *candidate_recovery = NULL; + for (size_t i = 0; i < recovery_count; ++i) { + if (recovery_entries[i].task_id == cp_task3_id) { + candidate_recovery = &recovery_entries[i]; + } + } + CHECK(candidate_recovery && + same_estimate(&candidate_recovery->snapshot.estimate, + &legacy_snapshot.estimate)); + Lardon3DProjectDbCandidatePairGenerateTask cursor_before; + CHECK(lardon3d_project_db_load_candidate_pair_generate_task( + state.project_db, cp_task3_id, &cursor_before) == + LARDON3D_PROJECT_DB_OK); + + Lardon3DTaskReconstructionContext reconstruction = { + .project_path = state.project_path, + .project_db = state.project_db, + .resource_governor = state.resource_governor, + .orb_vulkan_backend = state.orb_vulkan_backend, + }; + Lardon3DTask *recovered = NULL; + + /* Every field belongs to the historical signature. Neighboring estimates + * are malformed rather than evidence for legacy operational policy. */ + Lardon3DTaskDurableSnapshot near_snapshots[4]; + for (size_t i = 0; i < 4; ++i) near_snapshots[i] = legacy_snapshot; + near_snapshots[0].estimate.memory_fixed_bytes = 256 * 1024; + near_snapshots[1].estimate.memory_bytes_per_item = 64 * 1024 + 1; + near_snapshots[2].estimate.desired_cpu_threads = 2; + near_snapshots[3].estimate.desired_io_slots = 2; + for (size_t i = 0; i < 4; ++i) { + Lardon3DTask *near_task = NULL; + CHECK(lardon3d_task_kind_registry_restore( + lardon3d_task_kind_registry_production(), + candidate_recovery->task_kind, + candidate_recovery->task_kind_version, &near_snapshots[i], + &reconstruction, &near_task) == + LARDON3D_TASK_KIND_RECONSTRUCTION_FAILED && + near_task == NULL); + } + CHECK(lardon3d_task_kind_registry_restore( + lardon3d_task_kind_registry_production(), + candidate_recovery->task_kind, + candidate_recovery->task_kind_version, + &candidate_recovery->snapshot, &reconstruction, &recovered) == + LARDON3D_TASK_KIND_OK && + recovered && lardon3d_task_id(recovered) == cp_task3_id); + Lardon3DResourceEstimate recovered_estimate; + const Lardon3DResourceEstimate current_estimate = { + .memory_fixed_bytes = 256 * 1024, + .gpu_memory_fixed_bytes = 0, + .memory_bytes_per_item = 64 * 1024, + .gpu_memory_bytes_per_item = 0, + .minimum_batch_size = 1, + .maximum_batch_size = 64, + .desired_cpu_threads = 12, + .desired_gpu_slots = 0, + .desired_io_slots = 1, + .task_class = LARDON3D_RESOURCE_TASK_CPU, + }; + CHECK(lardon3d_task_resource_estimate(recovered, &recovered_estimate) && + same_estimate(&recovered_estimate, ¤t_estimate)); + + Lardon3DResourceSnapshot resources = { + .memory_available_bytes = UINT64_MAX, + .cpu_load_1m = 0.0, + }; + Lardon3DResourceDecision decision; + Lardon3DResourceReservation *reservation = NULL; + CHECK(lardon3d_test_resource_snapshot_make_fresh(&resources) && + lardon3d_resource_governor_reserve( + state.resource_governor, &resources, &recovered_estimate, + &decision, &reservation) && + decision.cpu_threads == 12 && decision.batch_size >= 1 && + decision.batch_size <= 64 && reservation); + lardon3d_resource_governor_release(state.resource_governor, reservation); + + Lardon3DHardwareProfile reduced_profile = { + .logical_cpu_count = 4, + .page_size_bytes = 4096, + .memory_total_bytes = UINT64_MAX, + .cpu_architecture = "test", + }; + Lardon3DResourcePolicy reduced_policy = { + .system_cpu_reserve = 2, + .maximum_cpu_load_ratio = 1, + .maximum_io_pressure_avg10 = 100, + .io_slot_capacity = 1, + }; + Lardon3DResourceGovernor *reduced_governor = + lardon3d_resource_governor_create(&reduced_profile, &reduced_policy); + reservation = NULL; + CHECK(reduced_governor && + lardon3d_resource_governor_reserve( + reduced_governor, &resources, &recovered_estimate, + &decision, &reservation) && + decision.kind == LARDON3D_RESOURCE_REDUCE_BATCH && + decision.cpu_threads == 2 && decision.batch_size >= 1 && + decision.batch_size <= 64 && reservation); + lardon3d_resource_governor_release(reduced_governor, reservation); + lardon3d_resource_governor_destroy(reduced_governor); + lardon3d_task_destroy(recovered); + + recovery_count = 0; + CHECK(lardon3d_project_list_recoverable( + &state, lardon3d_task_kind_registry_production(), 0, + recovery_entries, 8, &recovery_count) == LARDON3D_PROJECT_DB_OK); + candidate_recovery = NULL; + for (size_t i = 0; i < recovery_count; ++i) { + if (recovery_entries[i].task_id == cp_task3_id) { + candidate_recovery = &recovery_entries[i]; + } + } + Lardon3DProjectDbCandidatePairGenerateTask cursor_after; + CHECK(candidate_recovery && + candidate_recovery->snapshot.id == cp_task3_id && + same_estimate(&candidate_recovery->snapshot.estimate, + &legacy_snapshot.estimate) && + lardon3d_project_db_load_candidate_pair_generate_task( + state.project_db, cp_task3_id, &cursor_after) == + LARDON3D_PROJECT_DB_OK && + cursor_after.after_feature_set_id == + cursor_before.after_feature_set_id); + + /* A pre-terminal crash repeats exact ephemeral normalization. Recovery + * does not publish an estimate-only .chk or .chk.next with the same Task + * summary, so the durable historical snapshot remains unambiguous. */ + char candidate_staged_path[PATH_MAX]; + CHECK(snprintf(candidate_staged_path, sizeof(candidate_staged_path), + "%s.next", candidate_checkpoint_path) > 0 && + access(candidate_staged_path, F_OK) != 0 && errno == ENOENT); + Lardon3DTask *current_again = NULL; + CHECK(lardon3d_task_kind_registry_restore( + lardon3d_task_kind_registry_production(), + candidate_recovery->task_kind, + candidate_recovery->task_kind_version, + &candidate_recovery->snapshot, &reconstruction, + ¤t_again) == LARDON3D_TASK_KIND_OK && + current_again); + Lardon3DResourceEstimate current_again_estimate; + CHECK(lardon3d_task_resource_estimate(current_again, + ¤t_again_estimate) && + same_estimate(¤t_again_estimate, ¤t_estimate)); + lardon3d_task_destroy(current_again); + Lardon3DTaskDurableSnapshot still_durable; + CHECK(lardon3d_task_checkpoint_load(candidate_checkpoint_path, + &still_durable, NULL) == + LARDON3D_TASK_CHECKPOINT_OK && + same_estimate(&still_durable.estimate, &legacy_snapshot.estimate) && + access(candidate_staged_path, F_OK) != 0 && errno == ENOENT); + lardon3d_task_queue_destroy(state.task_queue); + state.task_queue = NULL; + lardon3d_project_close(&state); + lardon3d_resource_governor_destroy(state.resource_governor); + state.resource_governor = NULL; + + /* This project_open follows the real runner's durable recovery route. + * The paused task has not dispatched this window, so the counters below + * prove participant threads performed recovered production work. */ + lardon3d_candidate_pair_task_test_reset_parallel_counters(); lardon3d_app_state_init(&state); CHECK(runtime(&state) && lardon3d_project_open(&state, "CPTask")); @@ -258,9 +695,11 @@ static bool run_test(void) { CHECK(lardon3d_project_last_recovery_summary(&state, &summary) && summary.resumed == 1); CHECK(wait_state(state.task_queue, cp_task3_id, TASK_COMPLETED, &snap) && - snap.progress == 100); + snap.progress == 100 && + lardon3d_candidate_pair_task_test_started_participants() > 1 && + lardon3d_candidate_pair_task_test_computed_work_items() > 1); - /* --- Test 4: Reconstruction --- */ + /* --- Test 5: Reconstruction --- */ /* Attendre que la tâche se termine puis ré-ouvrir sans reprise */ lardon3d_task_queue_destroy(state.task_queue); state.task_queue = NULL; @@ -280,7 +719,7 @@ static bool run_test(void) { LARDON3D_PROJECT_DB_OK && saved.top_k == 16); - /* --- Test 5: Ordre canonique vérifié --- */ + /* --- Test 6: Ordre canonique vérifié --- */ size_t final_count = 0; Lardon3DProjectDbCandidatePair final_pairs[32]; CHECK(lardon3d_project_db_list_candidate_pairs( diff --git a/tests/test_feature_store.c b/tests/test_feature_store.c index 8730a24..6dfda56 100644 --- a/tests/test_feature_store.c +++ b/tests/test_feature_store.c @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -118,6 +119,29 @@ static bool write_pgm(const char *path, bool uniform) { return fclose(f) == 0; } +static bool write_dense_pgm(const char *path) { + FILE *f = fopen(path, "wb"); + if (!f) { + return false; + } + if (fprintf(f, "P5\n256 256\n255\n") <= 0) { + fclose(f); + return false; + } + uint32_t state = 0x4c33444fU; + for (unsigned int index = 0; index < 256U * 256U; ++index) { + state ^= state << 13U; + state ^= state >> 17U; + state ^= state << 5U; + unsigned char value = (unsigned char)state; + if (fwrite(&value, 1, 1, f) != 1) { + fclose(f); + return false; + } + } + return fclose(f) == 0; +} + static uint64_t read_u64_le(const unsigned char *bytes) { uint64_t value = 0; for (unsigned int index = 0; index < 8; ++index) { @@ -126,6 +150,34 @@ static uint64_t read_u64_le(const unsigned char *bytes) { return value; } +static bool extracted_features_equal(const Lardon3DExtractedFeatures *a, + const Lardon3DExtractedFeatures *b) { + if (!a || !b || a->image_width != b->image_width || a->image_height != b->image_height || + a->feature_count != b->feature_count || a->descriptor_bytes != b->descriptor_bytes) { + return false; + } + for (uint32_t index = 0; index < a->feature_count; ++index) { + const Lardon3DFeatureKeypoint *ka = &a->keypoints[index]; + const Lardon3DFeatureKeypoint *kb = &b->keypoints[index]; + if (memcmp(&ka->x, &kb->x, sizeof(ka->x)) != 0 || + memcmp(&ka->y, &kb->y, sizeof(ka->y)) != 0 || + memcmp(&ka->size, &kb->size, sizeof(ka->size)) != 0 || + memcmp(&ka->angle_degrees, &kb->angle_degrees, sizeof(ka->angle_degrees)) != 0 || + memcmp(&ka->response, &kb->response, sizeof(ka->response)) != 0 || + ka->octave != kb->octave) { + return false; + } + } + return a->descriptor_bytes == 0 || + memcmp(a->descriptors, b->descriptors, a->descriptor_bytes) == 0; +} + +static void print_sha256(const unsigned char sha256[32]) { + for (size_t index = 0; index < 32; ++index) { + printf("%02x", sha256[index]); + } +} + static bool restore_file(const char *path, const unsigned char *bytes, size_t size) { int descriptor = open(path, O_WRONLY | O_TRUNC | O_CLOEXEC); if (descriptor < 0) { @@ -156,10 +208,10 @@ static bool expect_reader_result(const char *root, const Lardon3DProjectDbFeatur static bool run_test(void) { char root[] = "/tmp/lardon3d-feature-store-XXXXXX"; CHECK(mkdtemp(root)); - char dbpath[PATH_MAX], structured[PATH_MAX], uniform[PATH_MAX]; + char dbpath[PATH_MAX], structured[PATH_MAX], uniform[PATH_MAX], dense[PATH_MAX]; CHECK(join_path(dbpath, root, "project.db") && join_path(structured, root, "structured.pgm") && - join_path(uniform, root, "uniform.pgm")); - CHECK(write_pgm(structured, false) && write_pgm(uniform, true)); + join_path(uniform, root, "uniform.pgm") && join_path(dense, root, "dense.pgm")); + CHECK(write_pgm(structured, false) && write_pgm(uniform, true) && write_dense_pgm(dense)); Lardon3DProjectDb *db = NULL; char error[256]; CHECK(lardon3d_project_db_open(dbpath, &db, error) == LARDON3D_PROJECT_DB_OK); @@ -180,6 +232,83 @@ static bool run_test(void) { aa.asset_id == ab.asset_id); CHECK(lardon3d_image_catalog_import_file(&state, a.scanset_id, uniform, 0, &iu, &au) == LARDON3D_IMAGE_CATALOG_IMPORTED); + + /* This fixture proves that OpenCV's supported process-wide thread counts do not change + * FeatureSet order, durable keypoint fields, descriptor rows, or serialized asset bytes. + * Separate image identities avoid reusing an already-published immutable FeatureSet. */ + const unsigned int audit_threads[] = {1, 2, 4, 8, 12}; + Lardon3DProjectDbImage audit_images[5]; + for (size_t index = 0; index < 5; ++index) { + char name[32]; + CHECK(snprintf(name, sizeof(name), "thread-audit-%u", audit_threads[index]) > 0); + Lardon3DProjectDbScanSet scan_set; + Lardon3DProjectDbImageAsset asset; + CHECK(lardon3d_image_catalog_create_scanset(&state, name, &scan_set) && + lardon3d_image_catalog_import_file(&state, scan_set.scanset_id, dense, 0, + &audit_images[index], &asset) == + LARDON3D_IMAGE_CATALOG_IMPORTED); + } + unsigned int original_threads = lardon3d_feature_opencv_thread_count(); + Lardon3DFeatureExtractorParameters audit_orb_parameters = {1024, 8, 12}; + Lardon3DSiftExtractorParameters audit_sift_parameters = + lardon3d_sift_precision_classic_v1(false); + unsigned char audit_sift_fingerprint[32]; + lardon3d_sift_extractor_parameter_fingerprint(&audit_sift_parameters, + audit_sift_fingerprint); + const uint32_t audit_capabilities = + LARDON3D_FEATURE_HAS_SCALE | LARDON3D_FEATURE_HAS_ORIENTATION | + LARDON3D_FEATURE_HAS_RESPONSE | LARDON3D_FEATURE_HAS_OCTAVE; + Lardon3DExtractedFeatures reference_orb = {0}, reference_sift = {0}; + Lardon3DProjectDbFeatureSet reference_orb_set = {0}, reference_sift_set = {0}; + for (size_t index = 0; index < 5; ++index) { + Lardon3DExtractedFeatures orb = {0}, sift_audit = {0}; + Lardon3DProjectDbFeatureSet orb_set, sift_audit_set; + CHECK(lardon3d_feature_opencv_configure_threads(audit_threads[index]) && + lardon3d_feature_opencv_thread_count() == audit_threads[index] && + lardon3d_feature_extract_orb(dense, &audit_orb_parameters, &orb) == + LARDON3D_FEATURE_EXTRACT_OK && + lardon3d_feature_extract_sift(dense, &audit_sift_parameters, &sift_audit) == + LARDON3D_FEATURE_EXTRACT_OK && + orb.feature_count > 0 && sift_audit.feature_count > 0); + if (index == 0) { + reference_orb = orb; + reference_sift = sift_audit; + } else { + CHECK(extracted_features_equal(&reference_orb, &orb) && + extracted_features_equal(&reference_sift, &sift_audit)); + } + CHECK(lardon3d_feature_store_publish(&state, audit_images[index].image_id, 0, + &audit_orb_parameters, &orb, &orb_set) == + LARDON3D_FEATURE_STORE_OK && + lardon3d_feature_store_publish_v2( + &state, audit_images[index].image_id, 0, LARDON3D_SIFT_EXTRACTOR_KIND, + LARDON3D_FEATURE_EXTRACTOR_VERSION, audit_sift_fingerprint, + LARDON3D_FEATURE_DESCRIPTOR_F32, 128, audit_capabilities, &sift_audit, + &sift_audit_set) == LARDON3D_FEATURE_STORE_OK); + if (index == 0) { + reference_orb_set = orb_set; + reference_sift_set = sift_audit_set; + } else { + CHECK(orb_set.feature_asset_id == reference_orb_set.feature_asset_id && + orb_set.asset.size_bytes == reference_orb_set.asset.size_bytes && + memcmp(orb_set.asset.sha256, reference_orb_set.asset.sha256, 32) == 0 && + sift_audit_set.feature_asset_id == reference_sift_set.feature_asset_id && + sift_audit_set.asset.size_bytes == reference_sift_set.asset.size_bytes && + memcmp(sift_audit_set.asset.sha256, reference_sift_set.asset.sha256, 32) == 0); + lardon3d_extracted_features_destroy(&orb); + lardon3d_extracted_features_destroy(&sift_audit); + } + printf("THREAD_AUDIT threads=%u orb_count=%u orb_size=%" PRIu64 " orb_sha256=", + audit_threads[index], orb_set.feature_count, orb_set.asset.size_bytes); + print_sha256(orb_set.asset.sha256); + printf(" sift_count=%u sift_size=%" PRIu64 " sift_sha256=", sift_audit_set.feature_count, + sift_audit_set.asset.size_bytes); + print_sha256(sift_audit_set.asset.sha256); + putchar('\n'); + } + CHECK(lardon3d_feature_opencv_configure_threads(original_threads)); + lardon3d_extracted_features_destroy(&reference_orb); + lardon3d_extracted_features_destroy(&reference_sift); Lardon3DFeatureExtractorParameters p = { .max_features = 512, .pyramid_levels = 4, .fast_threshold = 10}; CHECK(lardon3d_feature_extractor_parameters_valid(&p)); @@ -206,6 +335,14 @@ static bool run_test(void) { Lardon3DExtractedFeatures f; CHECK(lardon3d_feature_extract_orb(structured, &p, &f) == LARDON3D_FEATURE_EXTRACT_OK && f.feature_count > 0 && f.feature_count <= 512); + Lardon3DExtractedFeatures bounded; + Lardon3DFeatureExtractorParameters small = { + .max_features = 7, .pyramid_levels = 8, .fast_threshold = 20}; + /* OpenCV 5 produces eight ORB points for this fixture despite nfeatures=7. */ + CHECK(lardon3d_feature_extract_orb(dense, &small, &bounded) == + LARDON3D_FEATURE_EXTRACT_OK && + bounded.feature_count == 7 && bounded.descriptor_bytes == 7U * 32U); + lardon3d_extracted_features_destroy(&bounded); Lardon3DProjectDbFeatureSet sa, same, sb; CHECK(lardon3d_feature_store_publish(&state, ia.image_id, 0, &p, &f, &sa) == LARDON3D_FEATURE_STORE_OK); diff --git a/tests/test_feature_task.c b/tests/test_feature_task.c index eea277c..8d7d83e 100644 --- a/tests/test_feature_task.c +++ b/tests/test_feature_task.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #define CHECK(x) \ @@ -280,6 +281,76 @@ static bool run_test(void) { rootsift_fingerprint, &rootsift_set) == LARDON3D_PROJECT_DB_OK && rootsift_set.feature_set_id != sift_set.feature_set_id); + /* SIFT and RootSIFT CPU1 checkpoints are exact historical operational + * signatures. Recovery uses CPU12 in memory without publishing an + * estimate-only checkpoint; a neighboring CPU2 shape is corruption. */ + const uint64_t precision_task_ids[2] = {sift_task_id, rootsift_task_id}; + const char *precision_kinds[2] = {LARDON3D_SIFT_EXTRACT_TASK_KIND, + LARDON3D_ROOTSIFT_EXTRACT_TASK_KIND}; + Lardon3DTaskReconstructionContext precision_reconstruction = { + .project_path = state.project_path, + .project_db = state.project_db, + .resource_governor = state.resource_governor, + .orb_vulkan_backend = state.orb_vulkan_backend, + }; + for (size_t precision_index = 0; precision_index < 2; ++precision_index) { + char precision_checkpoint[PATH_MAX]; + char precision_staged[PATH_MAX]; + CHECK(snprintf(precision_checkpoint, sizeof(precision_checkpoint), + "%s/.lardon3d/checkpoints/%lu.chk", state.project_path, + (unsigned long)precision_task_ids[precision_index]) > 0 && + snprintf(precision_staged, sizeof(precision_staged), "%s.next", + precision_checkpoint) > 0); + Lardon3DTaskDurableSnapshot current_precision; + CHECK(lardon3d_task_checkpoint_load(precision_checkpoint, + ¤t_precision, NULL) == + LARDON3D_TASK_CHECKPOINT_OK); + Lardon3DTaskDurableSnapshot historical_precision = current_precision; + historical_precision.estimate.desired_cpu_threads = 1; + historical_precision.progress = 50; + historical_precision.saved_state = TASK_RUNNING; + historical_precision.recovery_state = TASK_PENDING; + historical_precision.finished_at = (struct timespec){0}; + CHECK(lardon3d_task_checkpoint_save(precision_checkpoint, + &historical_precision) == + LARDON3D_TASK_CHECKPOINT_OK); + Lardon3DTask *restored_precision = NULL; + Lardon3DTaskKindResult precision_restore = + lardon3d_task_kind_registry_restore( + lardon3d_task_kind_registry_production(), + precision_kinds[precision_index], + LARDON3D_SIFT_EXTRACT_TASK_KIND_VERSION, + &historical_precision, &precision_reconstruction, + &restored_precision); + if (precision_restore != LARDON3D_TASK_KIND_OK) { + fprintf(stderr, "precision restore %zu failed: %d\n", precision_index, + (int)precision_restore); + } + CHECK(precision_restore == LARDON3D_TASK_KIND_OK && + restored_precision); + Lardon3DResourceEstimate effective_precision; + CHECK(lardon3d_task_resource_estimate(restored_precision, + &effective_precision) && + effective_precision.desired_cpu_threads == 12); + lardon3d_task_destroy(restored_precision); + Lardon3DTaskDurableSnapshot durable_precision; + CHECK(lardon3d_task_checkpoint_load(precision_checkpoint, + &durable_precision, NULL) == + LARDON3D_TASK_CHECKPOINT_OK && + durable_precision.estimate.desired_cpu_threads == 1 && + access(precision_staged, F_OK) != 0 && errno == ENOENT); + Lardon3DTaskDurableSnapshot malformed_precision = historical_precision; + malformed_precision.estimate.desired_cpu_threads = 2; + restored_precision = NULL; + CHECK(lardon3d_task_kind_registry_restore( + lardon3d_task_kind_registry_production(), + precision_kinds[precision_index], + LARDON3D_SIFT_EXTRACT_TASK_KIND_VERSION, + &malformed_precision, &precision_reconstruction, + &restored_precision) == + LARDON3D_TASK_KIND_RECONSTRUCTION_FAILED && + restored_precision == NULL); + } CHECK(lardon3d_feature_reader_open(state.project_path, &rootsift_set, &reader, &metadata) == LARDON3D_FEATURE_STORE_OK); if (metadata.feature_count > 0) { diff --git a/tests/test_geometric_verifier_core.cpp b/tests/test_geometric_verifier_core.cpp index b9e2174..b08a5ac 100644 --- a/tests/test_geometric_verifier_core.cpp +++ b/tests/test_geometric_verifier_core.cpp @@ -127,6 +127,10 @@ static bool publish_features(Fixture *fixture, keypoints[index].x -= 5.0F; keypoints[index].size = 1.0F; } + // Distinct observation IDs may legitimately share coordinates; v2 support + // must count identities rather than unique Point2d values. + keypoints[1].x = keypoints[0].x; + keypoints[1].y = keypoints[0].y; Lardon3DExtractedFeatures features = {}; features.image_width = 1024; features.image_height = 768; @@ -216,8 +220,9 @@ static void put_u32(unsigned char bytes[4], uint32_t value) { bytes[index] = static_cast(value >> (8U * index)); } -static bool create_parent(Fixture *fixture, uint32_t count, - uint64_t *parent_id) { +static bool create_parent_with_entries( + Fixture *fixture, const std::vector &entries, + uint64_t *parent_id) { ++fixture->parent_sequence; char relative[64]; char full[PATH_MAX]; @@ -226,12 +231,7 @@ static bool create_parent(Fixture *fixture, uint32_t count, if (length <= 0 || length >= static_cast(sizeof(relative)) || !join_path(full, fixture->root, relative)) return false; - std::vector entries(count); - for (uint32_t index = 0; index < count; ++index) { - entries[index].feature_index_a = index; - entries[index].feature_index_b = count == 0 ? 0 : (index * 7U) % count; - entries[index].distance = static_cast(index + 1U); - } + const uint32_t count = static_cast(entries.size()); int descriptor = open(full, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0600); if (descriptor < 0) return false; @@ -260,6 +260,18 @@ static bool create_parent(Fixture *fixture, uint32_t count, return true; } +static bool create_parent(Fixture *fixture, uint32_t count, + uint64_t *parent_id) { + std::vector entries(count); + for (uint32_t index = 0; index < count; ++index) { + entries[index].feature_index_a = index; + entries[index].feature_index_b = + count == 0 ? 0 : (index * (count == 7 ? 1U : 7U)) % count; + entries[index].distance = static_cast(index + 1U); + } + return create_parent_with_entries(fixture, entries, parent_id); +} + static std::string mask_bits(uint32_t count, uint32_t inliers, uint32_t stride = 1) { std::string bits(count, '0'); @@ -277,10 +289,17 @@ static void test_parameters_and_fingerprint() { Lardon3DGeometricVerifierParameters parameters = lardon3d_geometric_verifier_default_parameters(); CHECK(lardon3d_geometric_verifier_parameters_valid(¶meters)); - unsigned char first[32], second[32], changed[32]; + unsigned char first[32], second[32], changed[32], v1[32], v2[32], v3[32]; lardon3d_geometric_verifier_fingerprint(¶meters, first); lardon3d_geometric_verifier_fingerprint(¶meters, second); CHECK(std::memcmp(first, second, sizeof(first)) == 0); + CHECK(lardon3d_geometric_verifier_fingerprint_for_version( + ¶meters, LARDON3D_GEOMETRIC_VERIFIER_VERSION_V1, v1)); + CHECK(lardon3d_geometric_verifier_fingerprint_for_version( + ¶meters, LARDON3D_GEOMETRIC_VERIFIER_VERSION_V2, v2)); + CHECK(lardon3d_geometric_verifier_fingerprint_for_version( + ¶meters, LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3, v3)); + CHECK(std::memcmp(first, v1, sizeof(first)) != 0); auto differs = [&](Lardon3DGeometricVerifierParameters changed_parameters) { lardon3d_geometric_verifier_fingerprint(&changed_parameters, changed); return std::memcmp(first, changed, sizeof(first)) != 0; @@ -345,7 +364,7 @@ static void test_parameters_and_fingerprint() { char encoded_hex[2 * LARDON3D_GEOMETRIC_VERIFIER_FINGERPRINT_SIZE + 1]; char fingerprint_hex[65]; to_hex(default_bytes, sizeof(default_bytes), encoded_hex); - to_hex(first, sizeof(first), fingerprint_hex); + to_hex(v1, sizeof(v1), fingerprint_hex); CHECK(std::strcmp( encoded_hex, "4c3344475646503101000000010000000100000001000000000000000000f83f" @@ -356,6 +375,18 @@ static void test_parameters_and_fingerprint() { fingerprint_hex, "ddb44bb070c62be66c405946e89cbb49c084f8f30a21d6f408dc239225b7bbd0") == 0); + to_hex(v2, sizeof(v2), fingerprint_hex); + CHECK(std::strcmp( + fingerprint_hex, + "7868a893437ee611a10008a093286997212fa8bd80b2afd2bb1d11f04f01c5ae") == + 0); + to_hex(v3, sizeof(v3), fingerprint_hex); + CHECK(std::strcmp( + fingerprint_hex, + "6944a471d611d8ffc59dac7cf15a5b79b97e2371d4c51785c477d68c1577f74c") == + 0); + CHECK(std::memcmp(first, v3, sizeof(first)) == 0); + CHECK(LARDON3D_GEOMETRIC_VERIFIER_VERSION == 3); } static void test_seed() { @@ -370,9 +401,21 @@ static void test_seed() { Lardon3DGeometricVerifierParameters parameters = lardon3d_geometric_verifier_default_parameters(); unsigned char production_fingerprint[32]; - lardon3d_geometric_verifier_fingerprint(¶meters, production_fingerprint); + CHECK(lardon3d_geometric_verifier_fingerprint_for_version( + ¶meters, LARDON3D_GEOMETRIC_VERIFIER_VERSION_V1, + production_fingerprint)); CHECK(lardon3d_geometric_verifier_seed(match, production_fingerprint) == 1910542150U); + CHECK(lardon3d_geometric_verifier_fingerprint_for_version( + ¶meters, LARDON3D_GEOMETRIC_VERIFIER_VERSION_V2, + production_fingerprint)); + CHECK(lardon3d_geometric_verifier_seed(match, production_fingerprint) == + 1528046088U); + CHECK(lardon3d_geometric_verifier_fingerprint_for_version( + ¶meters, LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3, + production_fingerprint)); + CHECK(lardon3d_geometric_verifier_seed(match, production_fingerprint) == + 188721673U); } static void test_canonicalization() { @@ -532,6 +575,143 @@ static void test_scientific_boundaries(Fixture *fixture) { } } +static void test_v2_observation_support(Fixture *fixture) { + const uint32_t a_insufficient[] = {0, 1, 2, 3, 4, 5, 0, 1}; + const uint32_t b_sufficient[] = {0, 1, 2, 3, 4, 5, 6, 7}; + const uint32_t both_a[] = {0, 1, 2, 3, 4, 5}; + const uint32_t both_b[] = {0, 1, 2, 3, 4, 5}; + CHECK(!lardon3d_geometric_verifier_test_has_minimal_support( + a_insufficient, b_sufficient, 8, 8, 8)); + CHECK(!lardon3d_geometric_verifier_test_has_minimal_support( + both_a, both_b, 6, 8, 8)); + + Lardon3DGeometricVerifierParameters parameters = + lardon3d_geometric_verifier_default_parameters(); + std::vector repeated_b(10); + for (uint32_t index = 0; index < repeated_b.size(); ++index) { + repeated_b[index] = {index, index % 3U, static_cast(index)}; + } + uint64_t parent = 0; + CHECK(create_parent_with_entries(fixture, repeated_b, &parent)); + CHECK(setenv("LARDON3D_TEST_GEOMETRIC_ESTIMATOR", "error", 1) == 0); + lardon3d_geometric_verifier_test_reset_estimator_calls(); + Lardon3DProjectDbGeometricVerificationResult v2; + bool reused = false; + CHECK(lardon3d_geometric_verifier_verify_and_publish_version( + fixture->root, fixture->state.project_db, parent, ¶meters, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V2, &v2, + &reused) == LARDON3D_GEOMETRIC_VERIFIER_OK); + CHECK(!reused && v2.verifier_version == 2 && + v2.status == LARDON3D_GEOMETRIC_REJECTED && v2.inlier_count == 0 && + !v2.has_model && v2.inlier_mask_size == 2 && v2.inlier_mask[0] == 0 && + v2.inlier_mask[1] == 0 && + lardon3d_geometric_verifier_test_estimator_calls() == 0); + + // Historical v1 deliberately uses raw row count. Publishing it on the same + // parent proves that v2 did not reinterpret or overwrite the v1 identity. + CHECK(lardon3d_geometric_verifier_verify_and_publish_version( + fixture->root, fixture->state.project_db, parent, ¶meters, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V1, &v2, + &reused) == LARDON3D_GEOMETRIC_VERIFIER_ESTIMATOR_ERROR); + CHECK(lardon3d_geometric_verifier_test_estimator_calls() == 1); + CHECK(unsetenv("LARDON3D_TEST_GEOMETRIC_ESTIMATOR") == 0); + + std::vector exact(7); + for (uint32_t index = 0; index < exact.size(); ++index) + exact[index] = {index, index, static_cast(index)}; + CHECK(create_parent_with_entries(fixture, exact, &parent)); + parameters.min_inlier_count = 1; + parameters.min_inlier_ratio = 0.0; + CHECK(setenv("LARDON3D_TEST_GEOMETRIC_ESTIMATOR", "controlled", 1) == 0); + CHECK(setenv("LARDON3D_TEST_GEOMETRIC_MASK", "1111111", 1) == 0); + lardon3d_geometric_verifier_test_reset_estimator_calls(); + CHECK(lardon3d_geometric_verifier_verify_and_publish_version( + fixture->root, fixture->state.project_db, parent, ¶meters, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V2, &v2, + &reused) == LARDON3D_GEOMETRIC_VERIFIER_OK); + CHECK(!reused && v2.status == LARDON3D_GEOMETRIC_VERIFIED && + v2.inlier_count == 7 && + lardon3d_geometric_verifier_test_estimator_calls() == 1); + CHECK(unsetenv("LARDON3D_TEST_GEOMETRIC_ESTIMATOR") == 0); + CHECK(unsetenv("LARDON3D_TEST_GEOMETRIC_MASK") == 0); +} + +static void test_v3_acceptance_feasibility(Fixture *fixture) { + Lardon3DGeometricVerifierParameters parameters = + lardon3d_geometric_verifier_default_parameters(); + Lardon3DProjectDbGeometricVerificationResult result; + bool reused = false; + + // V3's raw-count preflight is exactly the acceptance proof Nroot, fixture->state.project_db, below, ¶meters, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3, &result, + &reused) == LARDON3D_GEOMETRIC_VERIFIER_OK); + CHECK(!reused && result.status == LARDON3D_GEOMETRIC_REJECTED && + result.inlier_count == 0 && !result.has_model && + lardon3d_geometric_verifier_test_estimator_calls() == 0); + CHECK(lardon3d_geometric_verifier_verify_and_publish_version( + fixture->root, fixture->state.project_db, exact, ¶meters, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3, &result, + &reused) == LARDON3D_GEOMETRIC_VERIFIER_ESTIMATOR_ERROR); + CHECK(lardon3d_geometric_verifier_test_estimator_calls() == 1); + + parameters.min_inlier_count = 9; + CHECK(create_parent(fixture, 8, &below)); + CHECK(create_parent(fixture, 9, &exact)); + lardon3d_geometric_verifier_test_reset_estimator_calls(); + CHECK(lardon3d_geometric_verifier_verify_and_publish_version( + fixture->root, fixture->state.project_db, below, ¶meters, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3, &result, + &reused) == LARDON3D_GEOMETRIC_VERIFIER_OK); + CHECK(lardon3d_geometric_verifier_test_estimator_calls() == 0); + CHECK(lardon3d_geometric_verifier_verify_and_publish_version( + fixture->root, fixture->state.project_db, exact, ¶meters, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3, &result, + &reused) == LARDON3D_GEOMETRIC_VERIFIER_ESTIMATOR_ERROR); + CHECK(lardon3d_geometric_verifier_test_estimator_calls() == 1); + + parameters = lardon3d_geometric_verifier_default_parameters(); + std::vector repeated_b(20); + for (uint32_t index = 0; index < repeated_b.size(); ++index) + repeated_b[index] = {index, index % 3U, static_cast(index)}; + CHECK(create_parent_with_entries(fixture, repeated_b, &below)); + lardon3d_geometric_verifier_test_reset_estimator_calls(); + CHECK(lardon3d_geometric_verifier_verify_and_publish_version( + fixture->root, fixture->state.project_db, below, ¶meters, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3, &result, + &reused) == LARDON3D_GEOMETRIC_VERIFIER_OK); + CHECK(result.status == LARDON3D_GEOMETRIC_REJECTED && + lardon3d_geometric_verifier_test_estimator_calls() == 0); + CHECK(unsetenv("LARDON3D_TEST_GEOMETRIC_ESTIMATOR") == 0); + + // Acceptance impossibility never bypasses Match asset validation. This + // deliberately corrupt Nparent_sequence) > 0); + CHECK(join_path(corrupt_path, fixture->root, corrupt_relative)); + int corrupt_fd = open(corrupt_path, O_WRONLY | O_CLOEXEC); + CHECK(corrupt_fd >= 0); + const unsigned char changed = 0xff; + CHECK(pwrite(corrupt_fd, &changed, 1, 0) == 1); + CHECK(close(corrupt_fd) == 0); + CHECK(lardon3d_geometric_verifier_verify_and_publish_version( + fixture->root, fixture->state.project_db, corrupt, ¶meters, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3, &result, + &reused) == LARDON3D_GEOMETRIC_VERIFIER_CORRUPT); +} + static void test_mask_boundaries(Fixture *fixture) { Lardon3DGeometricVerifierParameters parameters = lardon3d_geometric_verifier_default_parameters(); @@ -584,7 +764,8 @@ static void test_failures(Fixture *fixture) { lardon3d_geometric_verifier_fingerprint(¶meters, fingerprint); CHECK(lardon3d_project_db_find_geometric_verification_result( fixture->state.project_db, parent, - LARDON3D_GEOMETRIC_VERIFIER_FUNDAMENTAL, 1, fingerprint, + LARDON3D_GEOMETRIC_VERIFIER_FUNDAMENTAL, + LARDON3D_GEOMETRIC_VERIFIER_VERSION, fingerprint, &result) == LARDON3D_PROJECT_DB_NOT_FOUND); } @@ -607,7 +788,8 @@ static void test_failures(Fixture *fixture) { lardon3d_geometric_verifier_fingerprint(¶meters, fingerprint); CHECK(lardon3d_project_db_find_geometric_verification_result( fixture->state.project_db, publication_parent, - LARDON3D_GEOMETRIC_VERIFIER_FUNDAMENTAL, 1, fingerprint, + LARDON3D_GEOMETRIC_VERIFIER_FUNDAMENTAL, + LARDON3D_GEOMETRIC_VERIFIER_VERSION, fingerprint, &result) == LARDON3D_PROJECT_DB_NOT_FOUND); CHECK(run_controlled(fixture, publication_parent, bits, ¶meters, &result, &reused)); @@ -794,6 +976,8 @@ int main() { if (fixture.state.project_db) { test_e2e_and_reuse(&fixture); test_scientific_boundaries(&fixture); + test_v2_observation_support(&fixture); + test_v3_acceptance_feasibility(&fixture); test_mask_boundaries(&fixture); test_failures(&fixture); test_parent_and_asset_failures(&fixture); diff --git a/tests/test_geometric_verifier_task.c b/tests/test_geometric_verifier_task.c index d2417f0..786477b 100644 --- a/tests/test_geometric_verifier_task.c +++ b/tests/test_geometric_verifier_task.c @@ -170,7 +170,7 @@ static bool seed_reusable_parents(const char *path, "geometric_verification_results(match_result_id,verifier_kind," "verifier_version,parameter_fingerprint,status,inlier_count,inlier_" "mask,created_at) " - "VALUES(?1,1,1,?2,1,0,zeroblob(2),1)", + "VALUES(?1,1,3,?2,1,0,zeroblob(2),1)", -1, &result, NULL) == SQLITE_OK); for (int index = 0; index < fixture_parent_count(); ++index) { char kind[32]; @@ -198,6 +198,7 @@ static bool seed_reusable_parents(const char *path, } static bool add_reusable_results(const char *path, + uint32_t verifier_version, const unsigned char fingerprint[32]) { sqlite3 *db = NULL; sqlite3_stmt *statement = NULL; @@ -209,9 +210,10 @@ static bool add_reusable_results(const char *path, "geometric_verification_results(match_result_id,verifier_kind," "verifier_version,parameter_fingerprint,status,inlier_count,inlier_" "mask,created_at) " - "SELECT match_result_id,1,1,?1,1,0,zeroblob(2),2 FROM match_results", + "SELECT match_result_id,1,?1,?2,1,0,zeroblob(2),2 FROM match_results", -1, &statement, NULL) == SQLITE_OK); - sqlite3_bind_blob(statement, 1, fingerprint, 32, SQLITE_TRANSIENT); + sqlite3_bind_int64(statement, 1, verifier_version); + sqlite3_bind_blob(statement, 2, fingerprint, 32, SQLITE_TRANSIENT); int code = sqlite3_step(statement); if (code != SQLITE_DONE) { (void)fprintf(stderr, "Insertion reuse: %s\n", sqlite3_errmsg(db)); @@ -221,6 +223,113 @@ static bool add_reusable_results(const char *path, return sqlite3_close(db) == SQLITE_OK; } +static bool update_task_fingerprint(const char *path, uint64_t task_id, + const unsigned char fingerprint[32]) { + sqlite3 *db = NULL; + sqlite3_stmt *statement = NULL; + bool ok = sqlite3_open_v2(path, &db, SQLITE_OPEN_READWRITE, NULL) == SQLITE_OK && + sqlite3_prepare_v2( + db, + "UPDATE geometric_verifier_tasks SET parameter_fingerprint=?1 " + "WHERE task_id=?2", + -1, &statement, NULL) == SQLITE_OK; + if (ok) { + sqlite3_bind_blob(statement, 1, fingerprint, 32, SQLITE_TRANSIENT); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)task_id); + ok = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(db) == 1; + } + if (statement) { + ok = sqlite3_finalize(statement) == SQLITE_OK && ok; + } + return sqlite3_close(db) == SQLITE_OK && ok; +} + +static bool run_task_mid_batch_failure_regression_test(void) { + char root[] = "/tmp/lardon3d-geometric-task-fail-XXXXXX"; + CHECK(mkdtemp(root) != NULL); + char internal[4096]; + char checkpoints[4096]; + char database_path[4096]; + (void)snprintf(internal, sizeof(internal), "%s/.lardon3d", root); + (void)snprintf(checkpoints, sizeof(checkpoints), "%s/checkpoints", internal); + (void)snprintf(database_path, sizeof(database_path), "%s/project.sqlite3", + internal); + CHECK(mkdir(internal, 0700) == 0 && mkdir(checkpoints, 0700) == 0); + + Lardon3DAppState state; + lardon3d_app_state_init(&state); + char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY]; + CHECK(lardon3d_project_db_open(database_path, &state.project_db, error) == + LARDON3D_PROJECT_DB_OK); + state.project_loaded = true; + (void)snprintf(state.project_path, sizeof(state.project_path), "%s", root); + state.hardware_profile = (Lardon3DHardwareProfile){ + .logical_cpu_count = 16, + .page_size_bytes = 4096, + .memory_total_bytes = 16ULL * 1024 * 1024 * 1024, + .cpu_architecture = "test", + }; + Lardon3DResourcePolicy resource_policy = policy(); + state.resource_governor = lardon3d_resource_governor_create( + &state.hardware_profile, &resource_policy); + state.task_queue = lardon3d_task_queue_create(state.resource_governor, 16); + CHECK(state.resource_governor && state.task_queue); + + Lardon3DGeometricVerifierTaskConfiguration configuration = { + .verifier = lardon3d_geometric_verifier_default_parameters(), + }; + unsigned char fingerprint[32]; + lardon3d_geometric_verifier_fingerprint(&configuration.verifier, fingerprint); + CHECK(setenv("LARDON3D_TEST_GEOMETRIC_PARENT_COUNT", "12", 1) == 0); + CHECK(seed_reusable_parents(database_path, fingerprint)); + CHECK(unsetenv("LARDON3D_TEST_GEOMETRIC_PARENT_COUNT") == 0); + + CHECK(exec_sql( + database_path, + "DELETE FROM geometric_verification_results WHERE " + "match_result_id != (SELECT MIN(match_result_id) FROM " + "geometric_verification_results);")); + + CHECK(setenv("LARDON3D_TEST_GEOMETRIC_ESTIMATOR", "error", 1) == 0); + + uint64_t task_id = 0; + CHECK(lardon3d_project_enqueue_geometric_verifier_task(&state, &configuration, + &task_id)); + Lardon3DTaskSnapshot snapshot; + CHECK(wait_state(state.task_queue, task_id, TASK_FAILED, &snapshot)); + CHECK(snapshot.message[0] != '\0' && + strstr(snapshot.message, "Contrat de lot") == NULL); + + for (size_t attempt = 0; attempt < 2000000; ++attempt) { + if (query_integer(database_path, + "SELECT after_match_result_id FROM " + "geometric_verifier_tasks", + 1)) { + break; + } + sched_yield(); + } + CHECK(query_integer(database_path, + "SELECT after_match_result_id FROM " + "geometric_verifier_tasks", + 1)); + + CHECK(query_integer(database_path, + "SELECT COUNT(*) FROM geometric_verification_results", 1)); + CHECK(query_integer( + database_path, + "SELECT COUNT(*) FROM geometric_verification_results WHERE " + "match_result_id > 1", + 0)); + CHECK(unsetenv("LARDON3D_TEST_GEOMETRIC_ESTIMATOR") == 0); + + lardon3d_task_queue_destroy(state.task_queue); + lardon3d_resource_governor_destroy(state.resource_governor); + lardon3d_project_db_close(state.project_db); + CHECK(remove_tree(root)); + return true; +} + static bool run_task_test(void) { char root[] = "/tmp/lardon3d-geometric-task-XXXXXX"; CHECK(mkdtemp(root) != NULL); @@ -277,8 +386,19 @@ static bool run_task_test(void) { memcmp(durable.parameter_fingerprint, fingerprint, 32) == 0); configuration.verifier.threshold_pixels = 1.6; - lardon3d_geometric_verifier_fingerprint(&configuration.verifier, fingerprint); - CHECK(add_reusable_results(database_path, fingerprint)); + CHECK(lardon3d_geometric_verifier_fingerprint_for_version( + &configuration.verifier, LARDON3D_GEOMETRIC_VERIFIER_VERSION_V2, + fingerprint)); + CHECK(add_reusable_results(database_path, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V2, + fingerprint)); + unsigned char current_fingerprint[32]; + CHECK(lardon3d_geometric_verifier_fingerprint_for_version( + &configuration.verifier, LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3, + current_fingerprint)); + CHECK(add_reusable_results(database_path, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3, + current_fingerprint)); CHECK(setenv("LARDON3D_TEST_GEOMETRIC_PAUSE_AFTER_PUBLICATION", "1", 1) == 0); CHECK(lardon3d_project_enqueue_geometric_verifier_task(&state, &configuration, &task_id)); @@ -292,6 +412,15 @@ static bool run_task_test(void) { state.task_queue = NULL; lardon3d_project_db_close(state.project_db); state.project_db = NULL; + // Simulate a historical v1 durable row at the crash boundary. Recovery must + // derive v1 from this exact fingerprint without a schema-version column. + CHECK(lardon3d_geometric_verifier_fingerprint_for_version( + &configuration.verifier, LARDON3D_GEOMETRIC_VERIFIER_VERSION_V1, + fingerprint)); + CHECK(update_task_fingerprint(database_path, task_id, fingerprint)); + CHECK(add_reusable_results(database_path, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V1, + fingerprint)); CHECK(lardon3d_project_db_open(database_path, &state.project_db, error) == LARDON3D_PROJECT_DB_OK); state.task_queue = lardon3d_task_queue_create(state.resource_governor, 16); @@ -316,8 +445,49 @@ static bool run_task_test(void) { durable.after_match_result_id == (uint64_t)fixture_parent_count()); configuration.verifier.threshold_pixels = 1.7; - lardon3d_geometric_verifier_fingerprint(&configuration.verifier, fingerprint); - CHECK(add_reusable_results(database_path, fingerprint)); + CHECK(lardon3d_geometric_verifier_fingerprint_for_version( + &configuration.verifier, LARDON3D_GEOMETRIC_VERIFIER_VERSION_V2, + fingerprint)); + CHECK(add_reusable_results(database_path, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V2, + fingerprint)); + CHECK(lardon3d_geometric_verifier_fingerprint_for_version( + &configuration.verifier, LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3, + current_fingerprint)); + CHECK(add_reusable_results(database_path, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3, + current_fingerprint)); + CHECK(setenv("LARDON3D_TEST_GEOMETRIC_PAUSE_AFTER_PUBLICATION", "1", 1) == 0); + CHECK(lardon3d_project_enqueue_geometric_verifier_task(&state, &configuration, + &task_id)); + CHECK(wait_state(state.task_queue, task_id, TASK_PAUSED, &snapshot)); + CHECK(setenv("LARDON3D_TEST_GEOMETRIC_SKIP_FINISHED_CHECKPOINT", "1", 1) == + 0); + lardon3d_task_queue_destroy(state.task_queue); + state.task_queue = NULL; + lardon3d_project_db_close(state.project_db); + state.project_db = NULL; + // Historical v2 reconstruction uses its exact immutable fingerprint; the + // current v3 task identity must never relabel this durable row. + CHECK(update_task_fingerprint(database_path, task_id, fingerprint)); + CHECK(lardon3d_project_db_open(database_path, &state.project_db, error) == + LARDON3D_PROJECT_DB_OK); + state.task_queue = lardon3d_task_queue_create(state.resource_governor, 16); + CHECK(state.task_queue != NULL); + CHECK(unsetenv("LARDON3D_TEST_GEOMETRIC_PAUSE_AFTER_PUBLICATION") == 0); + CHECK(unsetenv("LARDON3D_TEST_GEOMETRIC_SKIP_FINISHED_CHECKPOINT") == 0); + recovery_result = lardon3d_project_resume_recoverable_tasks( + &state, lardon3d_task_kind_registry_production(), &recovery); + CHECK(recovery_result == LARDON3D_PROJECT_DB_OK && recovery.resumed == 1); + CHECK(wait_durable(state.project_db, task_id, TASK_COMPLETED)); + + configuration.verifier.threshold_pixels = 1.8; + CHECK(lardon3d_geometric_verifier_fingerprint_for_version( + &configuration.verifier, LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3, + fingerprint)); + CHECK(add_reusable_results(database_path, + LARDON3D_GEOMETRIC_VERIFIER_VERSION_V3, + fingerprint)); CHECK(setenv("LARDON3D_TEST_GEOMETRIC_PAUSE_AFTER_PUBLICATION", "1", 1) == 0); CHECK(lardon3d_project_enqueue_geometric_verifier_task(&state, &configuration, &task_id)); @@ -375,4 +545,8 @@ static bool run_task_test(void) { return true; } -int main(void) { return run_task_test() ? EXIT_SUCCESS : EXIT_FAILURE; } +int main(void) { + return (run_task_test() && run_task_mid_batch_failure_regression_test()) ? + EXIT_SUCCESS : + EXIT_FAILURE; +} diff --git a/tests/test_matcher_task.c b/tests/test_matcher_task.c index e057349..b28f98f 100644 --- a/tests/test_matcher_task.c +++ b/tests/test_matcher_task.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #define CHECK(condition) \ @@ -30,6 +31,10 @@ enum { PERSISTED_PAIR_COUNT = PAIR_COUNT - 1, }; +void lardon3d_matcher_task_test_reset_backend_counters(void); +size_t lardon3d_matcher_task_test_vulkan_uses(void); +size_t lardon3d_matcher_task_test_forced_fallbacks(void); + typedef struct { char root[PATH_MAX]; Lardon3DAppState state; @@ -78,6 +83,20 @@ static bool query_integer(const char *path, const char *sql, return success; } +static bool candidate_results_have_one_evidence(const char *path, + uint64_t candidate_pair_id) { + char sql[512]; + int written = snprintf( + sql, sizeof(sql), + "SELECT count(*) FROM (SELECT candidate_pair_id FROM match_results " + "WHERE candidate_pair_id=%lu GROUP BY candidate_pair_id HAVING " + "count(DISTINCT result_status||':'||match_count||':'||" + "ifnull(hex(match_asset_sha256),''))<>1)", + (unsigned long)candidate_pair_id); + return written > 0 && (size_t)written < sizeof(sql) && + query_integer(path, sql, 0); +} + static bool downgrade_project_to_historical_v10(const char *database_path) { sqlite3 *connection = NULL; if (sqlite3_open_v2(database_path, &connection, SQLITE_OPEN_READWRITE, @@ -217,24 +236,42 @@ static bool register_image(Fixture *fixture, unsigned char seed, size_t index) { } static bool publish_features(Fixture *fixture, size_t image_index) { - unsigned char descriptor[32]; - memset(descriptor, (int)image_index, sizeof(descriptor)); - Lardon3DFeatureKeypoint keypoint = { - .size = 1.0F, - }; + uint32_t feature_count = image_index < 2 ? 769U : 1U; + Lardon3DFeatureKeypoint *keypoints = + calloc(feature_count, sizeof(*keypoints)); + unsigned char *descriptors = calloc(feature_count, 32); + if (!keypoints || !descriptors) { + free(keypoints); + free(descriptors); + return false; + } + for (uint32_t index = 0; index < feature_count; ++index) { + keypoints[index].size = 1.0F; + /* The first pair is identical and has more than 768 descriptors per side, + * forcing the audited Vulkan dispatch boundary while retaining a unique + * zero-distance nearest neighbor for exact CPU/file parity. */ + uint32_t value = image_index < 2 ? index : (uint32_t)image_index; + for (size_t byte = 0; byte < 32; ++byte) { + descriptors[(size_t)index * 32 + byte] = + (unsigned char)((value >> (8U * (byte % 4))) ^ (uint32_t)(byte * 29)); + } + } Lardon3DExtractedFeatures features = { .image_width = 64, .image_height = 64, - .feature_count = 1, - .keypoints = &keypoint, - .descriptors = descriptor, - .descriptor_bytes = sizeof(descriptor), + .feature_count = feature_count, + .keypoints = keypoints, + .descriptors = descriptors, + .descriptor_bytes = (size_t)feature_count * 32, }; Lardon3DProjectDbFeatureSet feature_set; - return lardon3d_feature_store_publish_v2( + bool success = lardon3d_feature_store_publish_v2( &fixture->state, fixture->images[image_index].image_id, 0, "orb", 1, fixture->feature_fingerprint, LARDON3D_FEATURE_DESCRIPTOR_U8, 32, 0, &features, &feature_set) == LARDON3D_FEATURE_STORE_OK; + free(keypoints); + free(descriptors); + return success; } static bool fixture_create(Fixture *fixture) { @@ -352,6 +389,77 @@ static bool count_results(Fixture *fixture, size_t *count) { } } +static bool same_estimate(const Lardon3DResourceEstimate *a, + const Lardon3DResourceEstimate *b) { + return a->memory_fixed_bytes == b->memory_fixed_bytes && + a->gpu_memory_fixed_bytes == b->gpu_memory_fixed_bytes && + a->memory_bytes_per_item == b->memory_bytes_per_item && + a->gpu_memory_bytes_per_item == b->gpu_memory_bytes_per_item && + a->minimum_batch_size == b->minimum_batch_size && + a->maximum_batch_size == b->maximum_batch_size && + a->desired_cpu_threads == b->desired_cpu_threads && + a->desired_gpu_slots == b->desired_gpu_slots && + a->desired_io_slots == b->desired_io_slots && + a->task_class == b->task_class; +} + +static bool run_completed_with_threads(Fixture *fixture, + Lardon3DMatcherTaskConfiguration settings, + unsigned int threads) { + char value[16]; + (void)snprintf(value, sizeof(value), "%u", threads); + if (setenv("LARDON3D_TEST_MATCHER_CPU_THREADS", value, 1) != 0) return false; + uint64_t task_id = 0; + Lardon3DTaskSnapshot snapshot; + Lardon3DProjectDbTask durable; + bool success = lardon3d_project_enqueue_matcher_task(&fixture->state, &settings, + &task_id) && + wait_state(fixture->state.task_queue, task_id, TASK_COMPLETED, + &snapshot) && + snapshot.progress == 100 && + wait_durable_state(fixture->state.project_db, task_id, + TASK_COMPLETED, &durable); + return unsetenv("LARDON3D_TEST_MATCHER_CPU_THREADS") == 0 && success; +} + +static bool run_failed_at_pair(Fixture *fixture, + Lardon3DMatcherTaskConfiguration settings, + const char *failure_variable, + uint64_t failed_pair_id, size_t expected_prefix, + uint64_t expected_cursor) { + size_t before = 0; + if (!count_results(fixture, &before)) return false; + char value[32]; + (void)snprintf(value, sizeof(value), "%lu", (unsigned long)failed_pair_id); + if (setenv(failure_variable, value, 1) != 0 || + setenv("LARDON3D_TEST_MATCHER_CPU_THREADS", "4", 1) != 0) { + return false; + } + uint64_t task_id = 0; + Lardon3DTaskSnapshot snapshot; + bool failed = lardon3d_project_enqueue_matcher_task(&fixture->state, &settings, + &task_id) && + wait_state(fixture->state.task_queue, task_id, TASK_FAILED, + &snapshot); + (void)unsetenv(failure_variable); + (void)unsetenv("LARDON3D_TEST_MATCHER_CPU_THREADS"); + size_t after = 0; + Lardon3DProjectDbMatcherTask saved; + bool cursor_saved = false; + for (size_t attempt = 0; attempt < 2000000; ++attempt) { + if (lardon3d_project_db_load_matcher_task(fixture->state.project_db, task_id, + &saved) == LARDON3D_PROJECT_DB_OK && + saved.after_candidate_pair_id == expected_cursor) { + cursor_saved = true; + break; + } + sched_yield(); + } + return failed && cursor_saved && count_results(fixture, &after) && + after == before + expected_prefix && + saved.after_candidate_pair_id == expected_cursor; +} + static bool run_test(void) { Fixture fixture; CHECK(fixture_create(&fixture)); @@ -466,7 +574,7 @@ static bool run_test(void) { &snapshot)); Lardon3DResourcePolicy pressure_policy = interactive_policy(); pressure_policy.system_memory_reserve_bytes = - fixture.state.hardware_profile.memory_total_bytes - 16ULL * 1024 * 1024; + fixture.state.hardware_profile.memory_total_bytes - 96ULL * 1024 * 1024; pressure_policy.emergency_memory_floor_bytes = pressure_policy.system_memory_reserve_bytes; CHECK(lardon3d_resource_governor_set_policy(fixture.state.resource_governor, @@ -488,15 +596,265 @@ static bool run_test(void) { CHECK(wait_state(fixture.state.task_queue, pressure_id, TASK_COMPLETED, &snapshot)); + /* Thread-count changes are operational only: all three runs must publish + * the same Candidate Pair cardinality and identical raw Match evidence. */ + size_t equivalence_before = 0; + CHECK(count_results(&fixture, &equivalence_before)); + Lardon3DMatcherTaskConfiguration one = settings; + one.matcher.ratio_threshold = 0.80F; + CHECK(run_completed_with_threads(&fixture, one, 1)); + Lardon3DMatcherTaskConfiguration two = settings; + two.matcher.ratio_threshold = 0.81F; + CHECK(run_completed_with_threads(&fixture, two, 2)); + Lardon3DMatcherTaskConfiguration four = settings; + four.matcher.ratio_threshold = 0.82F; + CHECK(run_completed_with_threads(&fixture, four, 4)); + Lardon3DMatcherTaskConfiguration eight = settings; + eight.matcher.ratio_threshold = 0.84F; + CHECK(run_completed_with_threads(&fixture, eight, 8)); + size_t equivalence_after = 0; + CHECK(count_results(&fixture, &equivalence_after)); + CHECK(equivalence_after == equivalence_before + 4 * PERSISTED_PAIR_COUNT); + CHECK(candidate_results_have_one_evidence( + database_path, fixture.pairs[0].candidate_pair_id)); + + /* Explicit GPU selection is operational only. A real Queue admission must + * reserve the exact GPU shape and dispatch the first 769x769 ORB pair to + * Vulkan; all Match evidence remains byte-identical to the CPU runs above. */ +#ifdef LARDON3D_MATCHER_TASK_VULKAN + lardon3d_matcher_task_test_reset_backend_counters(); + Lardon3DMatcherTaskConfiguration gpu = settings; + gpu.matcher.ratio_threshold = 0.68F; + uint64_t gpu_id = 0; + CHECK(lardon3d_project_enqueue_matcher_task_with_mode( + &fixture.state, &gpu, LARDON3D_MATCHER_TASK_MODE_ORB_VULKAN, &gpu_id)); + CHECK(wait_state(fixture.state.task_queue, gpu_id, TASK_COMPLETED, &snapshot)); + Lardon3DProjectDbTask gpu_durable; + CHECK(wait_durable_state(fixture.state.project_db, gpu_id, TASK_COMPLETED, + &gpu_durable)); + CHECK(lardon3d_matcher_task_test_vulkan_uses() >= 1); + CHECK(candidate_results_have_one_evidence( + database_path, fixture.pairs[0].candidate_pair_id)); + /* Runtime backend failure after valid GPU admission falls back through the + * exact CPU implementation; it never changes the immutable estimate or the + * scientific Match identity. */ + CHECK(setenv("LARDON3D_TEST_MATCHER_FORCE_FALLBACK", "1", 1) == 0); + lardon3d_matcher_task_test_reset_backend_counters(); + Lardon3DMatcherTaskConfiguration fallback = settings; + fallback.matcher.ratio_threshold = 0.67F; + uint64_t fallback_id = 0; + CHECK(lardon3d_project_enqueue_matcher_task_with_mode( + &fixture.state, &fallback, LARDON3D_MATCHER_TASK_MODE_ORB_VULKAN, + &fallback_id)); + CHECK(wait_state(fixture.state.task_queue, fallback_id, TASK_COMPLETED, + &snapshot)); + Lardon3DProjectDbTask fallback_durable; + CHECK(wait_durable_state(fixture.state.project_db, fallback_id, + TASK_COMPLETED, &fallback_durable)); + CHECK(lardon3d_matcher_task_test_forced_fallbacks() >= 1); + CHECK(lardon3d_matcher_task_test_vulkan_uses() == 0); + CHECK(unsetenv("LARDON3D_TEST_MATCHER_FORCE_FALLBACK") == 0); + CHECK(candidate_results_have_one_evidence( + database_path, fixture.pairs[0].candidate_pair_id)); +#else + /* The supported Vulkan-disabled build exposes no backend. An explicit GPU + * request must fail before Task-ID allocation instead of becoming CPU work. */ + Lardon3DMatcherTaskConfiguration gpu = settings; + gpu.matcher.ratio_threshold = 0.68F; + uint64_t gpu_id = 99; + CHECK(lardon3d_project_enqueue_matcher_task_with_mode( + &fixture.state, &gpu, LARDON3D_MATCHER_TASK_MODE_ORB_VULKAN, + &gpu_id) == false && + gpu_id == 0); +#endif + + /* The request remains eight, but only two CPUs are available after host + * headroom. Successful completion exercises the reduced Governor contract. */ + Lardon3DResourcePolicy reduced_policy = interactive_policy(); + reduced_policy.system_cpu_reserve = 14; + CHECK(lardon3d_resource_governor_set_policy(fixture.state.resource_governor, + &reduced_policy)); + Lardon3DMatcherTaskConfiguration reduced = settings; + reduced.matcher.ratio_threshold = 0.83F; + uint64_t reduced_id = 0; + CHECK(lardon3d_project_enqueue_matcher_task(&fixture.state, &reduced, + &reduced_id)); + CHECK(wait_state(fixture.state.task_queue, reduced_id, TASK_COMPLETED, + &snapshot)); + Lardon3DResourcePolicy restored_policy = interactive_policy(); + CHECK(lardon3d_resource_governor_set_policy(fixture.state.resource_governor, + &restored_policy)); + + /* A failed computation seals the ordered suffix. The finished checkpoint + * may retain only the already durable contiguous prefix. */ + Lardon3DMatcherTaskConfiguration fail_first = settings; + fail_first.matcher.ratio_threshold = 0.70F; + CHECK(run_failed_at_pair(&fixture, fail_first, + "LARDON3D_TEST_MATCHER_FAIL_COMPUTE_PAIR_ID", + fixture.pairs[0].candidate_pair_id, 0, 0)); + Lardon3DMatcherTaskConfiguration fail_middle = settings; + fail_middle.matcher.ratio_threshold = 0.71F; + CHECK(run_failed_at_pair(&fixture, fail_middle, + "LARDON3D_TEST_MATCHER_FAIL_COMPUTE_PAIR_ID", + fixture.pairs[5].candidate_pair_id, 4, + fixture.pairs[4].candidate_pair_id)); + Lardon3DMatcherTaskConfiguration fail_last = settings; + fail_last.matcher.ratio_threshold = 0.72F; + CHECK(run_failed_at_pair(&fixture, fail_last, + "LARDON3D_TEST_MATCHER_FAIL_COMPUTE_PAIR_ID", + fixture.pairs[PAIR_COUNT - 1].candidate_pair_id, + PERSISTED_PAIR_COUNT - 1, + fixture.pairs[PAIR_COUNT - 2].candidate_pair_id)); + Lardon3DMatcherTaskConfiguration publish_first = settings; + publish_first.matcher.ratio_threshold = 0.73F; + CHECK(run_failed_at_pair(&fixture, publish_first, + "LARDON3D_TEST_MATCHER_FAIL_PUBLISH_PAIR_ID", + fixture.pairs[0].candidate_pair_id, 0, 0)); + Lardon3DMatcherTaskConfiguration publish_middle = settings; + publish_middle.matcher.ratio_threshold = 0.74F; + CHECK(run_failed_at_pair(&fixture, publish_middle, + "LARDON3D_TEST_MATCHER_FAIL_PUBLISH_PAIR_ID", + fixture.pairs[5].candidate_pair_id, 4, + fixture.pairs[4].candidate_pair_id)); + Lardon3DMatcherTaskConfiguration publish_last = settings; + publish_last.matcher.ratio_threshold = 0.76F; + CHECK(run_failed_at_pair(&fixture, publish_last, + "LARDON3D_TEST_MATCHER_FAIL_PUBLISH_PAIR_ID", + fixture.pairs[PAIR_COUNT - 1].candidate_pair_id, + PERSISTED_PAIR_COUNT - 1, + fixture.pairs[PAIR_COUNT - 2].candidate_pair_id)); + uint64_t estimate_task_id = 0; Lardon3DTask *estimate_task = lardon3d_project_create_matcher_task( &fixture.state, &settings, &estimate_task_id); Lardon3DTaskDurableSnapshot estimate_snapshot; CHECK(estimate_task != NULL && estimate_task_id != 0); CHECK(lardon3d_task_durable_snapshot(estimate_task, &estimate_snapshot)); - CHECK(estimate_snapshot.estimate.gpu_memory_fixed_bytes == - LARDON3D_ORB_VULKAN_PERMANENT_BUFFER_BYTES); - CHECK(estimate_snapshot.estimate.desired_gpu_slots == 1); + /* Parallel Matcher execution is permanently CPU-only even if admission + * later grants one thread; its immutable estimate must reserve no GPU. */ + CHECK(estimate_snapshot.estimate.desired_cpu_threads > 1); + CHECK(estimate_snapshot.estimate.gpu_memory_fixed_bytes == 0); + CHECK(estimate_snapshot.estimate.desired_gpu_slots == 0); + Lardon3DTaskDurableSnapshot current_cpu = estimate_snapshot; + Lardon3DTaskDurableSnapshot current_gpu = current_cpu; + current_gpu.estimate.gpu_memory_fixed_bytes = + LARDON3D_ORB_VULKAN_PERMANENT_BUFFER_BYTES; + current_gpu.estimate.desired_cpu_threads = 1; + current_gpu.estimate.desired_gpu_slots = 1; + lardon3d_task_destroy(estimate_task); + estimate_task = NULL; + +#ifdef LARDON3D_MATCHER_TASK_VULKAN + estimate_task = lardon3d_project_create_matcher_task_with_mode( + &fixture.state, &settings, LARDON3D_MATCHER_TASK_MODE_ORB_VULKAN, + &estimate_task_id); + CHECK(estimate_task != NULL && + lardon3d_task_durable_snapshot(estimate_task, &estimate_snapshot)); + CHECK(same_estimate(&estimate_snapshot.estimate, ¤t_gpu.estimate)); +#endif + + /* Selection failures occur before Task-ID allocation and cannot silently + * become CPU work. The existing create API remains CPU-parallel. */ + Lardon3DMatcherTaskConfiguration wrong_kind = settings; + wrong_kind.matcher.kind = LARDON3D_MATCHER_SIFT_BF; + (void)snprintf(wrong_kind.feature_extractor_kind, + sizeof(wrong_kind.feature_extractor_kind), "sift"); + uint64_t rejected_id = 99; + CHECK(lardon3d_project_create_matcher_task_with_mode( + &fixture.state, &wrong_kind, + LARDON3D_MATCHER_TASK_MODE_ORB_VULKAN, &rejected_id) == NULL && + rejected_id == 0); + rejected_id = 99; + CHECK(lardon3d_project_create_matcher_task_with_mode( + &fixture.state, &settings, (Lardon3DMatcherTaskMode)99, + &rejected_id) == NULL && + rejected_id == 0); + bool gpu_available = fixture.state.hardware_profile.gpu_available; + fixture.state.hardware_profile.gpu_available = false; + rejected_id = 99; + CHECK(lardon3d_project_create_matcher_task_with_mode( + &fixture.state, &settings, + LARDON3D_MATCHER_TASK_MODE_ORB_VULKAN, &rejected_id) == NULL && + rejected_id == 0); + fixture.state.hardware_profile.gpu_available = gpu_available; + Lardon3DOrbVulkanBackend *backend = fixture.state.orb_vulkan_backend; + fixture.state.orb_vulkan_backend = NULL; + rejected_id = 99; + CHECK(lardon3d_project_create_matcher_task_with_mode( + &fixture.state, &settings, + LARDON3D_MATCHER_TASK_MODE_ORB_VULKAN, &rejected_id) == NULL && + rejected_id == 0); + fixture.state.orb_vulkan_backend = backend; + + /* Reconstruction accepts only the two current and two historical complete + * shapes. CPU12 signatures are normalized ephemerally for Task admission; + * neighboring shapes are corruption, not mode inference. */ + Lardon3DTaskReconstructionContext reconstruction = { + .project_path = fixture.state.project_path, + .project_db = fixture.state.project_db, + .resource_governor = fixture.state.resource_governor, + .orb_vulkan_backend = fixture.state.orb_vulkan_backend, + }; + Lardon3DTaskDurableSnapshot historical_cpu = current_cpu; + historical_cpu.estimate.memory_fixed_bytes = 10U * 1024U * 1024U; + historical_cpu.estimate.memory_bytes_per_item = 0; + historical_cpu.estimate.desired_cpu_threads = 12; + Lardon3DTaskDurableSnapshot historical_gpu = current_gpu; + historical_gpu.estimate.memory_fixed_bytes = 10U * 1024U * 1024U; + historical_gpu.estimate.memory_bytes_per_item = 0; + historical_gpu.estimate.desired_cpu_threads = 12; + char reconciled_path[PATH_MAX]; + CHECK(snprintf(reconciled_path, sizeof(reconciled_path), + "%s/.lardon3d/checkpoints/%lu.chk", + fixture.state.project_path, + (unsigned long)estimate_task_id) > 0 && + lardon3d_task_checkpoint_save(reconciled_path, &historical_gpu) == + LARDON3D_TASK_CHECKPOINT_OK); + const Lardon3DTaskDurableSnapshot *accepted[] = { + ¤t_cpu, ¤t_gpu, &historical_cpu, &historical_gpu}; + for (size_t index = 0; index < sizeof(accepted) / sizeof(accepted[0]); ++index) { + Lardon3DTask *restored = NULL; + CHECK(lardon3d_task_kind_registry_restore( + lardon3d_task_kind_registry_production(), + LARDON3D_MATCHER_TASK_KIND, LARDON3D_MATCHER_TASK_KIND_VERSION, + accepted[index], &reconstruction, &restored) == + LARDON3D_TASK_KIND_OK && + restored); + Lardon3DResourceEstimate restored_estimate; + CHECK(lardon3d_task_resource_estimate(restored, &restored_estimate)); + const Lardon3DResourceEstimate *expected = + index == 2 ? ¤t_cpu.estimate + : index == 3 ? ¤t_gpu.estimate + : &accepted[index]->estimate; + CHECK(same_estimate(&restored_estimate, expected)); + lardon3d_task_destroy(restored); + } + Lardon3DTaskDurableSnapshot reconciled_snapshot; + uint32_t reconciled_version = 0; + CHECK(lardon3d_task_checkpoint_load(reconciled_path, &reconciled_snapshot, + &reconciled_version) == + LARDON3D_TASK_CHECKPOINT_OK && + reconciled_version == LARDON3D_TASK_CHECKPOINT_VERSION && + same_estimate(&reconciled_snapshot.estimate, &historical_gpu.estimate)); + char staged_path[PATH_MAX]; + CHECK(snprintf(staged_path, sizeof(staged_path), "%s.next", + reconciled_path) > 0 && + access(staged_path, F_OK) != 0 && errno == ENOENT); + Lardon3DTaskDurableSnapshot malformed[4] = { + historical_cpu, historical_cpu, historical_gpu, historical_gpu}; + malformed[0].estimate.memory_fixed_bytes--; + malformed[1].estimate.memory_bytes_per_item = 1; + malformed[2].estimate.gpu_memory_fixed_bytes++; + malformed[3].estimate.desired_cpu_threads = 11; + for (size_t index = 0; index < sizeof(malformed) / sizeof(malformed[0]); ++index) { + Lardon3DTask *restored = NULL; + CHECK(lardon3d_task_kind_registry_restore( + lardon3d_task_kind_registry_production(), + LARDON3D_MATCHER_TASK_KIND, LARDON3D_MATCHER_TASK_KIND_VERSION, + &malformed[index], &reconstruction, &restored) == + LARDON3D_TASK_KIND_RECONSTRUCTION_FAILED && + restored == NULL); + } lardon3d_task_destroy(estimate_task); stop_runtime(&fixture); diff --git a/tests/test_photo_quality.cpp b/tests/test_photo_quality.cpp index 026df90..6310646 100644 --- a/tests/test_photo_quality.cpp +++ b/tests/test_photo_quality.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -28,6 +29,29 @@ static cv::Mat checker(int size) { return image; } +static std::vector read_file(const std::string &path) { + std::FILE *file = std::fopen(path.c_str(), "rb"); + if (!file) + return {}; + std::vector bytes; + unsigned char buffer[4096]; + size_t count = 0; + while ((count = std::fread(buffer, 1, sizeof(buffer), file)) != 0) + bytes.insert(bytes.end(), buffer, buffer + count); + if (std::ferror(file) != 0) + bytes.clear(); + std::fclose(file); + return bytes; +} + +static bool write_file(const std::string &path, const std::vector &bytes) { + std::FILE *file = std::fopen(path.c_str(), "wb"); + if (!file) + return false; + const bool written = std::fwrite(bytes.data(), 1, bytes.size(), file) == bytes.size(); + return std::fclose(file) == 0 && written; +} + int main() { char directory_template[] = "/tmp/lardon3d-photo-quality-XXXXXX"; char *directory = mkdtemp(directory_template); @@ -42,6 +66,9 @@ int main() { const std::string over_budget_oversized_path = std::string(directory) + "/over-budget-oversized.jpg"; const std::string malformed_path = std::string(directory) + "/malformed.jpg"; + const std::string mpf_path = std::string(directory) + "/mpf.jpg"; + const std::string mpf_garbage_path = std::string(directory) + "/mpf-garbage.jpg"; + const std::string ordinary_trailer_path = std::string(directory) + "/ordinary-trailer.jpg"; cv::Mat sharp = checker(512); cv::Mat blurred; @@ -53,6 +80,31 @@ int main() { CHECK(cv::imwrite(blur_path, blurred)); CHECK(cv::imwrite(large_path, large)); CHECK(cv::imwrite(white_path, white)); + { + const std::vector primary = read_file(sharp_path); + const std::vector secondary = read_file(blur_path); + CHECK(primary.size() > 2 && secondary.size() > 2 && primary[0] == 0xff && + primary[1] == 0xd8); + /* APP2 MPF evidence belongs to the primary marker stream. The secondary + * image and zero-only physical gaps model the frozen Sony container shape. */ + const unsigned char app2_mpf[] = {0xff, 0xe2, 0x00, 0x06, 'M', 'P', 'F', 0x00}; + std::vector mpf = {primary[0], primary[1]}; + mpf.reserve(primary.size() + secondary.size() + sizeof(app2_mpf) + 12); + mpf.insert(mpf.end(), std::begin(app2_mpf), std::end(app2_mpf)); + mpf.insert(mpf.end(), primary.begin() + 2, primary.end()); + mpf.insert(mpf.end(), 7, 0); + mpf.insert(mpf.end(), secondary.begin(), secondary.end()); + mpf.insert(mpf.end(), 5, 0); + CHECK(write_file(mpf_path, mpf)); + + std::vector mpf_garbage = mpf; + mpf_garbage.push_back(0x42); + CHECK(write_file(mpf_garbage_path, mpf_garbage)); + + std::vector ordinary_trailer = primary; + ordinary_trailer.insert(ordinary_trailer.end(), secondary.begin(), secondary.end()); + CHECK(write_file(ordinary_trailer_path, ordinary_trailer)); + } cv::Mat oversized(2, LARDON3D_PHOTO_QUALITY_JPEG_MAX_DIMENSION + 1, CV_8UC1, cv::Scalar(127)); CHECK(cv::imwrite(oversized_path, oversized)); @@ -103,6 +155,20 @@ int main() { CHECK(white_metrics.clipped_white_fraction > 0.99); CHECK(white_metrics.recommendation == LARDON3D_PHOTO_QUALITY_REJECT); + Lardon3DPhotoQualityMetrics mpf_metrics{}; + CHECK(lardon3d_photo_quality_analyze_jpeg(mpf_path.c_str(), &mpf_metrics) == + LARDON3D_PHOTO_QUALITY_METRIC_OK); + CHECK(mpf_metrics.decoded_width == sharp_metrics.decoded_width); + CHECK(mpf_metrics.decoded_height == sharp_metrics.decoded_height); + Lardon3DPhotoQualityMetrics mpf_garbage_metrics{}; + CHECK(lardon3d_photo_quality_analyze_jpeg(mpf_garbage_path.c_str(), + &mpf_garbage_metrics) == + LARDON3D_PHOTO_QUALITY_METRIC_DECODE_ERROR); + Lardon3DPhotoQualityMetrics ordinary_trailer_metrics{}; + CHECK(lardon3d_photo_quality_analyze_jpeg(ordinary_trailer_path.c_str(), + &ordinary_trailer_metrics) == + LARDON3D_PHOTO_QUALITY_METRIC_DECODE_ERROR); + CHECK(lardon3d_photo_quality_analyze_jpeg(large_path.c_str(), &large_metrics) == LARDON3D_PHOTO_QUALITY_METRIC_OK); const double scale_ratio = sharp_metrics.sharpness_normalized / diff --git a/tests/test_project.c b/tests/test_project.c index 8b4c282..d2e1f21 100644 --- a/tests/test_project.c +++ b/tests/test_project.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -68,6 +69,49 @@ checkpoint_thread(void *userdata) return NULL; } +typedef struct { + Lardon3DAppState *state; + const Lardon3DTaskKindRegistry *registry; + Lardon3DProjectRecoveryEntry entry; + size_t count; + Lardon3DProjectDbResult result; +} RecoveryThread; + +static void * +recovery_thread(void *userdata) +{ + RecoveryThread *context = userdata; + context->result = lardon3d_project_list_recoverable( + context->state, context->registry, 0, &context->entry, 1, &context->count); + return NULL; +} + +static bool +checkpoint_rendezvous(int sockets[2], const char *variable) +{ + char descriptor[32]; + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sockets) != 0 + || snprintf(descriptor, sizeof(descriptor), "%d", sockets[1]) <= 0 + || setenv(variable, descriptor, 1) != 0) { + return false; + } + return true; +} + +static bool +wait_rendezvous(int socket) +{ + unsigned char token = 0; + return read(socket, &token, sizeof(token)) == (ssize_t)sizeof(token); +} + +static bool +release_rendezvous(int socket) +{ + const unsigned char token = 1; + return write(socket, &token, sizeof(token)) == (ssize_t)sizeof(token); +} + static bool run_test(void) { @@ -136,9 +180,9 @@ run_test(void) CHECK(lardon3d_project_checkpoint_task(&state, task) == LARDON3D_PROJECT_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE); CHECK(unsetenv("LARDON3D_TEST_CHECKPOINT_SYNC_DIRECTORY_FAILURE") == 0); CHECK(lardon3d_project_db_load_task(state.project_db, 1, &db_task) == LARDON3D_PROJECT_DB_OK); - CHECK(db_task.checkpoint.durability == LARDON3D_DB_CHECKPOINT_PUBLISHED_NOT_DURABLE); + CHECK(db_task.checkpoint.durability == LARDON3D_DB_CHECKPOINT_DURABLE); CHECK(lardon3d_project_list_recoverable(&state, ®istry, 0, entries, 2, &count) == LARDON3D_PROJECT_DB_OK); - CHECK(count == 1 && entries[0].status == LARDON3D_PROJECT_RECOVERABLE_PUBLISHED_NOT_DURABLE); + CHECK(count == 1 && entries[0].status == LARDON3D_PROJECT_RECOVERABLE); CHECK(lardon3d_task_set_progress(task, 20, "frontière 20")); CHECK(setenv("LARDON3D_TEST_CHECKPOINT_PREPUBLICATION_FAILURE", "1", 1) == 0); @@ -160,20 +204,71 @@ run_test(void) CHECK(lardon3d_project_checkpoint_task(&state, task) == LARDON3D_PROJECT_TASK_CHECKPOINT_DB_BUSY); CHECK(unsetenv("LARDON3D_TEST_PROJECT_DB_BUSY_CHECKPOINT") == 0); CHECK(lardon3d_task_checkpoint_load(checkpoint_path, &disk_snapshot, &version) == LARDON3D_TASK_CHECKPOINT_OK); - CHECK(disk_snapshot.progress == 25); + CHECK(disk_snapshot.progress == 10); + CHECK(lardon3d_task_checkpoint_load_staged(checkpoint_path, &disk_snapshot, &version) + == LARDON3D_TASK_CHECKPOINT_OK && disk_snapshot.progress == 25); CHECK(lardon3d_project_db_load_task(state.project_db, 1, &db_task) == LARDON3D_PROJECT_DB_OK && db_task.progress == 10); + /* DB=S0 retains canonical S0 despite a fully durable, stale staged S1. */ + CHECK(lardon3d_project_list_recoverable(&state, ®istry, 0, entries, 2, &count) + == LARDON3D_PROJECT_DB_OK && count == 1 + && entries[0].status == LARDON3D_PROJECT_RECOVERABLE + && entries[0].snapshot.progress == 10); CHECK(lardon3d_task_set_progress(task, 20, "frontière 20")); CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_CHECKPOINT", "1", 1) == 0); CHECK(lardon3d_project_checkpoint_task(&state, task) == LARDON3D_PROJECT_TASK_CHECKPOINT_DB_ERROR); CHECK(unsetenv("LARDON3D_TEST_PROJECT_DB_FAIL_CHECKPOINT") == 0); CHECK(lardon3d_task_checkpoint_load(checkpoint_path, &disk_snapshot, &version) == LARDON3D_TASK_CHECKPOINT_OK); - CHECK(disk_snapshot.progress == 20); + CHECK(disk_snapshot.progress == 10); + CHECK(lardon3d_task_checkpoint_load_staged(checkpoint_path, &disk_snapshot, &version) + == LARDON3D_TASK_CHECKPOINT_OK && disk_snapshot.progress == 20); CHECK(lardon3d_project_db_load_task(state.project_db, 1, &db_task) == LARDON3D_PROJECT_DB_OK && db_task.progress == 10); CHECK(lardon3d_project_checkpoint_task(&state, task) == LARDON3D_PROJECT_TASK_CHECKPOINT_OK); + /* Forced interruption after the DB transaction leaves DB=S1, canonical + * S0, and staged S1. Reopen/list recovery must accept only staged S1 and + * repair canonical publication without any database reconciliation. */ + CHECK(lardon3d_task_set_progress(task, 30, "frontière 30")); + CHECK(setenv("LARDON3D_TEST_PROJECT_CHECKPOINT_AFTER_DB_COMMIT", "1", 1) == 0); + CHECK(lardon3d_project_checkpoint_task(&state, task) + == LARDON3D_PROJECT_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE); + CHECK(unsetenv("LARDON3D_TEST_PROJECT_CHECKPOINT_AFTER_DB_COMMIT") == 0); + CHECK(lardon3d_project_db_load_task(state.project_db, 1, &db_task) + == LARDON3D_PROJECT_DB_OK && db_task.progress == 30); + CHECK(lardon3d_task_checkpoint_load(checkpoint_path, &disk_snapshot, NULL) + == LARDON3D_TASK_CHECKPOINT_OK && disk_snapshot.progress == 20); + CHECK(lardon3d_task_checkpoint_load_staged(checkpoint_path, &disk_snapshot, NULL) + == LARDON3D_TASK_CHECKPOINT_OK && disk_snapshot.progress == 30); + lardon3d_project_close(&state); + CHECK(lardon3d_project_open(&state, "Projet Cycle")); + CHECK(lardon3d_project_list_recoverable(&state, ®istry, 0, entries, 2, &count) + == LARDON3D_PROJECT_DB_OK && count == 1 + && entries[0].snapshot.progress == 30); + CHECK(lardon3d_task_checkpoint_load(checkpoint_path, &disk_snapshot, NULL) + == LARDON3D_TASK_CHECKPOINT_OK && disk_snapshot.progress == 30); + CHECK(lardon3d_project_list_recoverable(&state, ®istry, 0, entries, 2, &count) == LARDON3D_PROJECT_DB_OK); - CHECK(count == 1 && entries[0].status == LARDON3D_PROJECT_RECOVERABLE && entries[0].snapshot.progress == 20); + CHECK(count == 1 && entries[0].status == LARDON3D_PROJECT_RECOVERABLE && entries[0].snapshot.progress == 30); + + /* Recovery enumerates S0 before waiting. The writer then publishes S1; + * recovery must reload S1 under .chk.lock rather than validate S0. */ + int stale_row_sockets[2]; + CHECK(checkpoint_rendezvous(stale_row_sockets, + "LARDON3D_TEST_RECOVERY_BEFORE_CHECKPOINT_LOCK_FD")); + RecoveryThread stale_row = {.state = &state, .registry = ®istry}; + pthread_t stale_row_thread; + CHECK(pthread_create(&stale_row_thread, NULL, recovery_thread, &stale_row) == 0); + CHECK(wait_rendezvous(stale_row_sockets[0])); + CHECK(lardon3d_task_set_progress(task, 35, "frontière 35")); + CHECK(lardon3d_project_checkpoint_task(&state, task) == LARDON3D_PROJECT_TASK_CHECKPOINT_OK); + CHECK(release_rendezvous(stale_row_sockets[0])); + CHECK(pthread_join(stale_row_thread, NULL) == 0); + CHECK(unsetenv("LARDON3D_TEST_RECOVERY_BEFORE_CHECKPOINT_LOCK_FD") == 0); + CHECK(close(stale_row_sockets[0]) == 0 && close(stale_row_sockets[1]) == 0); + CHECK(stale_row.result == LARDON3D_PROJECT_DB_OK && stale_row.count == 1 + && stale_row.entry.status == LARDON3D_PROJECT_RECOVERABLE + && stale_row.entry.snapshot.progress == 35); + CHECK(lardon3d_project_list_recoverable(&state, ®istry, 0, entries, 257, &count) == LARDON3D_PROJECT_DB_INVALID_ARGUMENT); CHECK(unlink(checkpoint_path) == 0); @@ -206,7 +301,10 @@ run_test(void) pthread_t threads[2]; CHECK(pthread_create(&threads[0], NULL, checkpoint_thread, &contexts[0]) == 0); CHECK(pthread_create(&threads[1], NULL, checkpoint_thread, &contexts[1]) == 0); CHECK(pthread_join(threads[0], NULL) == 0 && pthread_join(threads[1], NULL) == 0); - CHECK(contexts[0].result == LARDON3D_PROJECT_TASK_CHECKPOINT_OK && contexts[1].result == LARDON3D_PROJECT_TASK_CHECKPOINT_OK); + CHECK((contexts[0].result == LARDON3D_PROJECT_TASK_CHECKPOINT_OK + || contexts[0].result == LARDON3D_PROJECT_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE) + && (contexts[1].result == LARDON3D_PROJECT_TASK_CHECKPOINT_OK + || contexts[1].result == LARDON3D_PROJECT_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE)); Lardon3DTask *terminal = lardon3d_task_create_typed( "Terminale", &estimate, "test.persisted", 1, @@ -257,7 +355,7 @@ run_test(void) Lardon3DTaskSnapshot restored_snapshot; CHECK(restored && lardon3d_task_snapshot(restored, &restored_snapshot)); CHECK(restored_snapshot.id == 1 && restored_snapshot.state == TASK_PENDING - && restored_snapshot.progress == 20 + && restored_snapshot.progress == 35 && lardon3d_task_sequence_count(restored) == 0); lardon3d_task_destroy(restored); @@ -344,8 +442,13 @@ run_test(void) CHECK(unlink(checkpoint_path) == 0); CHECK(unlink(terminal_checkpoint) == 0); CHECK(unlink(unknown_checkpoint) == 0); CHECK(unlink(future_kind_checkpoint) == 0); CHECK(unlink(legacy_path) == 0); - CHECK(unlink(database_path) == 0); CHECK(unlink(ini_path) == 0); char path[512]; + const unsigned int locked_task_ids[] = {1, 2, 3, 4, 5}; + for (size_t index = 0; index < sizeof(locked_task_ids) / sizeof(locked_task_ids[0]); ++index) { + CHECK(snprintf(path, sizeof(path), "%s/.lardon3d/checkpoints/%u.chk.lock", project_path, + locked_task_ids[index]) > 0 && unlink(path) == 0); + } + CHECK(unlink(database_path) == 0); CHECK(unlink(ini_path) == 0); CHECK(snprintf(path, sizeof(path), "%s/.lardon3d/checkpoints", project_path) > 0 && rmdir(path) == 0); CHECK(snprintf(path, sizeof(path), "%s/.lardon3d", project_path) > 0 && rmdir(path) == 0); const char *directories[] = {"images", "reconstruction", "exports", "logs"}; diff --git a/tests/test_task_checkpoint.c b/tests/test_task_checkpoint.c index d9e1eb7..bf96e1a 100644 --- a/tests/test_task_checkpoint.c +++ b/tests/test_task_checkpoint.c @@ -246,6 +246,22 @@ run_test(void) == LARDON3D_TASK_CHECKPOINT_OK); CHECK(loaded.id == 99 && loaded.saved_state == TASK_COMPLETED); + /* .next is independently durable and may coexist with a valid canonical + * checkpoint until the Project DB snapshot has committed. */ + durable.id = 101; + CHECK(lardon3d_task_checkpoint_stage(path, &durable) + == LARDON3D_TASK_CHECKPOINT_OK); + CHECK(lardon3d_task_checkpoint_load(path, &loaded, NULL) + == LARDON3D_TASK_CHECKPOINT_OK && loaded.id == 99); + CHECK(lardon3d_task_checkpoint_load_staged(path, &loaded, NULL) + == LARDON3D_TASK_CHECKPOINT_OK && loaded.id == 101); + CHECK(lardon3d_task_checkpoint_promote_staged(path) + == LARDON3D_TASK_CHECKPOINT_OK); + CHECK(lardon3d_task_checkpoint_load(path, &loaded, NULL) + == LARDON3D_TASK_CHECKPOINT_OK && loaded.id == 101); + CHECK(lardon3d_task_checkpoint_load_staged(path, &loaded, NULL) + == LARDON3D_TASK_CHECKPOINT_NOT_FOUND); + CHECK(mkdir(uncertain_directory, 0700) == 0); Lardon3DTaskDurableSnapshot previous = durable; previous.id = 98; diff --git a/tests/test_task_kind_registry.c b/tests/test_task_kind_registry.c index d6e5319..cefd033 100644 --- a/tests/test_task_kind_registry.c +++ b/tests/test_task_kind_registry.c @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -8,6 +9,23 @@ #include "resource_snapshot_test_utils.h" +typedef struct { + Lardon3DTaskCallback callback; + void *userdata; + Lardon3DTaskUserdataDestroy userdata_destroy; + Lardon3DTaskFinishedCallback finished_callback; + void *finished_userdata; +} HistoricalTaskKindBinding; + +/* The registry binding is public C17 ABI. Legacy estimate normalization must + * remain private and must not append fields to this established layout. */ +_Static_assert(sizeof(Lardon3DTaskKindBinding) == + sizeof(HistoricalTaskKindBinding), + "Lardon3DTaskKindBinding ABI size changed"); +_Static_assert(offsetof(Lardon3DTaskKindBinding, finished_userdata) == + offsetof(HistoricalTaskKindBinding, finished_userdata), + "Lardon3DTaskKindBinding ABI layout changed"); + #define CHECK(condition) do { if (!(condition)) { \ (void)fprintf(stderr, "Échec ligne %d : %s\n", __LINE__, #condition); return false; \ } } while (0) diff --git a/tests/test_track_builder_project.cpp b/tests/test_track_builder_project.cpp index ac799cf..b3b4d76 100644 --- a/tests/test_track_builder_project.cpp +++ b/tests/test_track_builder_project.cpp @@ -366,6 +366,27 @@ void run_revalidation_and_duplicate_scope() { } void run_remaining_matrix() { + { + Fixture fixture; + auto v1 = fixture.add_gvr(0, 1, {{0, 0, 0.1F}}, {0x01}); + auto v2 = fixture.add_gvr(1, 2, {{0, 0, 0.1F}}, {0x01}); + char sql[512]; + std::snprintf( + sql, sizeof(sql), + "UPDATE geometric_verification_results SET verifier_version=2," + "parameter_fingerprint=X'4343434343434343434343434343434343434343434343434343434343434343' " + "WHERE geometric_verification_result_id=%llu", + static_cast(v2)); + execute_sql(fixture.db_path, sql); + uint64_t mixed_ids[] = {v1, v2}; + auto request = fixture.request(mixed_ids, 2); + Lardon3DTrackBuilderProjectResult result{}; + CHECK(lardon3d_track_builder_build_project(&request, &result) != + LARDON3D_TRACK_BUILDER_PROJECT_OK && + track_set_count(fixture.db_path) == 0); + std::puts("GV-V1/V2 SELECTOR: PASS (mixed lineage rejected)"); + } + { Fixture fixture; auto gvr = fixture.add_gvr(0, 1, {{0, 0, 0.1F}}, {0x01}); diff --git a/tests/test_visual_index.c b/tests/test_visual_index.c index 902ecb1..4df4e72 100644 --- a/tests/test_visual_index.c +++ b/tests/test_visual_index.c @@ -17,6 +17,8 @@ #include #include +#include "../src/visual_index_internal.h" + #define CHECK(condition) \ do { \ if (!(condition)) { \ @@ -300,6 +302,33 @@ static bool replace_segment_metadata(const char *database_path, uint64_t segment return sqlite3_close(connection) == SQLITE_OK && ok; } +static bool reset_visual_index_fixture(const char *database_path, uint64_t index_id) { + sqlite3 *connection = NULL; + sqlite3_stmt *memberships = NULL; + sqlite3_stmt *segments = NULL; + bool ok = sqlite3_open(database_path, &connection) == SQLITE_OK && + sqlite3_exec(connection, "PRAGMA foreign_keys=ON;BEGIN IMMEDIATE", NULL, NULL, + NULL) == SQLITE_OK && + sqlite3_prepare_v2(connection, + "DELETE FROM visual_index_memberships WHERE visual_index_id=?1", + -1, &memberships, NULL) == SQLITE_OK && + sqlite3_prepare_v2(connection, + "DELETE FROM visual_index_segments WHERE visual_index_id=?1", -1, + &segments, NULL) == SQLITE_OK; + if (ok) { + sqlite3_bind_int64(memberships, 1, (sqlite3_int64)index_id); + sqlite3_bind_int64(segments, 1, (sqlite3_int64)index_id); + ok = sqlite3_step(memberships) == SQLITE_DONE && sqlite3_step(segments) == SQLITE_DONE && + sqlite3_exec(connection, "COMMIT", NULL, NULL, NULL) == SQLITE_OK; + } + if (!ok && connection) { + (void)sqlite3_exec(connection, "ROLLBACK", NULL, NULL, NULL); + } + sqlite3_finalize(memberships); + sqlite3_finalize(segments); + return connection && sqlite3_close(connection) == SQLITE_OK && ok; +} + static bool publish_mutated_segment(const char *root, const char *database_path, const Lardon3DProjectDbVisualIndexSegment *segment, const unsigned char *bytes, size_t size, char path[PATH_MAX], @@ -515,6 +544,74 @@ static bool run_test(void) { CHECK(lardon3d_visual_index_update_once(root, database, index_id, 0, 0, 16, &last, &indexed) == LARDON3D_VISUAL_INDEX_OK && indexed == 5 && last == empty_feature.feature_set_id); + /* The parallel builder may complete Feature Files in any order, but its + * canonical reduction must publish the exact serial segment and queries. */ + Lardon3DProjectDbVisualIndexSegment serial_segment; + Lardon3DProjectDbVisualIndexSegment parallel_segment; + size_t serial_segment_count = 0; + CHECK(lardon3d_project_db_list_visual_index_segments(database, index_id, 0, &serial_segment, 1, + &serial_segment_count) == + LARDON3D_PROJECT_DB_OK && + serial_segment_count == 1); + uint64_t serial_memberships[LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX]; + uint64_t parallel_memberships[LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX]; + size_t serial_membership_count = 0; + CHECK(lardon3d_project_db_list_visual_index_memberships( + database, index_id, 0, serial_memberships, + LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX, &serial_membership_count) == + LARDON3D_PROJECT_DB_OK); + Lardon3DVisualIndexQueryOptions equality_options = { + .top_k = 4, + .minimum_evidence_count = 1, + .scanset_filter = LARDON3D_VISUAL_INDEX_ANY_SCANSET, + }; + Lardon3DVisualIndexCandidate serial_candidates[4]; + size_t serial_candidate_count = 0; + CHECK(lardon3d_visual_index_query(root, database, index_id, feature_a.feature_set_id, + &equality_options, serial_candidates, 4, + &serial_candidate_count) == LARDON3D_VISUAL_INDEX_OK && + reset_visual_index_fixture(database_path, index_id)); + uint64_t parallel_last = 0; + size_t parallel_indexed = 0; + size_t parallel_segment_count = 0; + size_t parallel_membership_count = 0; + Lardon3DVisualIndexCandidate parallel_candidates[4]; + size_t parallel_candidate_count = 0; + CHECK(lardon3d_visual_index_update_once_parallel(root, database, index_id, 0, 0, 16, 4, + ¶llel_last, ¶llel_indexed) == + LARDON3D_VISUAL_INDEX_OK && + parallel_indexed == indexed && parallel_last == last && + lardon3d_project_db_list_visual_index_segments(database, index_id, 0, ¶llel_segment, + 1, ¶llel_segment_count) == + LARDON3D_PROJECT_DB_OK && + parallel_segment_count == 1 && + lardon3d_project_db_list_visual_index_memberships( + database, index_id, 0, parallel_memberships, + LARDON3D_VISUAL_INDEX_SEGMENT_FEATURE_SET_MAX, ¶llel_membership_count) == + LARDON3D_PROJECT_DB_OK && + lardon3d_visual_index_query(root, database, index_id, feature_a.feature_set_id, + &equality_options, parallel_candidates, 4, + ¶llel_candidate_count) == LARDON3D_VISUAL_INDEX_OK && + serial_segment.generation == parallel_segment.generation && + serial_segment.size_bytes == parallel_segment.size_bytes && + serial_segment.posting_count == parallel_segment.posting_count && + serial_segment.feature_set_count == parallel_segment.feature_set_count && + serial_segment.durability == parallel_segment.durability && + memcmp(serial_segment.sha256, parallel_segment.sha256, + sizeof(serial_segment.sha256)) == 0 && + strcmp(serial_segment.path, parallel_segment.path) == 0 && + serial_membership_count == parallel_membership_count && + memcmp(serial_memberships, parallel_memberships, + serial_membership_count * sizeof(*serial_memberships)) == 0 && + serial_candidate_count == parallel_candidate_count); + for (size_t i = 0; i < serial_candidate_count; ++i) { + CHECK(serial_candidates[i].feature_set_id == parallel_candidates[i].feature_set_id && + serial_candidates[i].image_id == parallel_candidates[i].image_id && + serial_candidates[i].scanset_id == parallel_candidates[i].scanset_id && + serial_candidates[i].score == parallel_candidates[i].score && + serial_candidates[i].evidence_count == parallel_candidates[i].evidence_count && + serial_candidates[i].same_image_asset == parallel_candidates[i].same_image_asset); + } CHECK(lardon3d_visual_index_update_once(root, database, index_id, 0, last, 16, &last, &indexed) == LARDON3D_VISUAL_INDEX_NO_CHANGE && indexed == 0);