From 02c2ecb22cd4dd258e1274ef5e9416ac3dd6d43f Mon Sep 17 00:00:00 2001 From: fy59 Date: Sat, 8 Aug 2026 11:41:34 +0200 Subject: [PATCH] feat(import): migrate image import to durable task runtime --- README.md | 7 +- docs/architecture/persistence.md | 28 +- docs/architecture/project_database.md | 47 +- docs/architecture/reconstruction_pipeline.md | 4 +- docs/architecture/resource_governor.md | 7 + docs/architecture/runtime.md | 15 +- docs/architecture/task_kind_registry.md | 10 +- docs/architecture/task_system.md | 22 +- docs/concepts/scan_sets.md | 2 + docs/roadmap/roadmap.md | 7 +- include/lardon3d/import.h | 11 + include/lardon3d/import_task.h | 30 +- include/lardon3d/project.h | 6 + include/lardon3d/project_db.h | 25 +- include/lardon3d/resource_governor.h | 6 +- include/lardon3d/task.h | 11 + include/lardon3d/task_kind_registry.h | 3 + meson.build | 19 +- src/import.c | 185 +++++++- src/import_task.c | 453 +++++++++++-------- src/project.c | 44 +- src/project_db.c | 189 +++++++- src/resource_governor.c | 2 + src/task.c | 63 ++- src/task_kind_registry.c | 7 + src/task_kinds.c | 17 + src/task_queue.c | 20 +- tests/test_import.c | 25 +- tests/test_import_task.c | 403 ++++++----------- tests/test_project_db.c | 92 +++- tests/test_resource_governor.c | 20 + tests/test_task.c | 45 ++ tests/test_task_kind_registry.c | 26 +- tests/test_task_queue.c | 15 + 34 files changed, 1324 insertions(+), 542 deletions(-) create mode 100644 src/task_kinds.c diff --git a/README.md b/README.md index 9f5c7c9..a395d43 100644 --- a/README.md +++ b/README.md @@ -21,13 +21,12 @@ Lardon3D ne vise pas simplement "dossier de photos → objet 3D", mais "ensemble ### Briques implémentées (IMPLEMENTED) - **Project** : cycle de vie persistant, identité stable et Project Database ouverte -- **Import** : import asynchrone et annulable d'images -- **Import Task** : wrapper asynchrone avec états et progression +- **Import** : premier task kind de production, exécuté par la file générique en lots bornés et reprenables - **Image Catalog** : indexage des métadonnées d'images - **Image View** : vues triées et filtrées pour la TUI - **Task** : moteur de tâches avec pause/reprise, annulation et séquences - **Task Checkpoint v1** : snapshot durable, fichier atomique et reprise sûre -- **Project Database v2** : identité, tâches typées/checkpoints et inventaire d'artefacts SQLite +- **Project Database v3** : identité, tâches typées/checkpoints, paramètres d'import et inventaire d'artefacts SQLite - **Task Kind Registry** : identité métier durable et reconstruction runtime explicite - **Task Queue** : file FIFO avec sélection adaptative et backpressure - **Hardware Profile** : détection des capacités matérielles @@ -41,7 +40,7 @@ Lardon3D ne vise pas simplement "dossier de photos → objet 3D", mais "ensemble ### Briques prévues (PLANNED) -- Premiers types métier reconstructibles et resoumission contrôlée des tâches récupérables +- Resoumission contrôlée des tâches récupérables à l'ouverture - DAG de dépendances - Pools de workers multiples (CPU/GPU/IO) - Publication live validée diff --git a/docs/architecture/persistence.md b/docs/architecture/persistence.md index 95a3e2f..ad909fc 100644 --- a/docs/architecture/persistence.md +++ b/docs/architecture/persistence.md @@ -108,7 +108,7 @@ par le `time_t` local avant conversion. Le format reste donc lisible entre plateformes uniquement pour les valeurs communes à leurs domaines `size_t` et `time_t`. -## Project Database v2 +## Project Database v3 SQLite contient l'état logique interrogable et les références aux fichiers ; les checkpoints et artefacts volumineux restent externes. L'enregistrement du @@ -136,7 +136,7 @@ non durable, absent, invalide, version inconnue et erreur d'I/O. Aucune réparat ou suppression silencieuse n'est effectuée. Le format checkpoint reste en version 1 et ne contient pas de `task_kind`. Le -schéma SQLite v2 conserve `task_kind` et `task_kind_version` dans le résumé +schéma SQLite v3 conserve `task_kind` et `task_kind_version` dans le résumé logique interrogable. La migration v1→v2 laisse ces deux colonnes à `NULL` : une tâche legacy reste inspectable mais ne peut pas être reconstruite ou resoumise. Un kind inconnu ou une version non supportée est diagnostiqué sans exécuter de @@ -147,7 +147,7 @@ code. **IMPLEMENTED** — modèle durable, codec v1, lecture validée, publication atomique et restauration sûre d'une tâche isolée. -**IMPLEMENTED** — Project Database v2 pour identité, résumés de tâches typées, +**IMPLEMENTED** — Project Database v3 pour identité, résumés de tâches typées, références checkpoint et artefacts génériques. **IMPLEMENTED** — registry statique bornée et reconstruction explicite avec @@ -156,8 +156,24 @@ ownership du userdata. **IMPLEMENTED** — API projet de sauvegarde fichier+DB et inventaire validé au redémarrage. -**NOT_YET_WIRED** — autosave complet, premier type métier de production, -resoumission scheduler et réconciliation des fichiers orphelins. +**IMPLEMENTED** — `import.images` persiste son chemin source absolu dans une +table dédiée et publie un checkpoint après chaque lot validé. Le manifeste +publié rend le rejeu idempotent à la granularité d'une image. -**PLANNED** — catalogue d'artefacts photogrammétriques réels, migrations v3+ et +Le chemin source absolu est l'intention durable v1 : il doit rester accessible +après redémarrage et un projet déplacé ne rend pas une source externe portable. +Une source absente ou devenue non-répertoire fait échouer proprement la +reconstruction. Une image déjà inscrite au manifeste est un résultat validé et +les modifications ultérieures de sa source sont ignorées. Pour la fenêtre +« copie publiée, manifeste non publié », la reprise n'adopte la destination +orpheline qu'après comparaison octet par octet avec la source ; une collision +différente est une erreur. Le manifeste est republié atomiquement avant le +checkpoint de fin de lot. Si ce checkpoint ou sa transaction DB échoue, le +manifeste demeure la frontière idempotente plus récente et un checkpoint +orphelin peut subsister selon le protocole filesystem puis SQLite. + +**NOT_YET_WIRED** — autosave complet, resoumission scheduler et réconciliation +des fichiers orphelins. + +**PLANNED** — catalogue d'artefacts photogrammétriques réels, migrations v4+ et reprise globale. diff --git a/docs/architecture/project_database.md b/docs/architecture/project_database.md index 66e95c6..154332e 100644 --- a/docs/architecture/project_database.md +++ b/docs/architecture/project_database.md @@ -167,9 +167,10 @@ référence vers le fichier checkpoint. Le fichier checkpoint validé reste la source complète pour `lardon3d_task_restore()` ; la DB seule ne reconstruit jamais une tâche. Un écart ou un fichier invalide interdit la reprise. -## Schéma v2 implémenté +## Schéma v3 implémenté -- `metadata(key PRIMARY KEY, value)` contient uniquement `schema_version=2`. +- `metadata(key PRIMARY KEY, value)` contient `schema_version=3` et + `next_task_id`, prochain ID durable allouable. - `project(singleton=1, stable_id UNIQUE, name, created_at, updated_at)` décrit l'unique identité logique de la DB. - `tasks(task_id PRIMARY KEY, name, task_kind, task_kind_version, saved_state, recovery_state, progress, @@ -181,20 +182,22 @@ jamais une tâche. Un écart ou un fichier invalide interdit la reprise. - `artifacts(artifact_id PRIMARY KEY, kind, path, state, size_bytes, producer_task_id REFERENCES tasks, created_at, updated_at)` inventorie des fichiers externes. Les états v1 sont `STAGED` et `READY`. +- `image_import_tasks(task_id PRIMARY KEY REFERENCES tasks ON DELETE CASCADE, + source_path)` conserve l'unique paramètre métier v1 de `import.images`. Les indexes portent uniquement sur `tasks(recovery_state, task_id)`, `artifacts(state, artifact_id)` et `artifacts(producer_task_id)`. ## Ouverture et migrations -Une DB vide reçoit directement le schéma v2 dans une transaction +Une DB vide reçoit directement le schéma v3 dans une transaction `BEGIN IMMEDIATE`. Une DB v1 reçoit transactionnellement les colonnes nullable -`task_kind` et `task_kind_version`, puis passe à v2. Les anciennes lignes restent +`task_kind` et `task_kind_version`, puis les migrations v2→v3. Les anciennes lignes restent `NULL/NULL`, sans type inventé et sans perte des projets, tâches, checkpoints ou artefacts. Une interruption ou erreur provoque un rollback complet. Une DB v2 -est validée puis ouverte. Une version future est refusée et une DB contenant +est migrée vers v3 ; une DB v3 est validée puis ouverte. Une version future est refusée et une DB contenant des tables sans métadonnée de version est considérée corrompue. La fonction -interne de migration ne connaît que `0 → 2` et `1 → 2`. +interne de migration ne connaît que `0 → 3`, `1 → 2 → 3` et `2 → 3`. Migration v1→v2 exacte, exécutée entre `BEGIN IMMEDIATE` et `COMMIT` : @@ -206,7 +209,25 @@ UPDATE metadata SET value=2 WHERE key='schema_version' AND value=1; ``` -Configuration v2 : `foreign_keys=ON`, `journal_mode=DELETE`, +Migration v2→v3 exacte, exécutée entre `BEGIN IMMEDIATE` et `COMMIT` : + +```sql +CREATE TABLE image_import_tasks( + task_id INTEGER PRIMARY KEY REFERENCES tasks(task_id) ON DELETE CASCADE, + source_path TEXT NOT NULL +); +INSERT INTO metadata(key,value) +VALUES('next_task_id',( + SELECT CASE + WHEN COALESCE(MAX(task_id),0)>=9223372036854775807 THEN 0 + ELSE COALESCE(MAX(task_id),0)+1 + END FROM tasks +)); +UPDATE metadata SET value=3 + WHERE key='schema_version' AND value=2; +``` + +Configuration v3 : `foreign_keys=ON`, `journal_mode=DELETE`, `synchronous=FULL`, `busy_timeout=5000`. Le mode DELETE convient au propriétaire unique actuel, évite les fichiers WAL/SHM durables et conserve la synchronisation forte. Le timeout borne l'attente d'un verrou externe à cinq secondes. @@ -255,8 +276,9 @@ le mutex DB. Un upsert ne peut pas changer le couple kind/version d'un task ID. ## Statut -**IMPLEMENTED** — SQLite système, schéma v2 et migration v1→v2, identité projet, -transactions tâche+checkpoint, pagination de reprise et artefacts génériques. +**IMPLEMENTED** — SQLite système, schéma v3 et migrations v1→v2→v3, identité +projet, transactions tâche+checkpoint, pagination de reprise et artefacts +génériques. **IMPLEMENTED** — ouverture/fermeture avec le projet, identité INI/DB cohérente, publication de checkpoints par le projet et inventaire de reprise validé. @@ -264,9 +286,12 @@ publication de checkpoints par le projet et inventaire de reprise validé. **IMPLEMENTED** — kinds persistants, classification par registry et reconstruction explicite testée hors scheduler. -**NOT_YET_WIRED** — type métier de production, resoumission scheduler, autosave +**IMPLEMENTED** — allocation transactionnelle de task IDs, paramètres +immuables de `import.images` et reconstruction production explicite. + +**NOT_YET_WIRED** — resoumission automatique, autosave à toutes les transitions et réconciliation des checkpoints orphelins, ScanSet et catalogue image persistants, Feature Store et Visual Index. -**PLANNED** — migrations v3+, dépendances d'artefacts, graphe géométrique et +**PLANNED** — migrations v4+, dépendances d'artefacts, graphe géométrique et reconstruction incrémentale. diff --git a/docs/architecture/reconstruction_pipeline.md b/docs/architecture/reconstruction_pipeline.md index 9e5f118..2cbb7c8 100644 --- a/docs/architecture/reconstruction_pipeline.md +++ b/docs/architecture/reconstruction_pipeline.md @@ -38,7 +38,9 @@ pas encore structuré avec UUID et vague d'import. | **État de traitement** | Pour chaque image, le catalogue maintient un état : RAW, FEATURES_EXTRACTED, MATCHED, REGISTERED. Cet état est lu par le pipeline pour décider de l'étape suivante. | **Statut :** PARTIELLEMENT IMPLEMENTÉ — le `image_catalog` gère les -métadonnées de base mais pas encore l'état de traitement ni l'UUID. +métadonnées de base mais pas encore l'état de traitement ni l'UUID. L'import +`import.images` est exécuté par le scheduler générique, copie au plus 32 images +par lot et reprend à partir du manifeste publié sans dupliquer une entrée. --- diff --git a/docs/architecture/resource_governor.md b/docs/architecture/resource_governor.md index 201f749..0dc053b 100644 --- a/docs/architecture/resource_governor.md +++ b/docs/architecture/resource_governor.md @@ -64,6 +64,13 @@ Le Resource Governor est l'unique propriétaire des budgets (RAM, GPU, CPU, IO). - Sous-estimation temporaire possible avec des estimations statiques - Pas d'adaptation basée sur le débit (duration_ns non encore utilisé) +- L'import `import.images` est admis avec 128 Kio fixes, un coût borné par item, + un thread CPU, un slot I/O et des lots de 1 à 32. Il enregistre le nombre + d'images nouvellement publiées dans le manifeste et la durée réelle du lot. + Cela inclut une copie orpheline identique adoptée, même si aucun octet n'est + recopié. `peak_memory_bytes == 0` signifie explicitement « mesure inconnue » : + l'échantillon peut conserver taille/durée mais n'alimente jamais l'adaptation + mémoire. - Pas de communication inter-classes de tâches ## Limites actuelles diff --git a/docs/architecture/runtime.md b/docs/architecture/runtime.md index 56e2473..3f843f4 100644 --- a/docs/architecture/runtime.md +++ b/docs/architecture/runtime.md @@ -75,8 +75,11 @@ qu'à la terminaison de cette tentative. **IMPLEMENTED** — snapshot, codec v1 et restauration isolée. -**NOT_YET_WIRED** — sauvegarde périodique, reconstruction métier des tâches et -resoumission automatique. +**IMPLEMENTED** — l'import `import.images` se sauvegarde à chaque fin de lot et +se reconstruit explicitement avec un userdata neuf lié au projet rouvert. + +**NOT_YET_WIRED** — resoumission automatique à l'ouverture et autosave +générique des autres kinds. **PLANNED** — reprise globale du scheduler via la Project Database. @@ -92,9 +95,11 @@ fermeture finale du projet. **IMPLEMENTED** — la registry reconstruit explicitement callback/userdata hors mutex DB pour un kind connu ; elle ne soumet aucune tâche. -**NOT_YET_WIRED** — aucun type métier de production ni resoumission -automatique. Une future resoumission devra aussi permettre à la queue de -conserver l'identifiant stable restauré au lieu d'en assigner un nouveau. +**IMPLEMENTED** — la queue accepte un identifiant restauré préassigné s'il +n'entre en collision avec aucune tâche connue. L'import production peut donc +être reconstruit puis soumis explicitement. + +**NOT_YET_WIRED** — aucune resoumission automatique à `project_open()`. ## Invariants diff --git a/docs/architecture/task_kind_registry.md b/docs/architecture/task_kind_registry.md index ebd7d88..615b354 100644 --- a/docs/architecture/task_kind_registry.md +++ b/docs/architecture/task_kind_registry.md @@ -26,7 +26,7 @@ fin de l'exécution. Le constructeur métier n'est jamais appelé sous mutex DB. ## Persistance et legacy -Le checkpoint générique reste en version 1. Project Database v2 conserve le +Le checkpoint générique reste en version 1. Project Database v3 conserve le kind/version ; les lignes migrées depuis v1 restent `NULL/NULL` et sont classées `LEGACY_UNTYPED`. Un kind inconnu ou une version non supportée reste inspectable mais inexécutable. Aucun type n'est inventé et aucun code n'est sélectionné par @@ -37,8 +37,12 @@ adresse persistée. **IMPLEMENTED** — identité typée immutable, registry statique, lookup, migration DB v1→v2, classification recovery et restauration explicite testée. -**NOT_YET_WIRED** — type métier de production, paramètres métier persistants, -autosave complet, resoumission automatique et réconciliation orpheline. +**IMPLEMENTED** — le descriptor production `import.images`, version 1, charge +le chemin source borné depuis la table dédiée et reconstruit callback et +userdata sans `AppState *` ancien. + +**NOT_YET_WIRED** — autosave complet, resoumission automatique et +réconciliation orpheline. **PLANNED** — kinds des tâches ScanSet, Image Catalog, Feature Store, Visual Index et reconstruction lorsque ces traitements existeront réellement. diff --git a/docs/architecture/task_system.md b/docs/architecture/task_system.md index d0cfb84..e16bb5b 100644 --- a/docs/architecture/task_system.md +++ b/docs/architecture/task_system.md @@ -79,10 +79,26 @@ adaptatives et fondation de checkpoints persistants isolés. callback et le userdata d'un kind connu. Le destructeur du userdata est détenu par la tâche restaurée et exécuté après arrêt de son exécution. -**NOT_YET_WIRED** — sauvegarde automatique, types métier de production et -restauration par la file. +**IMPLEMENTED** — `import.images` utilise la pause, l'annulation, les ruptures +de séquence et les checkpoints génériques ; une tâche restaurée conserve son ID +lors de sa soumission explicite à la file. -La Project Database v2 peut enregistrer transactionnellement un résumé +**NOT_YET_WIRED** — autosave générique et resoumission automatique. + +Le chemin de production de l'import ne possède plus de thread ni de drapeau +d'annulation privés. Son wrapper TUI ne fait qu'enqueue/cancel/observer la +tâche générique. Chaque callback traite un lot borné, checkpoint hors mutex de +tâche, puis effectue une rupture de séquence afin d'obtenir un nouveau contrat +et une nouvelle réservation. + +Le callback terminal optionnel est notifié exactement une fois pour +`COMPLETED`, `FAILED` ou `CANCELLED`, jamais pour une pause ou une rupture de +séquence. L'état est fixé sous mutex, puis la réservation terminale est libérée +avant l'appel hors mutex. `join()` attend la fin du callback ; le userdata reste +donc valide pendant celui-ci et son destructeur n'est appelé qu'ensuite par la +destruction de la tâche. + +La Project Database v3 peut enregistrer transactionnellement un résumé `Lardon3DTaskDurableSnapshot` et la référence de son checkpoint. Elle ne stocke ni estimation sérialisée complète, ni callback, ni réservation, et ne remplace pas la validation du fichier checkpoint avant `task_restore()`. diff --git a/docs/concepts/scan_sets.md b/docs/concepts/scan_sets.md index 88ad8bb..6ba8f9c 100644 --- a/docs/concepts/scan_sets.md +++ b/docs/concepts/scan_sets.md @@ -9,6 +9,8 @@ Une **acquisition** désigne le processus physique de capture des images : posit ## Statut **PLANNED** — Concept définissant la structure d'entrée, pas encore implémenté comme module distinct. Actuellement, le module `import` et `image_catalog` gèrent les images individuellement sans regroupement en scan sets. +Le chemin source durable de `import.images` ne constitue donc pas encore une +identité de ScanSet. ## Place dans le pipeline diff --git a/docs/roadmap/roadmap.md b/docs/roadmap/roadmap.md index ea80ebf..e3b4a4f 100644 --- a/docs/roadmap/roadmap.md +++ b/docs/roadmap/roadmap.md @@ -9,7 +9,7 @@ Lardon3D suit une feuille de route ordonnée qui privilégie la stabilité et la ### Phase 1 : Fondations - ✅ TUI modulaire avec ncursesw - ✅ Gestion persistante des projets -- ✅ Import asynchrone et annulable +- ✅ Import d'images migré vers le scheduler générique, borné et reconstructible - ✅ Catalogue d'images et vues - ✅ Moteur de tâches avec pause/reprise, annulation, checkpoints - ✅ File FIFO avec sélection adaptative et backpressure @@ -29,10 +29,11 @@ Lardon3D suit une feuille de route ordonnée qui privilégie la stabilité et la ### Phase 3 : Persistance - ✅ Fondation versionnée des checkpoints de tâches -- ✅ Project Database v2 (SQLite, migration v1 et tâches typées) +- ✅ Project Database v3 (paramètres d'import et allocation durable des task IDs) - ✅ Branchement Project Database au cycle de vie projet et inventaire de reprise - ✅ Registry durable des types métier de tâches -- 📋 Premier type métier reconstructible et resoumission des tâches récupérables +- ✅ Premier type métier reconstructible (`import.images`) +- 📋 Resoumission automatique contrôlée des tâches récupérables - 📋 ScanSet et Image Catalog persistants ### Phase 4 : Pipeline avancé diff --git a/include/lardon3d/import.h b/include/lardon3d/import.h index a075e85..a48ed19 100644 --- a/include/lardon3d/import.h +++ b/include/lardon3d/import.h @@ -8,6 +8,8 @@ typedef struct { size_t admissible_found; + size_t processed; + size_t newly_manifested; size_t copied; size_t already_present; size_t ignored; @@ -53,4 +55,13 @@ Lardon3DImportOutcome lardon3d_import_directory_controlled( const Lardon3DImportControl *control ); +Lardon3DImportOutcome lardon3d_import_directory_batch( + Lardon3DAppState *state, + const char *source_directory, + size_t batch_size, + Lardon3DImportResult *result, + const Lardon3DImportControl *control, + bool *complete +); + #endif diff --git a/include/lardon3d/import_task.h b/include/lardon3d/import_task.h index 05beddb..3678cca 100644 --- a/include/lardon3d/import_task.h +++ b/include/lardon3d/import_task.h @@ -3,8 +3,13 @@ #include #include +#include #include +#include + +#define LARDON3D_IMAGE_IMPORT_TASK_KIND "import.images" +enum { LARDON3D_IMAGE_IMPORT_TASK_KIND_VERSION = 1 }; typedef enum { LARDON3D_IMPORT_TASK_IDLE = 0, @@ -26,10 +31,33 @@ typedef struct { typedef struct Lardon3DImportTask Lardon3DImportTask; +typedef struct { + const char *project_path; + Lardon3DProjectDb *project_db; + Lardon3DResourceGovernor *resource_governor; +} Lardon3DImageImportReconstructionContext; + +Lardon3DTask *lardon3d_project_create_image_import_task( + Lardon3DAppState *state, + const char *source_directory, + uint64_t *task_id +); +bool lardon3d_project_enqueue_image_import( + Lardon3DAppState *state, + const char *source_directory, + uint64_t *task_id +); +bool lardon3d_image_import_reconstruct( + const Lardon3DTaskDurableSnapshot *snapshot, + void *context, + Lardon3DTaskKindBinding *binding +); + +/* Compatibilité TUI : handle léger sur une tâche de la queue, sans thread. */ Lardon3DImportTask *lardon3d_import_task_create(void); bool lardon3d_import_task_start( Lardon3DImportTask *task, - const Lardon3DAppState *state, + Lardon3DAppState *state, const char *source_directory ); void lardon3d_import_task_request_cancel(Lardon3DImportTask *task); diff --git a/include/lardon3d/project.h b/include/lardon3d/project.h index b904455..b5faf4d 100644 --- a/include/lardon3d/project.h +++ b/include/lardon3d/project.h @@ -54,6 +54,12 @@ Lardon3DProjectTaskCheckpointResult lardon3d_project_checkpoint_task( Lardon3DAppState *state, const Lardon3DTask *task ); +Lardon3DProjectTaskCheckpointResult +lardon3d_project_checkpoint_image_import_task( + Lardon3DAppState *state, + const Lardon3DTask *task, + const char *source_path +); Lardon3DProjectDbResult lardon3d_project_list_recoverable( Lardon3DAppState *state, const Lardon3DTaskKindRegistry *registry, diff --git a/include/lardon3d/project_db.h b/include/lardon3d/project_db.h index 817c8a8..29d0a17 100644 --- a/include/lardon3d/project_db.h +++ b/include/lardon3d/project_db.h @@ -9,7 +9,7 @@ #include enum { - LARDON3D_PROJECT_DB_SCHEMA_VERSION = 2, + LARDON3D_PROJECT_DB_SCHEMA_VERSION = 3, LARDON3D_PROJECT_DB_ID_CAPACITY = 65, LARDON3D_PROJECT_DB_KIND_CAPACITY = 65, LARDON3D_PROJECT_DB_PATH_CAPACITY = 4096, @@ -83,6 +83,11 @@ typedef struct { int64_t updated_at; } Lardon3DProjectDbArtifact; +typedef struct { + uint64_t task_id; + char source_path[LARDON3D_PROJECT_DB_PATH_CAPACITY]; +} Lardon3DProjectDbImageImport; + Lardon3DProjectDbResult lardon3d_project_db_open( const char *path, Lardon3DProjectDb **database, @@ -111,6 +116,24 @@ Lardon3DProjectDbResult lardon3d_project_db_record_task( const Lardon3DProjectDbCheckpoint *checkpoint, int64_t updated_at ); +Lardon3DProjectDbResult lardon3d_project_db_record_image_import_task( + Lardon3DProjectDb *database, + const Lardon3DTaskDurableSnapshot *snapshot, + const char *task_kind, + uint32_t task_kind_version, + const Lardon3DProjectDbCheckpoint *checkpoint, + const char *source_path, + int64_t updated_at +); +Lardon3DProjectDbResult lardon3d_project_db_load_image_import( + Lardon3DProjectDb *database, + uint64_t task_id, + Lardon3DProjectDbImageImport *parameters +); +Lardon3DProjectDbResult lardon3d_project_db_allocate_task_id( + Lardon3DProjectDb *database, + uint64_t *task_id +); Lardon3DProjectDbResult lardon3d_project_db_load_task( Lardon3DProjectDb *database, uint64_t task_id, diff --git a/include/lardon3d/resource_governor.h b/include/lardon3d/resource_governor.h index 710a02c..62dcedf 100644 --- a/include/lardon3d/resource_governor.h +++ b/include/lardon3d/resource_governor.h @@ -188,8 +188,10 @@ const char *lardon3d_resource_decision_name( Lardon3DResourceDecisionKind kind ); /* Enregistre les métriques d'un lot terminé pour l'adaptation dynamique - * de la taille des lots futurs. Le buffer est borné (8 entrées par classe - * de tâche). Thread-safe. */ + * de la taille des lots futurs. batch_size est le nombre d'éléments dont le + * traitement a été validé dans ce lot. peak_memory_bytes == 0 signifie que + * la mesure est inconnue et n'alimente jamais l'adaptation mémoire. Le buffer + * est borné (8 entrées par classe de tâche). Thread-safe. */ bool lardon3d_resource_governor_record_batch( Lardon3DResourceGovernor *governor, Lardon3DResourceTaskClass task_class, diff --git a/include/lardon3d/task.h b/include/lardon3d/task.h index 04a8553..2100640 100644 --- a/include/lardon3d/task.h +++ b/include/lardon3d/task.h @@ -25,6 +25,10 @@ typedef enum { typedef struct Lardon3DTask Lardon3DTask; typedef bool (*Lardon3DTaskCallback)(Lardon3DTask *task, void *userdata); typedef void (*Lardon3DTaskUserdataDestroy)(void *userdata); +typedef void (*Lardon3DTaskFinishedCallback)( + const Lardon3DTask *task, + void *userdata +); typedef struct { uint64_t id; @@ -120,6 +124,13 @@ bool lardon3d_task_kind( char task_kind[LARDON3D_TASK_KIND_CAPACITY], uint32_t *task_kind_version ); +/* Appelé au plus une fois, hors mutex de tâche et après libération de la + * réservation terminale. Le userdata de tâche reste vivant jusqu'au retour. */ +bool lardon3d_task_set_finished_callback( + Lardon3DTask *task, + Lardon3DTaskFinishedCallback callback, + void *userdata +); uint64_t lardon3d_task_id(const Lardon3DTask *task); bool lardon3d_task_assign_id(Lardon3DTask *task, uint64_t id); bool lardon3d_task_resource_estimate( diff --git a/include/lardon3d/task_kind_registry.h b/include/lardon3d/task_kind_registry.h index a74dede..10ffba8 100644 --- a/include/lardon3d/task_kind_registry.h +++ b/include/lardon3d/task_kind_registry.h @@ -14,6 +14,8 @@ typedef struct { Lardon3DTaskCallback callback; void *userdata; Lardon3DTaskUserdataDestroy userdata_destroy; + Lardon3DTaskFinishedCallback finished_callback; + void *finished_userdata; } Lardon3DTaskKindBinding; typedef bool (*Lardon3DTaskKindReconstruct)( @@ -61,5 +63,6 @@ Lardon3DTaskKindResult lardon3d_task_kind_registry_restore( void *context, Lardon3DTask **task ); +const Lardon3DTaskKindRegistry *lardon3d_task_kind_registry_production(void); #endif diff --git a/meson.build b/meson.build index 53b0e4f..16e9fe8 100644 --- a/meson.build +++ b/meson.build @@ -39,6 +39,7 @@ executable( 'src/task.c', 'src/task_checkpoint.c', 'src/task_kind_registry.c', + 'src/task_kinds.c', 'src/task_queue.c', 'src/resource_governor.c', 'src/resource_snapshot.c', @@ -67,9 +68,25 @@ import_task_test = executable( 'src/app_state.c', 'src/import.c', 'src/import_task.c', + 'src/project.c', + 'src/project_db.c', + 'src/task.c', + 'src/task_checkpoint.c', + 'src/task_kind_registry.c', + 'src/task_kinds.c', + 'src/task_queue.c', + 'src/resource_governor.c', + 'src/resource_snapshot.c', + 'src/image_catalog.c', + 'src/image_view.c', + ], + c_args: [ + '-DLARDON3D_IMPORT_TASK_TESTING', + '-DLARDON3D_PROJECT_DB_TESTING', + '-DLARDON3D_CHECKPOINT_TESTING', ], include_directories: include_directories('include'), - dependencies: [threads], + dependencies: [threads, sqlite3], ) test('import-task', import_task_test, timeout: 30) diff --git a/src/import.c b/src/import.c index 134bfe1..82b4959 100644 --- a/src/import.c +++ b/src/import.c @@ -450,6 +450,16 @@ manifest_commit(Lardon3DAppState *state, ManifestWriter *writer) return success; } +static ssize_t +read_retry(int descriptor, void *buffer, size_t capacity) +{ + ssize_t count; + do { + count = read(descriptor, buffer, capacity); + } while (count < 0 && errno == EINTR); + return count; +} + static int copy_image( const char *source_path, @@ -479,8 +489,38 @@ copy_image( mode ); if (destination < 0) { - (void)close(source); - return errno == EEXIST ? 0 : -1; + if (errno != EEXIST) { + (void)close(source); + return COPY_IMAGE_ERROR; + } + destination = open(destination_path, O_RDONLY | O_NOFOLLOW); + struct stat destination_info; + bool identical = destination >= 0 + && fstat(destination, &destination_info) == 0 + && S_ISREG(destination_info.st_mode) + && destination_info.st_size == source_info.st_size; + char source_buffer[64 * 1024], destination_buffer[64 * 1024]; + while (identical) { + ssize_t source_count = read_retry(source, source_buffer, + sizeof(source_buffer)); + ssize_t destination_count = read_retry(destination, + destination_buffer, sizeof(destination_buffer)); + if (source_count < 0 || destination_count < 0 + || source_count != destination_count) { + identical = false; + break; + } + if (source_count == 0) break; + if (memcmp(source_buffer, destination_buffer, + (size_t)source_count) != 0) { + identical = false; + } + } + if (destination >= 0 && close(destination) != 0) identical = false; + if (close(source) != 0) identical = false; + if (!identical) return COPY_IMAGE_ERROR; + *destination_size = source_info.st_size; + return COPY_IMAGE_COLLISION; } bool success = true; @@ -722,6 +762,147 @@ cancel_import( return LARDON3D_IMPORT_CANCELLED; } +Lardon3DImportOutcome +lardon3d_import_directory_batch( + Lardon3DAppState *state, + const char *source_directory, + size_t batch_size, + Lardon3DImportResult *result, + const Lardon3DImportControl *control, + bool *complete +) +{ + if (complete) *complete = false; + if (!state || !result || !complete || batch_size == 0 + || !state->project_loaded) { + return LARDON3D_IMPORT_FAILED; + } + *result = (Lardon3DImportResult) {0}; + char trimmed_source[PATH_MAX], absolute_source[PATH_MAX]; + if (!trim_source_path(state, source_directory, trimmed_source) + || !resolve_source_path(state, trimmed_source, absolute_source) + || has_forbidden_manifest_character(absolute_source)) { + return LARDON3D_IMPORT_FAILED; + } + struct stat source_info; + if (lstat(absolute_source, &source_info) != 0 + || !S_ISDIR(source_info.st_mode)) { + set_status(state, "Erreur : dossier source absent ou invalide."); + return LARDON3D_IMPORT_FAILED; + } + char images_path[PATH_MAX], originals_path[PATH_MAX]; + if (!join_path(images_path, state->project_path, "images") + || !join_path(originals_path, images_path, "originals") + || !ensure_originals_directory(state, images_path, originals_path)) { + return LARDON3D_IMPORT_FAILED; + } + ManifestWriter manifest; + if (!manifest_begin(state, images_path, &manifest)) { + return LARDON3D_IMPORT_FAILED; + } + DIR *directory = opendir(absolute_source); + if (!directory) { + manifest_abort(&manifest); + set_status(state, "Erreur : impossible d'ouvrir le dossier source."); + return LARDON3D_IMPORT_FAILED; + } + CandidateList created = {0}; + bool success = true, cancelled = false; + size_t added = 0; + for (;;) { + if (import_is_cancelled(control)) { + cancelled = true; + break; + } + errno = 0; + struct dirent *entry = readdir(directory); + if (!entry) { + if (errno != 0) success = false; + break; + } + if (strcmp(entry->d_name, ".") == 0 + || strcmp(entry->d_name, "..") == 0) continue; + char source_path[PATH_MAX], destination_path[PATH_MAX]; + struct stat item_info; + if (!join_path(source_path, absolute_source, entry->d_name)) { + success = false; break; + } + if (lstat(source_path, &item_info) != 0 + || !S_ISREG(item_info.st_mode) + || !has_supported_extension(entry->d_name)) { + ++result->ignored; + continue; + } + ++result->admissible_found; + if (has_forbidden_manifest_character(entry->d_name)) { + success = false; break; + } + int listed = manifest_contains(&manifest, entry->d_name); + if (listed < 0) { + success = false; break; + } + if (listed > 0) { + ++result->processed; + ++result->already_present; + continue; + } + if (added >= batch_size + || !join_path(destination_path, originals_path, entry->d_name)) { + if (added < batch_size) success = false; + continue; + } + off_t size = 0; + int copied = copy_image(source_path, destination_path, &size, control); + if (copied == COPY_IMAGE_CANCELLED) { + cancelled = true; break; + } + if (copied == COPY_IMAGE_ERROR) { + success = false; break; + } + if (copied == COPY_IMAGE_COLLISION) { + struct stat destination_info; + if (lstat(destination_path, &destination_info) != 0 + || !S_ISREG(destination_info.st_mode)) { + success = false; break; + } + size = destination_info.st_size; + ++result->already_present; + } else { + if (!candidate_list_append(state, &created, entry->d_name)) { + (void)unlink(destination_path); + success = false; break; + } + created.items[created.count - 1].created = true; + ++result->copied; + } + if (!manifest_append(&manifest, entry->d_name, size, source_path)) { + success = false; break; + } + ++added; + ++result->newly_manifested; + ++result->processed; + } + if (closedir(directory) != 0) success = false; + if (!success || cancelled) { + manifest_abort(&manifest); + (void)rollback_created_files(&created, originals_path); + free(created.items); + set_status(state, cancelled ? "Import annulé à une frontière sûre." + : "Erreur pendant un lot d'import."); + return cancelled ? LARDON3D_IMPORT_CANCELLED : LARDON3D_IMPORT_FAILED; + } + if (!manifest_commit(state, &manifest)) { + (void)rollback_created_files(&created, originals_path); + free(created.items); + return LARDON3D_IMPORT_FAILED; + } + free(created.items); + *complete = result->processed == result->admissible_found; + set_status(state, *complete ? "Import terminé." : "Lot d'import publié."); + publish_progress(control, result, result->processed, state->status_message); + return LARDON3D_IMPORT_SUCCEEDED; +} + Lardon3DImportOutcome lardon3d_import_directory_controlled( Lardon3DAppState *state, diff --git a/src/import_task.c b/src/import_task.c index 2979c2f..f4a297a 100644 --- a/src/import_task.c +++ b/src/import_task.c @@ -1,229 +1,316 @@ -#include +#include #include +#include #include #include #include +#include +#include +#include #include #include +#include +#include -struct Lardon3DImportTask { - pthread_mutex_t mutex; - pthread_t thread; - bool thread_started; - bool joined; - bool cancel_requested; - Lardon3DAppState worker_state; - char source_directory[PATH_MAX]; - Lardon3DImportTaskSnapshot snapshot; +enum { + IMAGE_IMPORT_MINIMUM_BATCH = 1, + IMAGE_IMPORT_MAXIMUM_BATCH = 32, + IMAGE_IMPORT_FIXED_MEMORY = 128 * 1024, + IMAGE_IMPORT_MEMORY_PER_ITEM = NAME_MAX + 64, }; -static void -copy_message(char destination[256], const char *message) +typedef struct { + char project_path[PATH_MAX]; + char source_path[PATH_MAX]; + Lardon3DProjectDb *project_db; + Lardon3DResourceGovernor *governor; +} Lardon3DImageImportContext; + +struct Lardon3DImportTask { + Lardon3DAppState *state; + uint64_t task_id; +}; + +static bool +canonical_source(const char *source, char output[PATH_MAX]) { - (void)snprintf(destination, 256, "%s", message ? message : ""); + if (!source || !source[0] || strnlen(source, PATH_MAX) >= PATH_MAX) return false; + char absolute[PATH_MAX]; + int written; + if (source[0] == '/') { + written = snprintf(absolute, sizeof(absolute), "%s", source); + } else { + char current[PATH_MAX]; + if (!getcwd(current, sizeof(current))) return false; + written = snprintf(absolute, sizeof(absolute), "%s/%s", current, source); + } + if (written <= 0 || (size_t)written >= sizeof(absolute)) return false; + struct stat information; + if (lstat(absolute, &information) != 0 || S_ISLNK(information.st_mode) + || !S_ISDIR(information.st_mode)) return false; + written = snprintf(output, PATH_MAX, "%s", absolute); + return written > 0 && (size_t)written < PATH_MAX; +} + +static void +destroy_context(void *userdata) +{ + free(userdata); +} + +static void +runtime_project(const Lardon3DImageImportContext *context, Lardon3DAppState *state) +{ + lardon3d_app_state_init(state); + state->project_loaded = true; + state->project_db = context->project_db; + state->resource_governor = context->governor; + (void)snprintf(state->project_path, sizeof(state->project_path), "%s", + context->project_path); } static bool -task_is_cancelled(void *context) +cooperative_continue(void *userdata) { - Lardon3DImportTask *task = context; - bool cancelled; - (void)pthread_mutex_lock(&task->mutex); - cancelled = task->cancel_requested; - (void)pthread_mutex_unlock(&task->mutex); - return cancelled; + return !lardon3d_task_checkpoint(userdata); } static void -task_progressed(void *context, const Lardon3DImportProgress *progress) +import_finished(const Lardon3DTask *task, void *userdata) { - Lardon3DImportTask *task = context; - (void)pthread_mutex_lock(&task->mutex); - task->snapshot.total = progress->total; - task->snapshot.processed = progress->processed; - task->snapshot.copied = progress->copied; - task->snapshot.already_present = progress->already_present; - task->snapshot.ignored = progress->ignored; - copy_message(task->snapshot.message, progress->message); - (void)pthread_mutex_unlock(&task->mutex); +#ifdef LARDON3D_IMPORT_TASK_TESTING + const char *skip = getenv("LARDON3D_TEST_IMPORT_SKIP_FINISHED_CHECKPOINT"); + if (skip && strcmp(skip, "1") == 0) return; +#endif + Lardon3DImageImportContext *context = userdata; + Lardon3DAppState state; + runtime_project(context, &state); + (void)lardon3d_project_checkpoint_task(&state, task); } -static void * -run_import(void *argument) +static uint64_t +elapsed_ns(struct timespec begin, struct timespec end) { - Lardon3DImportTask *task = argument; - Lardon3DImportResult result; - const Lardon3DImportControl control = { - .context = task, - .is_cancelled = task_is_cancelled, - .progressed = task_progressed, + uint64_t seconds = end.tv_sec >= begin.tv_sec + ? (uint64_t)(end.tv_sec - begin.tv_sec) : 0; + long nanoseconds = end.tv_nsec - begin.tv_nsec; + if (nanoseconds < 0 && seconds > 0) { + --seconds; + nanoseconds += 1000000000L; + } + return seconds <= UINT64_MAX / UINT64_C(1000000000) + ? seconds * UINT64_C(1000000000) + (uint64_t)nanoseconds : UINT64_MAX; +} + +static bool +run_image_import(Lardon3DTask *task, void *userdata) +{ + Lardon3DImageImportContext *context = userdata; + Lardon3DAppState state; + runtime_project(context, &state); + for (;;) { + Lardon3DTaskExecutionContract contract; + if (!lardon3d_task_execution_contract(task, &contract) + || contract.batch_size < IMAGE_IMPORT_MINIMUM_BATCH + || contract.batch_size > IMAGE_IMPORT_MAXIMUM_BATCH) { + return lardon3d_task_fail(task, "Contrat de lot import invalide."); + } + Lardon3DImportControl control = { + .context = task, + .is_cancelled = cooperative_continue, + }; + Lardon3DImportResult result; + bool complete = false; + struct timespec begin, end; + (void)clock_gettime(CLOCK_MONOTONIC, &begin); + Lardon3DImportOutcome outcome = lardon3d_import_directory_batch( + &state, context->source_path, contract.batch_size, &result, + &control, &complete); + (void)clock_gettime(CLOCK_MONOTONIC, &end); + if (outcome == LARDON3D_IMPORT_CANCELLED) return false; + if (outcome != LARDON3D_IMPORT_SUCCEEDED) { + return lardon3d_task_fail(task, state.status_message); + } + unsigned int progress = complete ? 100U : result.admissible_found == 0 + ? 99U : (unsigned int)((result.processed * 99U) + / result.admissible_found); + if (!lardon3d_task_set_progress(task, progress, state.status_message)) { + return false; + } + (void)lardon3d_resource_governor_record_batch( + context->governor, LARDON3D_RESOURCE_TASK_IMPORT, + result.newly_manifested, + elapsed_ns(begin, end), 0); + if (lardon3d_project_checkpoint_image_import_task( + &state, task, context->source_path) + != LARDON3D_PROJECT_TASK_CHECKPOINT_OK) { + return lardon3d_task_fail(task, "Checkpoint import impossible."); + } + if (complete) return true; +#ifdef LARDON3D_IMPORT_TASK_TESTING + const char *pause = getenv("LARDON3D_TEST_IMPORT_PAUSE_AFTER_BATCH"); + if (pause && strcmp(pause, "1") == 0) { + (void)lardon3d_task_pause(task); + } +#endif + Lardon3DResourceReservation *reservation = NULL; + if (!lardon3d_task_sequence_break(task, context->governor, + &reservation, &contract)) return false; + } +} + +static Lardon3DImageImportContext * +create_context(const char *project_path, const char *source_path, + Lardon3DProjectDb *database, Lardon3DResourceGovernor *governor) +{ + if (!project_path || !source_path || !database || !governor) return NULL; + Lardon3DImageImportContext *context = calloc(1, sizeof(*context)); + if (!context) return NULL; + int project_written = snprintf(context->project_path, + sizeof(context->project_path), "%s", project_path); + int source_written = snprintf(context->source_path, + sizeof(context->source_path), "%s", source_path); + if (project_written <= 0 || (size_t)project_written >= sizeof(context->project_path) + || source_written <= 0 || (size_t)source_written >= sizeof(context->source_path)) { + free(context); return NULL; + } + context->project_db = database; + context->governor = governor; + return context; +} + +bool +lardon3d_image_import_reconstruct( + const Lardon3DTaskDurableSnapshot *snapshot, + void *userdata, + Lardon3DTaskKindBinding *binding +) +{ + Lardon3DImageImportReconstructionContext *runtime = userdata; + if (!snapshot || !runtime || !runtime->project_path || !runtime->project_db + || !runtime->resource_governor || !binding) return false; + Lardon3DProjectDbImageImport parameters; + if (lardon3d_project_db_load_image_import(runtime->project_db, snapshot->id, + ¶meters) != LARDON3D_PROJECT_DB_OK) return false; + char source[PATH_MAX]; + if (!canonical_source(parameters.source_path, source)) return false; + Lardon3DImageImportContext *context = create_context(runtime->project_path, + source, runtime->project_db, runtime->resource_governor); + if (!context) return false; + *binding = (Lardon3DTaskKindBinding) { + .callback = run_image_import, + .userdata = context, + .userdata_destroy = destroy_context, + .finished_callback = import_finished, + .finished_userdata = context, }; - Lardon3DImportOutcome outcome = lardon3d_import_directory_controlled( - &task->worker_state, - task->source_directory, - &result, - &control - ); - - (void)pthread_mutex_lock(&task->mutex); - task->snapshot.status = outcome == LARDON3D_IMPORT_SUCCEEDED - ? LARDON3D_IMPORT_TASK_SUCCEEDED - : outcome == LARDON3D_IMPORT_CANCELLED - ? LARDON3D_IMPORT_TASK_CANCELLED - : LARDON3D_IMPORT_TASK_FAILED; - task->snapshot.total = result.admissible_found; - task->snapshot.copied = result.copied; - task->snapshot.already_present = result.already_present; - copy_message(task->snapshot.message, task->worker_state.status_message); - (void)pthread_mutex_unlock(&task->mutex); - return NULL; + return true; } -Lardon3DImportTask * -lardon3d_import_task_create(void) +Lardon3DTask * +lardon3d_project_create_image_import_task( + Lardon3DAppState *state, + const char *source_directory, + uint64_t *task_id +) { - Lardon3DImportTask *task = calloc(1, sizeof(*task)); - if (!task) { + if (task_id) *task_id = 0; + if (!state || !state->project_loaded || !state->project_db + || !state->resource_governor || !task_id) return NULL; + char source[PATH_MAX]; + if (!canonical_source(source_directory, source)) return NULL; + uint64_t id = 0; + if (lardon3d_project_db_allocate_task_id(state->project_db, &id) + != LARDON3D_PROJECT_DB_OK) return NULL; + Lardon3DImageImportContext *context = create_context(state->project_path, + source, state->project_db, state->resource_governor); + if (!context) return NULL; + const Lardon3DResourceEstimate estimate = { + .memory_fixed_bytes = IMAGE_IMPORT_FIXED_MEMORY, + .memory_bytes_per_item = IMAGE_IMPORT_MEMORY_PER_ITEM, + .minimum_batch_size = IMAGE_IMPORT_MINIMUM_BATCH, + .maximum_batch_size = IMAGE_IMPORT_MAXIMUM_BATCH, + .desired_cpu_threads = 1, + .desired_io_slots = 1, + .task_class = LARDON3D_RESOURCE_TASK_IMPORT, + }; + Lardon3DTask *task = lardon3d_task_create_typed("Import d'images", + &estimate, LARDON3D_IMAGE_IMPORT_TASK_KIND, + LARDON3D_IMAGE_IMPORT_TASK_KIND_VERSION, run_image_import, context, + destroy_context); + if (!task || !lardon3d_task_assign_id(task, id) + || !lardon3d_task_set_finished_callback(task, import_finished, context) + || lardon3d_project_checkpoint_image_import_task(state, task, source) + != LARDON3D_PROJECT_TASK_CHECKPOINT_OK) { + lardon3d_task_destroy(task); return NULL; } - if (pthread_mutex_init(&task->mutex, NULL) != 0) { - free(task); - return NULL; - } - task->snapshot.status = LARDON3D_IMPORT_TASK_IDLE; + *task_id = id; return task; } bool -lardon3d_import_task_start( - Lardon3DImportTask *task, - const Lardon3DAppState *state, - const char *source_directory -) +lardon3d_project_enqueue_image_import(Lardon3DAppState *state, + const char *source_directory, uint64_t *task_id) { - if (!task || !state || !source_directory) { + if (!state || !state->task_queue) return false; + Lardon3DTask *task = lardon3d_project_create_image_import_task( + state, source_directory, task_id); + if (!task) return false; + if (!lardon3d_task_queue_add(state->task_queue, task, NULL)) { + lardon3d_task_destroy(task); return false; } + return true; +} - (void)pthread_mutex_lock(&task->mutex); - if (task->thread_started) { - (void)pthread_mutex_unlock(&task->mutex); - return false; - } - int written = snprintf( - task->source_directory, - sizeof(task->source_directory), - "%s", - source_directory - ); - if (written < 0 || (size_t)written >= sizeof(task->source_directory)) { - copy_message(task->snapshot.message, "Erreur : chemin source trop long."); - task->snapshot.status = LARDON3D_IMPORT_TASK_FAILED; - (void)pthread_mutex_unlock(&task->mutex); - return false; - } - task->worker_state = *state; - task->cancel_requested = false; - task->joined = false; - task->snapshot = (Lardon3DImportTaskSnapshot) { - .status = LARDON3D_IMPORT_TASK_RUNNING, - .message = "Analyse du dossier source...", +Lardon3DImportTask *lardon3d_import_task_create(void) { return calloc(1, sizeof(Lardon3DImportTask)); } + +bool +lardon3d_import_task_start(Lardon3DImportTask *task, Lardon3DAppState *state, + const char *source_directory) +{ + if (!task || task->task_id != 0 || !state) return false; + task->state = state; + return lardon3d_project_enqueue_image_import(state, source_directory, + &task->task_id); +} + +void lardon3d_import_task_request_cancel(Lardon3DImportTask *task) +{ if (task && task->state) (void)lardon3d_task_queue_cancel(task->state->task_queue, task->task_id); } + +bool +lardon3d_import_task_snapshot(Lardon3DImportTask *task, + Lardon3DImportTaskSnapshot *snapshot) +{ + if (!task || !task->state || !snapshot) return false; + Lardon3DTaskSnapshot generic; + if (!lardon3d_task_queue_get(task->state->task_queue, task->task_id, + &generic)) return false; + *snapshot = (Lardon3DImportTaskSnapshot) { + .status = generic.state == TASK_COMPLETED ? LARDON3D_IMPORT_TASK_SUCCEEDED + : generic.state == TASK_CANCELLED ? LARDON3D_IMPORT_TASK_CANCELLED + : generic.state == TASK_FAILED ? LARDON3D_IMPORT_TASK_FAILED + : LARDON3D_IMPORT_TASK_RUNNING, + .total = 100, + .processed = generic.progress, }; - task->thread_started = true; - int error = pthread_create(&task->thread, NULL, run_import, task); - if (error != 0) { - task->thread_started = false; - task->snapshot.status = LARDON3D_IMPORT_TASK_FAILED; - copy_message( - task->snapshot.message, - "Erreur : impossible de lancer la tâche d'import." - ); - (void)pthread_mutex_unlock(&task->mutex); - return false; - } - (void)pthread_mutex_unlock(&task->mutex); - return true; -} - -void -lardon3d_import_task_request_cancel(Lardon3DImportTask *task) -{ - if (!task) { - return; - } - (void)pthread_mutex_lock(&task->mutex); - if (task->snapshot.status == LARDON3D_IMPORT_TASK_RUNNING) { - task->cancel_requested = true; - copy_message(task->snapshot.message, "Annulation demandée..."); - } - (void)pthread_mutex_unlock(&task->mutex); -} - -bool -lardon3d_import_task_snapshot( - Lardon3DImportTask *task, - Lardon3DImportTaskSnapshot *snapshot -) -{ - if (!task || !snapshot) { - return false; - } - (void)pthread_mutex_lock(&task->mutex); - *snapshot = task->snapshot; - (void)pthread_mutex_unlock(&task->mutex); - return true; -} - -bool -lardon3d_import_task_is_finished(Lardon3DImportTask *task) -{ - Lardon3DImportTaskSnapshot snapshot; - if (!lardon3d_import_task_snapshot(task, &snapshot)) { - return false; - } - return snapshot.status == LARDON3D_IMPORT_TASK_SUCCEEDED - || snapshot.status == LARDON3D_IMPORT_TASK_CANCELLED - || snapshot.status == LARDON3D_IMPORT_TASK_FAILED; -} - -bool -lardon3d_import_task_join(Lardon3DImportTask *task) -{ - if (!task) { - return false; - } - - (void)pthread_mutex_lock(&task->mutex); - bool started = task->thread_started; - bool joined = task->joined; - bool should_join = started && !joined; - pthread_t thread = task->thread; - (void)pthread_mutex_unlock(&task->mutex); - if (!should_join) { - return started && joined; - } - if (pthread_join(thread, NULL) != 0) { - return false; - } - (void)pthread_mutex_lock(&task->mutex); - task->joined = true; - (void)pthread_mutex_unlock(&task->mutex); + (void)snprintf(snapshot->message, sizeof(snapshot->message), "%s", + generic.message); return true; } +bool lardon3d_import_task_is_finished(Lardon3DImportTask *task) +{ Lardon3DImportTaskSnapshot snapshot; return lardon3d_import_task_snapshot(task, &snapshot) + && snapshot.status != LARDON3D_IMPORT_TASK_RUNNING; } +bool lardon3d_import_task_join(Lardon3DImportTask *task) +{ return task && lardon3d_import_task_is_finished(task); } void lardon3d_import_task_destroy(Lardon3DImportTask *task) { - if (!task) { - return; + if (task && lardon3d_import_task_is_finished(task)) { + (void)lardon3d_task_queue_remove(task->state->task_queue, task->task_id); } - if (task->thread_started && !task->joined) { - lardon3d_import_task_request_cancel(task); - if (!lardon3d_import_task_join(task)) { - return; - } - } - (void)pthread_mutex_destroy(&task->mutex); free(task); } diff --git a/src/project.c b/src/project.c index 021f7a0..41ca0f5 100644 --- a/src/project.c +++ b/src/project.c @@ -848,10 +848,11 @@ checkpoint_paths( && join_path(absolute, PATH_MAX, state->project_path, relative); } -Lardon3DProjectTaskCheckpointResult -lardon3d_project_checkpoint_task( +static Lardon3DProjectTaskCheckpointResult +checkpoint_task_internal( Lardon3DAppState *state, - const Lardon3DTask *task + const Lardon3DTask *task, + const char *image_import_source ) { if (!state || !state->project_loaded || !state->project_db) { @@ -894,14 +895,13 @@ lardon3d_project_checkpoint_task( .updated_at = now.tv_sec, }; (void)snprintf(checkpoint.path, sizeof(checkpoint.path), "%s", relative); - Lardon3DProjectDbResult recorded = lardon3d_project_db_record_task( - state->project_db, - &snapshot, - task_kind, - task_kind_version, - &checkpoint, - now.tv_sec - ); + Lardon3DProjectDbResult recorded = image_import_source + ? lardon3d_project_db_record_image_import_task( + state->project_db, &snapshot, task_kind, task_kind_version, + &checkpoint, image_import_source, now.tv_sec) + : lardon3d_project_db_record_task( + state->project_db, &snapshot, task_kind, task_kind_version, + &checkpoint, now.tv_sec); if (recorded == LARDON3D_PROJECT_DB_BUSY) { return LARDON3D_PROJECT_TASK_CHECKPOINT_DB_BUSY; } @@ -913,6 +913,28 @@ lardon3d_project_checkpoint_task( : LARDON3D_PROJECT_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE; } +Lardon3DProjectTaskCheckpointResult +lardon3d_project_checkpoint_task( + Lardon3DAppState *state, + const Lardon3DTask *task +) +{ + return checkpoint_task_internal(state, task, NULL); +} + +Lardon3DProjectTaskCheckpointResult +lardon3d_project_checkpoint_image_import_task( + Lardon3DAppState *state, + const Lardon3DTask *task, + const char *source_path +) +{ + if (!source_path || !source_path[0]) { + return LARDON3D_PROJECT_TASK_CHECKPOINT_INVALID_TASK; + } + return checkpoint_task_internal(state, task, source_path); +} + static bool coherent_recovery( const Lardon3DProjectDbTask *database_task, diff --git a/src/project_db.c b/src/project_db.c index c998ab8..a519798 100644 --- a/src/project_db.c +++ b/src/project_db.c @@ -17,9 +17,10 @@ struct Lardon3DProjectDb { char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY]; }; -static const char schema_v2[] = +static const char schema_v3[] = "CREATE TABLE metadata(key TEXT PRIMARY KEY,value INTEGER NOT NULL);" - "INSERT INTO metadata(key,value) VALUES('schema_version',2);" + "INSERT INTO metadata(key,value) VALUES('schema_version',3);" + "INSERT INTO metadata(key,value) VALUES('next_task_id',1);" "CREATE TABLE project(singleton INTEGER PRIMARY KEY CHECK(singleton=1)," "stable_id TEXT NOT NULL UNIQUE,name TEXT NOT NULL,created_at INTEGER NOT NULL,updated_at INTEGER NOT NULL);" "CREATE TABLE tasks(task_id INTEGER PRIMARY KEY CHECK(task_id>0),name TEXT NOT NULL," @@ -38,7 +39,9 @@ static const char schema_v2[] = "state INTEGER NOT NULL CHECK(state BETWEEN 0 AND 1),size_bytes INTEGER NOT NULL CHECK(size_bytes>=0)," "producer_task_id INTEGER REFERENCES tasks(task_id),created_at INTEGER NOT NULL,updated_at INTEGER NOT NULL);" "CREATE INDEX artifacts_state_idx ON artifacts(state,artifact_id);" - "CREATE INDEX artifacts_producer_idx ON artifacts(producer_task_id);"; + "CREATE INDEX artifacts_producer_idx ON artifacts(producer_task_id);" + "CREATE TABLE image_import_tasks(task_id INTEGER PRIMARY KEY REFERENCES tasks(task_id) ON DELETE CASCADE," + "source_path TEXT NOT NULL);"; static void copy_error(char destination[LARDON3D_PROJECT_DB_ERROR_CAPACITY], const char *text) @@ -127,12 +130,12 @@ migrate(Lardon3DProjectDb *database, unsigned int from_version) if (from_version == LARDON3D_PROJECT_DB_SCHEMA_VERSION) { return LARDON3D_PROJECT_DB_OK; } - if (from_version != 0 && from_version != 1) { + if (from_version != 0 && from_version != 1 && from_version != 2) { return LARDON3D_PROJECT_DB_CORRUPT; } Lardon3DProjectDbResult result = execute(database, "BEGIN IMMEDIATE", "begin migration"); if (result == LARDON3D_PROJECT_DB_OK && from_version == 0) { - result = execute(database, schema_v2, "create schema v2"); + result = execute(database, schema_v3, "create schema v3"); } if (result == LARDON3D_PROJECT_DB_OK && from_version == 1) { result = execute(database, @@ -153,6 +156,25 @@ migrate(Lardon3DProjectDb *database, unsigned int from_version) "finish schema v2 migration"); } } + if (result == LARDON3D_PROJECT_DB_OK && from_version != 0) { + result = execute(database, + "CREATE TABLE image_import_tasks(task_id INTEGER PRIMARY KEY REFERENCES tasks(task_id) ON DELETE CASCADE,source_path TEXT NOT NULL);" + "INSERT INTO metadata(key,value) VALUES('next_task_id',(SELECT CASE WHEN COALESCE(MAX(task_id),0)>=9223372036854775807 THEN 0 ELSE COALESCE(MAX(task_id),0)+1 END FROM tasks))", + "migrate schema v2 to v3"); +#ifdef LARDON3D_PROJECT_DB_TESTING + const char *forced_failure = getenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V3"); + if (result == LARDON3D_PROJECT_DB_OK && forced_failure + && strcmp(forced_failure, "1") == 0) { + result = execute(database, "INSERT INTO missing_test_table VALUES(1)", + "forced migration failure"); + } +#endif + if (result == LARDON3D_PROJECT_DB_OK) { + result = execute(database, + "UPDATE metadata SET value=3 WHERE key='schema_version' AND value=2", + "finish schema v3 migration"); + } + } if (result == LARDON3D_PROJECT_DB_OK) result = execute(database, "COMMIT", "commit migration"); if (result != LARDON3D_PROJECT_DB_OK) (void)execute(database, "ROLLBACK", "rollback migration"); return result; @@ -215,8 +237,8 @@ lardon3d_project_db_open(const char *path, Lardon3DProjectDb **output, char erro if (result == LARDON3D_PROJECT_DB_OK) result = read_schema_version(database, &version); if (result == LARDON3D_PROJECT_DB_OK) result = migrate(database, version); if (result == LARDON3D_PROJECT_DB_OK) { - const char *required[] = {"project", "tasks", "checkpoints", "artifacts"}; - for (size_t index = 0; index < 4 && result == LARDON3D_PROJECT_DB_OK; ++index) { + const char *required[] = {"project", "tasks", "checkpoints", "artifacts", "image_import_tasks"}; + for (size_t index = 0; index < 5 && result == LARDON3D_PROJECT_DB_OK; ++index) { if (!table_exists(database->connection, required[index])) { copy_error(database->error, "Schéma v1 incomplet."); result = LARDON3D_PROJECT_DB_CORRUPT; @@ -348,16 +370,19 @@ valid_durable_task(const Lardon3DTaskDurableSnapshot *snapshot, int64_t updated_ && updated_at >= 0; } -Lardon3DProjectDbResult -lardon3d_project_db_record_task(Lardon3DProjectDb *database, const Lardon3DTaskDurableSnapshot *snapshot, +static Lardon3DProjectDbResult +record_task_internal(Lardon3DProjectDb *database, const Lardon3DTaskDurableSnapshot *snapshot, const char *task_kind, uint32_t task_kind_version, - const Lardon3DProjectDbCheckpoint *checkpoint, int64_t updated_at) + const Lardon3DProjectDbCheckpoint *checkpoint, const char *source_path, + int64_t updated_at) { bool typed = task_kind != NULL; if (!database || !valid_durable_task(snapshot, updated_at) || (typed && (!lardon3d_task_kind_is_valid(task_kind) || task_kind_version == 0)) || (!typed && task_kind_version != 0) + || (source_path && !bounded_text(source_path, + LARDON3D_PROJECT_DB_PATH_CAPACITY, false)) || (checkpoint && !valid_checkpoint(checkpoint))) return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; #ifdef LARDON3D_PROJECT_DB_TESTING const char *forced_busy = getenv("LARDON3D_TEST_PROJECT_DB_BUSY_CHECKPOINT"); @@ -402,6 +427,35 @@ lardon3d_project_db_record_task(Lardon3DProjectDb *database, const Lardon3DTaskD } #endif } + if (result == LARDON3D_PROJECT_DB_OK) { + result = prepare(database, + "UPDATE metadata SET value=CASE WHEN ?1=9223372036854775807 THEN 0 ELSE ?1+1 END " + "WHERE key='next_task_id' AND value>0 AND value<=?1", + &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + (void)sqlite3_bind_int64(statement, 1, + (sqlite3_int64)snapshot->id); + result = step_done(database, statement, "advance recorded task id"); + } + } + if (result == LARDON3D_PROJECT_DB_OK && source_path) { + result = prepare(database, + "INSERT INTO image_import_tasks(task_id,source_path) VALUES(?1,?2) " + "ON CONFLICT(task_id) DO UPDATE SET source_path=excluded.source_path " + "WHERE image_import_tasks.source_path=excluded.source_path", + &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + (void)sqlite3_bind_int64(statement, 1, (sqlite3_int64)snapshot->id); + (void)sqlite3_bind_text(statement, 2, source_path, -1, + SQLITE_TRANSIENT); + result = step_done(database, statement, "upsert image import"); + if (result == LARDON3D_PROJECT_DB_OK + && sqlite3_changes(database->connection) != 1) { + copy_error(database->error, "Source d'import immuable."); + result = LARDON3D_PROJECT_DB_CONSTRAINT; + } + } + } if (result == LARDON3D_PROJECT_DB_OK && checkpoint) { result = prepare(database, "INSERT INTO checkpoints(task_id,path,format_version,durability,updated_at) VALUES(?1,?2,?3,?4,?5) " "ON CONFLICT(task_id) DO UPDATE SET path=excluded.path,format_version=excluded.format_version,durability=excluded.durability,updated_at=excluded.updated_at", &statement); @@ -417,6 +471,36 @@ lardon3d_project_db_record_task(Lardon3DProjectDb *database, const Lardon3DTaskD return result; } +Lardon3DProjectDbResult +lardon3d_project_db_record_task( + Lardon3DProjectDb *database, + const Lardon3DTaskDurableSnapshot *snapshot, + const char *task_kind, + uint32_t task_kind_version, + const Lardon3DProjectDbCheckpoint *checkpoint, + int64_t updated_at +) +{ + return record_task_internal(database, snapshot, task_kind, + task_kind_version, checkpoint, NULL, updated_at); +} + +Lardon3DProjectDbResult +lardon3d_project_db_record_image_import_task( + Lardon3DProjectDb *database, + const Lardon3DTaskDurableSnapshot *snapshot, + const char *task_kind, + uint32_t task_kind_version, + const Lardon3DProjectDbCheckpoint *checkpoint, + const char *source_path, + int64_t updated_at +) +{ + if (!source_path) return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + return record_task_internal(database, snapshot, task_kind, + task_kind_version, checkpoint, source_path, updated_at); +} + static bool read_task(sqlite3_stmt *statement, Lardon3DProjectDbTask *task) { @@ -493,6 +577,91 @@ lardon3d_project_db_list_recoverable(Lardon3DProjectDb *database, uint64_t after (void)pthread_mutex_unlock(&database->mutex); return result; } +Lardon3DProjectDbResult +lardon3d_project_db_allocate_task_id( + Lardon3DProjectDb *database, + uint64_t *task_id +) +{ + if (task_id) *task_id = 0; + if (!database || !task_id) return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + (void)pthread_mutex_lock(&database->mutex); + Lardon3DProjectDbResult result = execute(database, "BEGIN IMMEDIATE", + "begin task id allocation"); + sqlite3_stmt *statement = NULL; + sqlite3_int64 next = 0; + if (result == LARDON3D_PROJECT_DB_OK) { + result = prepare(database, + "SELECT value FROM metadata WHERE key='next_task_id'", &statement); + } + if (result == LARDON3D_PROJECT_DB_OK) { + int code = sqlite3_step(statement); + if (code != SQLITE_ROW || sqlite3_column_type(statement, 0) != SQLITE_INTEGER) { + result = LARDON3D_PROJECT_DB_CORRUPT; + } else { + next = sqlite3_column_int64(statement, 0); + if (next <= 0 || next > INT64_MAX) result = LARDON3D_PROJECT_DB_CONSTRAINT; + } + (void)sqlite3_finalize(statement); statement = NULL; + } + if (result == LARDON3D_PROJECT_DB_OK) { + result = prepare(database, + "UPDATE metadata SET value=CASE WHEN value=9223372036854775807 THEN 0 ELSE value+1 END WHERE key='next_task_id' AND value=?1", + &statement); + } + if (result == LARDON3D_PROJECT_DB_OK) { + (void)sqlite3_bind_int64(statement, 1, next); + result = step_done(database, statement, "advance task id"); + statement = NULL; + if (result == LARDON3D_PROJECT_DB_OK + && sqlite3_changes(database->connection) != 1) { + result = LARDON3D_PROJECT_DB_CONSTRAINT; + } + } + if (result == LARDON3D_PROJECT_DB_OK) { + result = execute(database, "COMMIT", "commit task id allocation"); + } + if (result != LARDON3D_PROJECT_DB_OK) { + (void)execute(database, "ROLLBACK", "rollback task id allocation"); + } else { + *task_id = (uint64_t)next; + } + (void)pthread_mutex_unlock(&database->mutex); + return result; +} + +Lardon3DProjectDbResult +lardon3d_project_db_load_image_import( + Lardon3DProjectDb *database, + uint64_t task_id, + Lardon3DProjectDbImageImport *parameters +) +{ + if (!database || !valid_task_id(task_id) || !parameters) { + return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; + } + memset(parameters, 0, sizeof(*parameters)); + (void)pthread_mutex_lock(&database->mutex); + sqlite3_stmt *statement = NULL; + Lardon3DProjectDbResult result = prepare(database, + "SELECT source_path FROM image_import_tasks WHERE task_id=?1", + &statement); + if (result == LARDON3D_PROJECT_DB_OK) { + (void)sqlite3_bind_int64(statement, 1, (sqlite3_int64)task_id); + int code = sqlite3_step(statement); + if (code == SQLITE_DONE) result = LARDON3D_PROJECT_DB_NOT_FOUND; + else if (code != SQLITE_ROW || !copy_column(statement, 0, + parameters->source_path, sizeof(parameters->source_path))) { + result = LARDON3D_PROJECT_DB_CORRUPT; + } else { + parameters->task_id = task_id; + } + (void)sqlite3_finalize(statement); + } + (void)pthread_mutex_unlock(&database->mutex); + return result; +} + Lardon3DProjectDbResult lardon3d_project_db_create_artifact(Lardon3DProjectDb *database, const Lardon3DProjectDbArtifact *artifact) { diff --git a/src/resource_governor.c b/src/resource_governor.c index f65f6af..15fc4a9 100644 --- a/src/resource_governor.c +++ b/src/resource_governor.c @@ -180,6 +180,8 @@ record_batch_locked( metrics[head] = (Lardon3DBatchMetrics) { .batch_size = batch_size, .duration_ns = duration_ns, + /* Zéro est le marqueur persistant « mesure inconnue ». La boucle + * d'adaptation ignore explicitement ces échantillons. */ .peak_memory_bytes = peak_memory_bytes, }; head = (head + 1) % LARDON3D_BATCH_METRICS_CAPACITY; diff --git a/src/task.c b/src/task.c index 5e5dec4..ff73884 100644 --- a/src/task.c +++ b/src/task.c @@ -19,6 +19,10 @@ struct Lardon3DTask { Lardon3DTaskCallback callback; void *userdata; Lardon3DTaskUserdataDestroy userdata_destroy; + Lardon3DTaskFinishedCallback finished_callback; + void *finished_userdata; + bool finished_notified; + bool finished_callback_running; char task_kind[LARDON3D_TASK_KIND_CAPACITY]; uint32_t task_kind_version; Lardon3DResourceEstimate estimate; @@ -98,6 +102,28 @@ finish_locked( (void)pthread_cond_broadcast(&task->condition); } +static void +notify_finished(Lardon3DTask *task) +{ + Lardon3DTaskFinishedCallback callback = NULL; + void *userdata = NULL; + (void)pthread_mutex_lock(&task->mutex); + if (is_terminal(task->state) && !task->finished_notified) { + task->finished_notified = true; + callback = task->finished_callback; + userdata = task->finished_userdata; + task->finished_callback_running = callback != NULL; + } + (void)pthread_mutex_unlock(&task->mutex); + if (callback) { + callback(task, userdata); + (void)pthread_mutex_lock(&task->mutex); + task->finished_callback_running = false; + (void)pthread_cond_broadcast(&task->condition); + (void)pthread_mutex_unlock(&task->mutex); + } +} + Lardon3DTask * lardon3d_task_create( const char *name, @@ -218,6 +244,7 @@ lardon3d_task_start( reservation_copy ); } + notify_finished(task); return true; } while (task->pause_requested && !task->cancel_requested) { @@ -237,6 +264,7 @@ lardon3d_task_start( reservation_copy ); } + notify_finished(task); return true; } task->state = TASK_RUNNING; @@ -257,33 +285,57 @@ lardon3d_task_start( } else { finish_locked(task, TASK_COMPLETED, "Tâche terminée."); } + Lardon3DResourceReservation *reservation_copy = task->current_reservation; + task->current_reservation = NULL; (void)pthread_mutex_unlock(&task->mutex); - if (task->current_reservation) { + if (reservation_copy) { (void)lardon3d_resource_governor_release( governor, - task->current_reservation + reservation_copy ); - task->current_reservation = NULL; } + notify_finished(task); return true; } +bool +lardon3d_task_set_finished_callback( + Lardon3DTask *task, + Lardon3DTaskFinishedCallback callback, + void *userdata +) +{ + if (!task || !callback) return false; + (void)pthread_mutex_lock(&task->mutex); + bool accepted = !task->executing && task->state == TASK_PENDING + && !task->finished_callback; + if (accepted) { + task->finished_callback = callback; + task->finished_userdata = userdata; + } + (void)pthread_mutex_unlock(&task->mutex); + return accepted; +} + void lardon3d_task_request_cancel(Lardon3DTask *task) { if (!task) { return; } + bool finished_here = false; (void)pthread_mutex_lock(&task->mutex); if (!is_terminal(task->state)) { task->cancel_requested = true; copy_text(task->message, sizeof(task->message), "Annulation demandée."); if (!task->executing) { finish_locked(task, TASK_CANCELLED, "Tâche annulée."); + finished_here = true; } (void)pthread_cond_broadcast(&task->condition); } (void)pthread_mutex_unlock(&task->mutex); + if (finished_here) notify_finished(task); } bool @@ -332,7 +384,9 @@ lardon3d_task_join(Lardon3DTask *task) return false; } (void)pthread_mutex_lock(&task->mutex); - while (task->executing) { + while (task->executing + || (is_terminal(task->state) && task->finished_callback + && (!task->finished_notified || task->finished_callback_running))) { (void)pthread_cond_wait(&task->condition, &task->mutex); } bool terminal = is_terminal(task->state); @@ -817,6 +871,7 @@ lardon3d_task_reject(Lardon3DTask *task, const char *message) ); } (void)pthread_mutex_unlock(&task->mutex); + if (accepted) notify_finished(task); return accepted; } diff --git a/src/task_kind_registry.c b/src/task_kind_registry.c index 3c3cae6..559ccec 100644 --- a/src/task_kind_registry.c +++ b/src/task_kind_registry.c @@ -112,5 +112,12 @@ lardon3d_task_kind_registry_restore( } return LARDON3D_TASK_KIND_RESTORE_FAILED; } + if (binding.finished_callback + && !lardon3d_task_set_finished_callback(*task, + binding.finished_callback, binding.finished_userdata)) { + lardon3d_task_destroy(*task); + *task = NULL; + return LARDON3D_TASK_KIND_RESTORE_FAILED; + } return LARDON3D_TASK_KIND_OK; } diff --git a/src/task_kinds.c b/src/task_kinds.c new file mode 100644 index 0000000..63e8d5a --- /dev/null +++ b/src/task_kinds.c @@ -0,0 +1,17 @@ +#include +#include + +const Lardon3DTaskKindRegistry * +lardon3d_task_kind_registry_production(void) +{ + static const Lardon3DTaskKindDescriptor descriptors[] = {{ + .kind = LARDON3D_IMAGE_IMPORT_TASK_KIND, + .kind_version = LARDON3D_IMAGE_IMPORT_TASK_KIND_VERSION, + .reconstruct = lardon3d_image_import_reconstruct, + }}; + static const Lardon3DTaskKindRegistry registry = { + .descriptors = descriptors, + .count = sizeof(descriptors) / sizeof(descriptors[0]), + }; + return ®istry; +} diff --git a/src/task_queue.c b/src/task_queue.c index 735510b..96b41a4 100644 --- a/src/task_queue.c +++ b/src/task_queue.c @@ -281,11 +281,25 @@ enqueue_locked( uint64_t *task_id ) { - if (queue->stopping || queue->next_id == 0 - || !lardon3d_task_assign_id(task, queue->next_id)) { + uint64_t id = lardon3d_task_id(task); + if (queue->stopping) { return false; } - uint64_t id = queue->next_id++; + for (TaskNode *existing = queue->all_head; existing; + existing = existing->next_all) { + if (id != 0 && lardon3d_task_id(existing->task) == id) { + return false; + } + } + if (id == 0) { + if (queue->next_id == 0 + || !lardon3d_task_assign_id(task, queue->next_id)) { + return false; + } + id = queue->next_id++; + } else if (id >= queue->next_id) { + queue->next_id = id == UINT64_MAX ? 0 : id + 1; + } node->task = task; if (queue->all_tail) { queue->all_tail->next_all = node; diff --git a/tests/test_import.c b/tests/test_import.c index 37b44b5..d43dc02 100644 --- a/tests/test_import.c +++ b/tests/test_import.c @@ -189,6 +189,9 @@ run_test(void) CHECK(create_file(path, "one")); CHECK(join_path(path, valid_source, "two.PNG")); CHECK(create_file(path, "two-two")); + CHECK(create_directory(originals)); + CHECK(join_path(path, originals, "one.jpg")); + CHECK(create_file(path, "one")); Lardon3DAppState state; lardon3d_app_state_init(&state); @@ -201,10 +204,14 @@ run_test(void) ) > 0); Lardon3DImportResult result; - CHECK(lardon3d_import_directory(&state, valid_source, &result)); + bool batch_complete = false; + CHECK(lardon3d_import_directory_batch(&state, valid_source, 8, &result, + NULL, &batch_complete) == LARDON3D_IMPORT_SUCCEEDED); + CHECK(batch_complete); CHECK(result.admissible_found == 2); - CHECK(result.copied == 2); - CHECK(result.already_present == 0); + CHECK(result.newly_manifested == 2); + CHECK(result.copied == 1); + CHECK(result.already_present == 1); char manifest[PATH_MAX]; CHECK(join_path(manifest, images, "manifest.tsv")); @@ -234,6 +241,18 @@ run_test(void) CHECK(strcmp(manifest_before, manifest_after) == 0); CHECK(!has_manifest_temporary(images)); + char collision_source[PATH_MAX], collision_destination[PATH_MAX]; + CHECK(join_path(collision_source, valid_source, "collision.jpg")); + CHECK(join_path(collision_destination, originals, "collision.jpg")); + CHECK(create_file(collision_source, "new-source")); + CHECK(create_file(collision_destination, "different-existing-output")); + batch_complete = false; + CHECK(lardon3d_import_directory_batch(&state, valid_source, 8, &result, + NULL, &batch_complete) == LARDON3D_IMPORT_FAILED); + CHECK(read_file(manifest, manifest_after, sizeof(manifest_after))); + CHECK(strcmp(manifest_before, manifest_after) == 0); + CHECK(unlink(collision_source) == 0 && unlink(collision_destination) == 0); + CHECK(join_path(path, valid_source, "three.tiff")); CHECK(create_file(path, "three-three-three")); CHECK(lardon3d_import_directory(&state, valid_source, &result)); diff --git a/tests/test_import_task.c b/tests/test_import_task.c index 7b4734a..0a1a22f 100644 --- a/tests/test_import_task.c +++ b/tests/test_import_task.c @@ -1,324 +1,175 @@ #include #include #include +#include #include #include #include #include #include -#include -#include #include -#include +#include #include +#include +#include -#define CHECK(condition) \ - do { \ - if (!(condition)) { \ - (void)fprintf(stderr, "Échec ligne %d : %s\n", __LINE__, #condition); \ - return false; \ - } \ - } while (0) +#define CHECK(condition) do { if (!(condition)) { \ + (void)fprintf(stderr, "Échec ligne %d : %s\n", __LINE__, #condition); return false; \ +} } while (0) static bool -join_path(char destination[PATH_MAX], const char *parent, const char *child) +join_path(char output[PATH_MAX], const char *parent, const char *child) { - int written = snprintf(destination, PATH_MAX, "%s/%s", parent, child); - return written >= 0 && (size_t)written < PATH_MAX; + int written = snprintf(output, PATH_MAX, "%s/%s", parent, child); + return written > 0 && (size_t)written < PATH_MAX; } static bool -write_all(int descriptor, const void *data, size_t size) +write_fixture(const char *path, unsigned int value) { - const char *bytes = data; - size_t total = 0; - while (total < size) { - ssize_t written = write(descriptor, bytes + total, size - total); - if (written < 0 && errno == EINTR) { - continue; - } - if (written <= 0) { - return false; - } - total += (size_t)written; - } - return true; -} - -static bool -create_file(const char *path, size_t size) -{ - int descriptor = open(path, O_WRONLY | O_CREAT | O_EXCL, 0644); - if (descriptor < 0) { - return false; - } - char block[64 * 1024]; - (void)memset(block, 'L', sizeof(block)); - bool success = true; - while (size > 0) { - size_t chunk = size < sizeof(block) ? size : sizeof(block); - if (!write_all(descriptor, block, chunk)) { - success = false; - break; - } - size -= chunk; - } - if (close(descriptor) != 0) { - success = false; - } - return success; -} - -static bool -read_file(const char *path, char *content, size_t capacity) -{ - int descriptor = open(path, O_RDONLY); - if (descriptor < 0 || capacity == 0) { - return false; - } - size_t total = 0; - while (total + 1 < capacity) { - ssize_t count = read(descriptor, content + total, capacity - total - 1); - if (count == 0) { - break; - } - if (count < 0 && errno == EINTR) { - continue; - } - if (count < 0) { - (void)close(descriptor); - return false; - } - total += (size_t)count; - } - content[total] = '\0'; - return close(descriptor) == 0; + int descriptor = open(path, O_WRONLY | O_CREAT | O_EXCL, 0600); + if (descriptor < 0) return false; + unsigned char data[64]; + memset(data, (int)(value & 0xffU), sizeof(data)); + bool ok = write(descriptor, data, sizeof(data)) == (ssize_t)sizeof(data); + return close(descriptor) == 0 && ok; } static bool remove_tree(const char *path) { struct stat info; - if (lstat(path, &info) != 0) { - return errno == ENOENT; - } - if (!S_ISDIR(info.st_mode)) { - return unlink(path) == 0; - } - + if (lstat(path, &info) != 0) return errno == ENOENT; + if (!S_ISDIR(info.st_mode)) return unlink(path) == 0; DIR *directory = opendir(path); - if (!directory) { - return false; - } - bool success = true; - for (struct dirent *entry = readdir(directory); - entry; - entry = readdir(directory)) { - if (strcmp(entry->d_name, ".") == 0 - || strcmp(entry->d_name, "..") == 0) { - continue; - } + if (!directory) return false; + bool ok = true; + for (struct dirent *entry = readdir(directory); entry; entry = readdir(directory)) { + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) continue; char child[PATH_MAX]; - if (!join_path(child, path, entry->d_name) || !remove_tree(child)) { - success = false; - } + if (!join_path(child, path, entry->d_name) || !remove_tree(child)) ok = false; } - if (closedir(directory) != 0 || rmdir(path) != 0) { - success = false; - } - return success; + if (closedir(directory) != 0 || rmdir(path) != 0) ok = false; + return ok; } static bool -has_temporary_file(const char *images_path) +setup_runtime(Lardon3DAppState *state) { - DIR *directory = opendir(images_path); - if (!directory) { - return true; - } - bool found = false; - for (struct dirent *entry = readdir(directory); - entry; - entry = readdir(directory)) { - if (strncmp(entry->d_name, ".manifest.tsv.tmp.", 18) == 0) { - found = true; - } - } - if (closedir(directory) != 0) { - found = true; - } - return found; + state->hardware_profile = (Lardon3DHardwareProfile) { + .logical_cpu_count = 1024, + .page_size_bytes = 4096, + .memory_total_bytes = UINT64_MAX, + .cpu_architecture = "test", + }; + Lardon3DResourcePolicy policy = { + .maximum_cpu_load_ratio = 1.0, + .maximum_io_pressure_avg10 = 100.0, + .io_slot_capacity = 1, + }; + state->resource_governor = lardon3d_resource_governor_create( + &state->hardware_profile, &policy); + state->task_queue = state->resource_governor + ? lardon3d_task_queue_create(state->resource_governor, 4) : NULL; + return state->task_queue != NULL; } static bool -wait_until_finished( - Lardon3DImportTask *task, - Lardon3DImportTaskSnapshot *snapshot -) +wait_for_state(Lardon3DTaskQueue *queue, uint64_t id, + Lardon3DTaskState expected, Lardon3DTaskSnapshot *snapshot) { - const struct timespec pause = {.tv_sec = 0, .tv_nsec = 1000000}; - for (size_t attempt = 0; attempt < 10000; ++attempt) { - if (!lardon3d_import_task_snapshot(task, snapshot) - || snapshot->processed > snapshot->total) { - return false; - } - if (snapshot->status != LARDON3D_IMPORT_TASK_RUNNING) { - return true; - } - (void)nanosleep(&pause, NULL); + for (size_t attempt = 0; attempt < 1000000; ++attempt) { + if (lardon3d_task_queue_get(queue, id, snapshot) + && snapshot->state == expected) return true; + (void)sched_yield(); } return false; } -static bool -run_success_tests( - Lardon3DAppState *state, - const char *source, - const char *manifest -) -{ - Lardon3DImportTask *unused = lardon3d_import_task_create(); - CHECK(unused); - CHECK(!lardon3d_import_task_start(NULL, state, source)); - CHECK(!lardon3d_import_task_start(unused, NULL, source)); - CHECK(!lardon3d_import_task_start(unused, state, NULL)); - lardon3d_import_task_destroy(unused); - - Lardon3DImportTask *task = lardon3d_import_task_create(); - CHECK(task); - CHECK(lardon3d_import_task_start(task, state, source)); - CHECK(!lardon3d_import_task_start(task, state, source)); - Lardon3DImportTaskSnapshot snapshot; - CHECK(wait_until_finished(task, &snapshot)); - CHECK(snapshot.status == LARDON3D_IMPORT_TASK_SUCCEEDED); - CHECK(snapshot.total == 2); - CHECK(snapshot.processed == 2); - CHECK(snapshot.copied == 2); - CHECK(snapshot.already_present == 0); - CHECK(lardon3d_import_task_join(task)); - CHECK(lardon3d_import_task_join(task)); - lardon3d_import_task_destroy(task); - CHECK(access(manifest, F_OK) == 0); - - task = lardon3d_import_task_create(); - CHECK(task && lardon3d_import_task_start(task, state, source)); - CHECK(wait_until_finished(task, &snapshot)); - CHECK(snapshot.status == LARDON3D_IMPORT_TASK_SUCCEEDED); - CHECK(snapshot.copied == 0); - CHECK(snapshot.already_present == 2); - CHECK(lardon3d_import_task_join(task)); - lardon3d_import_task_destroy(task); - return true; -} - -static bool -run_cancellation_test( - Lardon3DAppState *state, - const char *source, - const char *images, - const char *originals, - const char *manifest -) -{ - char manifest_before[8192]; - CHECK(read_file(manifest, manifest_before, sizeof(manifest_before))); - - Lardon3DImportTask *task = lardon3d_import_task_create(); - CHECK(task && lardon3d_import_task_start(task, state, source)); - Lardon3DImportTaskSnapshot snapshot; - const struct timespec pause = {.tv_sec = 0, .tv_nsec = 1000000}; - bool copy_observed = false; - for (size_t attempt = 0; attempt < 10000; ++attempt) { - CHECK(lardon3d_import_task_snapshot(task, &snapshot)); - CHECK(snapshot.processed <= snapshot.total); - if (snapshot.status != LARDON3D_IMPORT_TASK_RUNNING) { - break; - } - if (snapshot.copied > 0) { - copy_observed = true; - break; - } - (void)nanosleep(&pause, NULL); - } - CHECK(copy_observed); - lardon3d_import_task_request_cancel(task); - CHECK(wait_until_finished(task, &snapshot)); - CHECK(snapshot.status == LARDON3D_IMPORT_TASK_CANCELLED); - CHECK(lardon3d_import_task_join(task)); - lardon3d_import_task_destroy(task); - - char manifest_after[8192]; - CHECK(read_file(manifest, manifest_after, sizeof(manifest_after))); - CHECK(strcmp(manifest_before, manifest_after) == 0); - CHECK(!has_temporary_file(images)); - for (size_t index = 0; index < 8; ++index) { - char filename[64]; - CHECK(snprintf(filename, sizeof(filename), "large-%zu.jpg", index) > 0); - char destination[PATH_MAX]; - CHECK(join_path(destination, originals, filename)); - CHECK(access(destination, F_OK) != 0); - } - return true; -} - static bool run_test(void) { - char base[] = "/tmp/lardon3d-import-task-test.XXXXXX"; - CHECK(mkdtemp(base)); - char project[PATH_MAX]; - char images[PATH_MAX]; - char originals[PATH_MAX]; - char source[PATH_MAX]; - char large_source[PATH_MAX]; - char manifest[PATH_MAX]; - CHECK(join_path(project, base, "project")); - CHECK(join_path(images, project, "images")); - CHECK(join_path(originals, images, "originals")); - CHECK(join_path(source, base, "source")); - CHECK(join_path(large_source, base, "large-source")); - CHECK(join_path(manifest, images, "manifest.tsv")); - CHECK(mkdir(project, 0755) == 0); - CHECK(mkdir(images, 0755) == 0); - CHECK(mkdir(source, 0755) == 0); - CHECK(mkdir(large_source, 0755) == 0); - - char path[PATH_MAX]; - CHECK(join_path(path, source, "one.jpg")); - CHECK(create_file(path, 3)); - CHECK(join_path(path, source, "two.png")); - CHECK(create_file(path, 7)); - for (size_t index = 0; index < 8; ++index) { - char filename[64]; - CHECK(snprintf(filename, sizeof(filename), "large-%zu.jpg", index) > 0); - CHECK(join_path(path, large_source, filename)); - CHECK(create_file(path, 16 * 1024 * 1024)); + char root[] = "/tmp/lardon3d-import-generic-XXXXXX"; + CHECK(mkdtemp(root)); + char source[PATH_MAX]; CHECK(join_path(source, root, "source")); + CHECK(mkdir(source, 0700) == 0); + for (unsigned int index = 0; index < 80; ++index) { + char name[32], path[PATH_MAX]; + CHECK(snprintf(name, sizeof(name), "image-%03u.jpg", index) > 0); + CHECK(join_path(path, source, name) && write_fixture(path, index)); } + CHECK(setenv("LARDON3D_PROJECTS_ROOT", root, 1) == 0); + Lardon3DAppState state; lardon3d_app_state_init(&state); + CHECK(setup_runtime(&state)); + CHECK(lardon3d_project_create(&state, "Persistent Import")); - Lardon3DAppState state; - lardon3d_app_state_init(&state); - state.project_loaded = true; - CHECK(snprintf( - state.project_path, - sizeof(state.project_path), - "%s", - project - ) > 0); - CHECK(run_success_tests(&state, source, manifest)); - CHECK(run_cancellation_test( - &state, - large_source, - images, - originals, - manifest - )); - CHECK(remove_tree(base)); + CHECK(setenv("LARDON3D_TEST_IMPORT_PAUSE_AFTER_BATCH", "1", 1) == 0); + CHECK(setenv("LARDON3D_TEST_IMPORT_SKIP_FINISHED_CHECKPOINT", "1", 1) == 0); + uint64_t task_id = 0; + CHECK(lardon3d_project_enqueue_image_import(&state, source, &task_id)); + CHECK(task_id > 0 && task_id <= INT64_MAX); + Lardon3DTaskSnapshot runtime; + CHECK(wait_for_state(state.task_queue, task_id, TASK_PAUSED, &runtime)); + CHECK(runtime.progress > 0 && runtime.progress < 100); + lardon3d_task_queue_destroy(state.task_queue); state.task_queue = NULL; + lardon3d_project_close(&state); + CHECK(unsetenv("LARDON3D_TEST_IMPORT_PAUSE_AFTER_BATCH") == 0); + CHECK(unsetenv("LARDON3D_TEST_IMPORT_SKIP_FINISHED_CHECKPOINT") == 0); + + state.task_queue = lardon3d_task_queue_create(state.resource_governor, 4); + CHECK(state.task_queue && lardon3d_project_open(&state, "Persistent Import")); + Lardon3DProjectRecoveryEntry entry; size_t count = 0; + const Lardon3DTaskKindRegistry *registry = + lardon3d_task_kind_registry_production(); + CHECK(lardon3d_project_list_recoverable(&state, registry, 0, &entry, 1, + &count) == LARDON3D_PROJECT_DB_OK && count == 1); + CHECK(entry.task_id == task_id + && strcmp(entry.task_kind, LARDON3D_IMAGE_IMPORT_TASK_KIND) == 0 + && entry.task_kind_version == LARDON3D_IMAGE_IMPORT_TASK_KIND_VERSION); + Lardon3DProjectDbImageImport persisted_parameters; + CHECK(lardon3d_project_db_load_image_import(state.project_db, task_id, + &persisted_parameters) == LARDON3D_PROJECT_DB_OK); + CHECK(strcmp(persisted_parameters.source_path, source) == 0); + Lardon3DImageImportReconstructionContext reconstruction = { + .project_path = state.project_path, + .project_db = state.project_db, + .resource_governor = state.resource_governor, + }; + Lardon3DTask *restored = NULL; + char unavailable_source[PATH_MAX]; + CHECK(join_path(unavailable_source, root, "source-unavailable")); + CHECK(rename(source, unavailable_source) == 0); + CHECK(lardon3d_task_kind_registry_restore(registry, entry.task_kind, + entry.task_kind_version, &entry.snapshot, &reconstruction, &restored) + == LARDON3D_TASK_KIND_RECONSTRUCTION_FAILED); + CHECK(!restored && rename(unavailable_source, source) == 0); + CHECK(lardon3d_task_kind_registry_restore(registry, entry.task_kind, + entry.task_kind_version, &entry.snapshot, &reconstruction, &restored) + == LARDON3D_TASK_KIND_OK); + CHECK(restored && lardon3d_task_id(restored) == task_id); + CHECK(lardon3d_task_queue_add(state.task_queue, restored, NULL)); + CHECK(wait_for_state(state.task_queue, task_id, TASK_COMPLETED, &runtime)); + CHECK(runtime.progress == 100); + + char error[256]; + Lardon3DImageCatalog *catalog = lardon3d_image_catalog_load( + &state, error, sizeof(error)); + CHECK(catalog && lardon3d_image_catalog_count(catalog) == 80); + lardon3d_image_catalog_destroy(catalog); + lardon3d_task_queue_destroy(state.task_queue); state.task_queue = NULL; + lardon3d_project_close(&state); + + state.task_queue = lardon3d_task_queue_create(state.resource_governor, 4); + CHECK(state.task_queue && lardon3d_project_open(&state, "Persistent Import")); + CHECK(lardon3d_project_list_recoverable(&state, registry, 0, &entry, 1, + &count) == LARDON3D_PROJECT_DB_OK && count == 0); + lardon3d_task_queue_destroy(state.task_queue); state.task_queue = NULL; + lardon3d_project_close(&state); + lardon3d_resource_governor_destroy(state.resource_governor); + CHECK(unsetenv("LARDON3D_PROJECTS_ROOT") == 0); + CHECK(remove_tree(root)); return true; } -int -main(void) -{ - return run_test() ? EXIT_SUCCESS : EXIT_FAILURE; -} +int main(void) { return run_test() ? EXIT_SUCCESS : EXIT_FAILURE; } diff --git a/tests/test_project_db.c b/tests/test_project_db.c index 7184569..ac022ea 100644 --- a/tests/test_project_db.c +++ b/tests/test_project_db.c @@ -35,6 +35,7 @@ task_snapshot(uint64_t id, Lardon3DTaskState saved) } typedef struct { Lardon3DProjectDb *database; bool success; } ThreadContext; +typedef struct { Lardon3DProjectDb *database; uint64_t id; bool success; } IdThreadContext; static void * read_thread(void *userdata) @@ -51,13 +52,22 @@ read_thread(void *userdata) return NULL; } +static void * +allocate_id_thread(void *userdata) +{ + IdThreadContext *context = userdata; + context->success = lardon3d_project_db_allocate_task_id( + context->database, &context->id) == LARDON3D_PROJECT_DB_OK; + return NULL; +} + static bool create_future_database(const char *path) { sqlite3 *connection = NULL; if (sqlite3_open(path, &connection) != SQLITE_OK) return false; bool ok = sqlite3_exec(connection, "CREATE TABLE metadata(key TEXT PRIMARY KEY,value INTEGER NOT NULL);" - "INSERT INTO metadata VALUES('schema_version',3);", NULL, NULL, NULL) == SQLITE_OK; + "INSERT INTO metadata VALUES('schema_version',4);", NULL, NULL, NULL) == SQLITE_OK; return sqlite3_close(connection) == SQLITE_OK && ok; } @@ -85,6 +95,21 @@ create_v1_database(const char *path) return sqlite3_close(connection) == SQLITE_OK && ok; } +static bool +create_v2_database(const char *path) +{ + if (!create_v1_database(path)) return false; + sqlite3 *connection = NULL; + if (sqlite3_open(path, &connection) != SQLITE_OK) return false; + bool ok = sqlite3_exec(connection, + "ALTER TABLE tasks ADD COLUMN task_kind TEXT;" + "ALTER TABLE tasks ADD COLUMN task_kind_version INTEGER CHECK(task_kind_version IS NULL OR task_kind_version>0);" + "UPDATE tasks SET task_kind='test.work',task_kind_version=1 WHERE task_id=9;" + "UPDATE metadata SET value=2 WHERE key='schema_version'", + NULL, NULL, NULL) == SQLITE_OK; + return sqlite3_close(connection) == SQLITE_OK && ok; +} + static bool query_integer(const char *path, const char *sql, sqlite3_int64 expected) { @@ -109,18 +134,22 @@ run_test(void) char directory[] = "/tmp/lardon3d-project-db-XXXXXX"; CHECK(mkdtemp(directory)); char database_path[512], artifact_path[512], future_path[512], corrupt_path[512]; - char legacy_path[512], failed_migration_path[512]; + char legacy_path[512], failed_migration_path[512], v2_path[512]; + char failed_v3_migration_path[512]; CHECK(snprintf(database_path, sizeof(database_path), "%s/project.db", directory) > 0); CHECK(snprintf(artifact_path, sizeof(artifact_path), "%s/artifact.bin", directory) > 0); CHECK(snprintf(future_path, sizeof(future_path), "%s/future.db", directory) > 0); CHECK(snprintf(corrupt_path, sizeof(corrupt_path), "%s/corrupt.db", directory) > 0); CHECK(snprintf(legacy_path, sizeof(legacy_path), "%s/legacy.db", directory) > 0); CHECK(snprintf(failed_migration_path, sizeof(failed_migration_path), "%s/failed-migration.db", directory) > 0); + CHECK(snprintf(v2_path, sizeof(v2_path), "%s/v2.db", directory) > 0); + CHECK(snprintf(failed_v3_migration_path, sizeof(failed_v3_migration_path), + "%s/failed-v3-migration.db", directory) > 0); char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY]; Lardon3DProjectDb *database = NULL; CHECK(lardon3d_project_db_open(database_path, &database, error) == LARDON3D_PROJECT_DB_OK); - CHECK(database && lardon3d_project_db_schema_version(database) == 2); + CHECK(database && lardon3d_project_db_schema_version(database) == 3); Lardon3DProjectDbProject project = {.created_at = 100, .updated_at = 100}; (void)snprintf(project.stable_id, sizeof(project.stable_id), "project-0001"); @@ -161,6 +190,9 @@ run_test(void) &checkpoint, 202) == LARDON3D_PROJECT_DB_INVALID_ARGUMENT); CHECK(lardon3d_project_db_record_task(database, &running, "test.work", 0, &checkpoint, 202) == LARDON3D_PROJECT_DB_INVALID_ARGUMENT); + CHECK(lardon3d_project_db_record_image_import_task(database, &running, + "import.images", 1, &checkpoint, "/tmp/source-a", 202) + == LARDON3D_PROJECT_DB_CONSTRAINT); Lardon3DTaskDurableSnapshot completed = task_snapshot(2, TASK_COMPLETED); CHECK(lardon3d_project_db_record_task(database, &completed, "test.work", 1, NULL, 202) == LARDON3D_PROJECT_DB_OK); @@ -172,6 +204,19 @@ run_test(void) CHECK(lardon3d_project_db_list_recoverable(database, 1, page, 1, &count) == LARDON3D_PROJECT_DB_OK && count == 0); CHECK(lardon3d_project_db_list_recoverable(database, 0, page, LARDON3D_PROJECT_DB_RECOVERY_PAGE_MAX + 1, &count) == LARDON3D_PROJECT_DB_INVALID_ARGUMENT); + Lardon3DTaskDurableSnapshot image_import = task_snapshot(5, TASK_PENDING); + CHECK(lardon3d_project_db_record_image_import_task(database, &image_import, + "import.images", 1, &checkpoint, "/tmp/source-a", 202) + == LARDON3D_PROJECT_DB_OK); + Lardon3DProjectDbImageImport import_parameters; + CHECK(lardon3d_project_db_load_image_import(database, 5, + &import_parameters) == LARDON3D_PROJECT_DB_OK); + CHECK(import_parameters.task_id == 5 + && strcmp(import_parameters.source_path, "/tmp/source-a") == 0); + CHECK(lardon3d_project_db_record_image_import_task(database, &image_import, + "import.images", 1, &checkpoint, "/tmp/source-b", 203) + == LARDON3D_PROJECT_DB_CONSTRAINT); + Lardon3DTaskDurableSnapshot rollback_task = task_snapshot(3, TASK_PENDING); CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_CHECKPOINT", "1", 1) == 0); CHECK(lardon3d_project_db_record_task(database, &rollback_task, "test.work", 1, &checkpoint, 203) == LARDON3D_PROJECT_DB_IO_ERROR); @@ -206,11 +251,21 @@ run_test(void) CHECK(pthread_create(&threads[1], NULL, read_thread, &contexts[1]) == 0); CHECK(pthread_join(threads[0], NULL) == 0 && pthread_join(threads[1], NULL) == 0); CHECK(contexts[0].success && contexts[1].success); + IdThreadContext id_contexts[2] = {{.database = database}, {.database = database}}; + CHECK(pthread_create(&threads[0], NULL, allocate_id_thread, + &id_contexts[0]) == 0); + CHECK(pthread_create(&threads[1], NULL, allocate_id_thread, + &id_contexts[1]) == 0); + CHECK(pthread_join(threads[0], NULL) == 0 + && pthread_join(threads[1], NULL) == 0); + CHECK(id_contexts[0].success && id_contexts[1].success + && id_contexts[0].id != id_contexts[1].id + && id_contexts[0].id > 5 && id_contexts[1].id > 5); char too_long[LARDON3D_PROJECT_DB_PATH_CAPACITY + 1]; memset(too_long, 'x', sizeof(too_long)); too_long[sizeof(too_long) - 1] = '\0'; CHECK(lardon3d_project_db_open(too_long, &database, error) == LARDON3D_PROJECT_DB_INVALID_ARGUMENT); lardon3d_project_db_close(contexts[0].database); database = NULL; - CHECK(query_integer(database_path, "SELECT value FROM metadata WHERE key='schema_version'", 2)); + CHECK(query_integer(database_path, "SELECT value FROM metadata WHERE key='schema_version'", 3)); CHECK(query_integer(database_path, "SELECT count(*) FROM tasks WHERE task_id=1", 1)); CHECK(lardon3d_project_db_open(database_path, &database, error) == LARDON3D_PROJECT_DB_OK); CHECK(lardon3d_project_db_load_task(database, 1, &task) == LARDON3D_PROJECT_DB_OK); @@ -225,7 +280,7 @@ run_test(void) CHECK(create_v1_database(legacy_path)); CHECK(lardon3d_project_db_open(legacy_path, &database, error) == LARDON3D_PROJECT_DB_OK); - CHECK(lardon3d_project_db_schema_version(database) == 2); + CHECK(lardon3d_project_db_schema_version(database) == 3); CHECK(lardon3d_project_db_get_project(database, &loaded_project) == LARDON3D_PROJECT_DB_OK && strcmp(loaded_project.stable_id, "legacy-project") == 0); CHECK(lardon3d_project_db_load_task(database, 9, &task) == LARDON3D_PROJECT_DB_OK); @@ -234,7 +289,7 @@ run_test(void) CHECK(lardon3d_project_db_load_artifact(database, "legacy-artifact", &loaded_artifact) == LARDON3D_PROJECT_DB_OK); lardon3d_project_db_close(database); database = NULL; - CHECK(query_integer(legacy_path, "SELECT value FROM metadata WHERE key='schema_version'", 2)); + CHECK(query_integer(legacy_path, "SELECT value FROM metadata WHERE key='schema_version'", 3)); CHECK(create_v1_database(failed_migration_path)); CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V2", "1", 1) == 0); @@ -249,8 +304,33 @@ run_test(void) && !task.has_task_kind); lardon3d_project_db_close(database); database = NULL; + CHECK(create_v2_database(v2_path)); + CHECK(lardon3d_project_db_open(v2_path, &database, error) + == LARDON3D_PROJECT_DB_OK); + CHECK(lardon3d_project_db_load_task(database, 9, &task) + == LARDON3D_PROJECT_DB_OK && task.has_task_kind + && strcmp(task.task_kind, "test.work") == 0); + CHECK(task.has_checkpoint + && strcmp(task.checkpoint.path, "legacy.chk") == 0); + CHECK(lardon3d_project_db_load_artifact(database, "legacy-artifact", + &loaded_artifact) == LARDON3D_PROJECT_DB_OK); + lardon3d_project_db_close(database); database = NULL; + CHECK(query_integer(v2_path, + "SELECT value FROM metadata WHERE key='schema_version'", 3)); + + CHECK(create_v2_database(failed_v3_migration_path)); + CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V3", "1", 1) == 0); + CHECK(lardon3d_project_db_open(failed_v3_migration_path, &database, error) + == LARDON3D_PROJECT_DB_IO_ERROR); + CHECK(unsetenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V3") == 0); + CHECK(query_integer(failed_v3_migration_path, + "SELECT value FROM metadata WHERE key='schema_version'", 2)); + CHECK(query_integer(failed_v3_migration_path, + "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='image_import_tasks'", 0)); + CHECK(unlink(artifact_path) == 0); CHECK(unlink(database_path) == 0); CHECK(unlink(future_path) == 0); CHECK(unlink(corrupt_path) == 0); CHECK(unlink(legacy_path) == 0); CHECK(unlink(failed_migration_path) == 0); + CHECK(unlink(v2_path) == 0); CHECK(unlink(failed_v3_migration_path) == 0); CHECK(rmdir(directory) == 0); return true; } diff --git a/tests/test_resource_governor.c b/tests/test_resource_governor.c index 4baa9be..6910def 100644 --- a/tests/test_resource_governor.c +++ b/tests/test_resource_governor.c @@ -430,6 +430,26 @@ run_adaptive_batch_test(void) CHECK(decision.kind == LARDON3D_RESOURCE_START); CHECK(decision.batch_size == 8); + /* Une mesure mémoire à zéro est inconnue : elle ne doit pas réduire un + * lot, même si sa taille et sa durée sont enregistrées. */ + CHECK(lardon3d_resource_governor_record_batch( + governor, + LARDON3D_RESOURCE_TASK_IO, + 8, + 1000000000ULL, + 0 + )); + CHECK(lardon3d_resource_governor_decide(governor, &snapshot, + &(Lardon3DResourceRequest) { + .memory_bytes_per_item = MEBIBYTES(100), + .minimum_batch_size = 2, + .preferred_batch_size = 8, + .requested_cpu_threads = 4, + .io_intensive = true, + }, &decision)); + CHECK(decision.kind == LARDON3D_RESOURCE_START); + CHECK(decision.batch_size == 8); + /* Test 2: Record batch matching estimate → no reduction */ CHECK(lardon3d_resource_governor_record_batch( governor, diff --git a/tests/test_task.c b/tests/test_task.c index dfe5a16..8414ad6 100644 --- a/tests/test_task.c +++ b/tests/test_task.c @@ -26,6 +26,26 @@ typedef struct { Lardon3DResourceReservation *reservation; } StartContext; +typedef struct { + Lardon3DResourceGovernor *governor; + size_t count; + Lardon3DTaskState state; + bool snapshot_succeeded; + bool reservation_released; +} FinishProbe; + +static void +finished_callback(const Lardon3DTask *task, void *userdata) +{ + FinishProbe *probe = userdata; + Lardon3DTaskSnapshot snapshot; + ++probe->count; + probe->snapshot_succeeded = lardon3d_task_snapshot(task, &snapshot); + if (probe->snapshot_succeeded) probe->state = snapshot.state; + probe->reservation_released = !probe->governor + || lardon3d_resource_governor_reservation_count(probe->governor) == 0; +} + static void short_pause(long nanoseconds) { @@ -128,6 +148,9 @@ run_test(void) &work ); CHECK(task); + FinishProbe completed_probe = {.governor = governor}; + CHECK(lardon3d_task_set_finished_callback(task, finished_callback, + &completed_probe)); CHECK(lardon3d_task_assign_id(task, 42)); CHECK(!lardon3d_task_assign_id(task, 43)); CHECK(lardon3d_task_id(task) == 42); @@ -146,6 +169,7 @@ run_test(void) CHECK(wait_for_state(task, TASK_RUNNING)); CHECK(lardon3d_task_pause(task)); CHECK(wait_for_state(task, TASK_PAUSED)); + CHECK(completed_probe.count == 0); CHECK(lardon3d_task_snapshot(task, &snapshot)); unsigned int paused_progress = snapshot.progress; short_pause(5000000); @@ -163,12 +187,18 @@ run_test(void) CHECK(snapshot.progress == 100); CHECK(snapshot.started_at.tv_sec > 0); CHECK(snapshot.finished_at.tv_sec > 0); + CHECK(completed_probe.count == 1 && completed_probe.snapshot_succeeded + && completed_probe.state == TASK_COMPLETED + && completed_probe.reservation_released); CHECK(!lardon3d_task_start(task, NULL, NULL)); lardon3d_task_destroy(task); work = (Work) {.steps = 1000, .pause_ns = 1000000}; task = lardon3d_task_create("Annulation", &estimate, work_callback, &work); CHECK(task); + FinishProbe cancelled_probe = {.governor = governor}; + CHECK(lardon3d_task_set_finished_callback(task, finished_callback, + &cancelled_probe)); CHECK(lardon3d_resource_governor_reserve( governor, &resource_snapshot, &estimate, &decision, &reservation )); @@ -182,10 +212,16 @@ run_test(void) CHECK(lardon3d_task_snapshot(task, &snapshot)); CHECK(snapshot.state == TASK_CANCELLED); CHECK(snapshot.progress < 100); + CHECK(cancelled_probe.count == 1 && cancelled_probe.snapshot_succeeded + && cancelled_probe.state == TASK_CANCELLED + && cancelled_probe.reservation_released); lardon3d_task_destroy(task); task = lardon3d_task_create("Échec", &estimate, failure_callback, NULL); CHECK(task); + FinishProbe failed_probe = {.governor = governor}; + CHECK(lardon3d_task_set_finished_callback(task, finished_callback, + &failed_probe)); CHECK(lardon3d_resource_governor_reserve( governor, &resource_snapshot, &estimate, &decision, &reservation )); @@ -194,6 +230,9 @@ run_test(void) CHECK(lardon3d_task_snapshot(task, &snapshot)); CHECK(snapshot.state == TASK_FAILED); CHECK(strcmp(snapshot.message, "Erreur contrôlée.") == 0); + CHECK(failed_probe.count == 1 && failed_probe.snapshot_succeeded + && failed_probe.state == TASK_FAILED + && failed_probe.reservation_released); lardon3d_task_destroy(task); task = lardon3d_task_create( @@ -206,8 +245,14 @@ run_test(void) CHECK(lardon3d_task_snapshot(task, &snapshot)); CHECK(snapshot.state == TASK_PAUSED); CHECK(lardon3d_task_resume(task)); + FinishProbe pending_cancel_probe = {0}; + CHECK(lardon3d_task_set_finished_callback(task, finished_callback, + &pending_cancel_probe)); + lardon3d_task_request_cancel(task); lardon3d_task_request_cancel(task); CHECK(lardon3d_task_join(task)); + CHECK(pending_cancel_probe.count == 1 + && pending_cancel_probe.state == TASK_CANCELLED); lardon3d_task_destroy(task); lardon3d_resource_governor_destroy(governor); return true; diff --git a/tests/test_task_kind_registry.c b/tests/test_task_kind_registry.c index 4d48560..68c88fd 100644 --- a/tests/test_task_kind_registry.c +++ b/tests/test_task_kind_registry.c @@ -12,11 +12,13 @@ typedef struct { int *destroyed; + int *finished; uint64_t expected_id; } TestUserdata; typedef struct { int *destroyed; + int *finished; bool fail; } ReconstructContext; @@ -37,6 +39,17 @@ destroy_userdata(void *userdata) } } +static void +finished_callback(const Lardon3DTask *task, void *userdata) +{ + TestUserdata *data = userdata; + Lardon3DTaskSnapshot terminal; + if (data && lardon3d_task_snapshot(task, &terminal) + && terminal.state == TASK_COMPLETED && *data->destroyed == 0) { + ++*data->finished; + } +} + static bool reconstruct( const Lardon3DTaskDurableSnapshot *snapshot, @@ -51,11 +64,14 @@ reconstruct( } *data = (TestUserdata) { .destroyed = settings->destroyed, + .finished = settings->finished, .expected_id = snapshot->id, }; binding->callback = test_callback; binding->userdata = data; binding->userdata_destroy = destroy_userdata; + binding->finished_callback = finished_callback; + binding->finished_userdata = data; return !settings->fail; } @@ -113,8 +129,11 @@ run_test(void) CHECK(!lardon3d_task_kind_registry_init(®istry, duplicate, 2)); CHECK(lardon3d_task_kind_registry_init(®istry, descriptors, 2)); - int destroyed = 0; - ReconstructContext context = {.destroyed = &destroyed}; + int destroyed = 0, finished = 0; + ReconstructContext context = { + .destroyed = &destroyed, + .finished = &finished, + }; Lardon3DTaskDurableSnapshot durable = snapshot(); Lardon3DTask *task = NULL; CHECK(lardon3d_task_kind_registry_restore(®istry, "test.recovery", 1, @@ -149,9 +168,10 @@ run_test(void) Lardon3DTaskSnapshot runtime; CHECK(lardon3d_task_snapshot(task, &runtime) && runtime.state == TASK_COMPLETED); + CHECK(finished == 1 && destroyed == 0); lardon3d_task_destroy(task); lardon3d_resource_governor_destroy(governor); - CHECK(destroyed == 1); + CHECK(destroyed == 1 && finished == 1); context.fail = true; CHECK(lardon3d_task_kind_registry_restore(®istry, "test.recovery", 1, diff --git a/tests/test_task_queue.c b/tests/test_task_queue.c index 21b4b9f..9759819 100644 --- a/tests/test_task_queue.c +++ b/tests/test_task_queue.c @@ -223,6 +223,21 @@ run_test(void) queue = lardon3d_task_queue_create(governor, 1024); CHECK(queue); + OrderLog restored_log = {0}; + CHECK(pthread_mutex_init(&restored_log.mutex, NULL) == 0); + QueueWork restored_work = {.log = &restored_log, .value = 0, .steps = 1}; + Lardon3DTask *restored = lardon3d_task_create( + "Restaurée", &estimate, queue_callback, &restored_work); + CHECK(restored && lardon3d_task_assign_id(restored, 100)); + CHECK(lardon3d_task_queue_add(queue, restored, NULL)); + Lardon3DTask *duplicate = lardon3d_task_create( + "Doublon", &estimate, queue_callback, &restored_work); + CHECK(duplicate && lardon3d_task_assign_id(duplicate, 100)); + CHECK(!lardon3d_task_queue_add(queue, duplicate, NULL)); + lardon3d_task_destroy(duplicate); + CHECK(wait_terminal(queue, 100, &snapshot)); + CHECK(lardon3d_task_queue_remove(queue, 100)); + CHECK(pthread_mutex_destroy(&restored_log.mutex) == 0); OrderLog control_log = {0}; CHECK(pthread_mutex_init(&control_log.mutex, NULL) == 0); QueueWork slow = {.log = &control_log, .value = 1, .steps = 500};