feat(import): migrate image import to durable task runtime

This commit is contained in:
fy59 2026-08-08 11:41:34 +02:00
parent 2c122e495f
commit 02c2ecb22c
34 changed files with 1324 additions and 542 deletions

View file

@ -21,13 +21,12 @@ Lardon3D ne vise pas simplement "dossier de photos → objet 3D", mais "ensemble
### Briques implémentées (IMPLEMENTED) ### Briques implémentées (IMPLEMENTED)
- **Project** : cycle de vie persistant, identité stable et Project Database ouverte - **Project** : cycle de vie persistant, identité stable et Project Database ouverte
- **Import** : import asynchrone et annulable d'images - **Import** : premier task kind de production, exécuté par la file générique en lots bornés et reprenables
- **Import Task** : wrapper asynchrone avec états et progression
- **Image Catalog** : indexage des métadonnées d'images - **Image Catalog** : indexage des métadonnées d'images
- **Image View** : vues triées et filtrées pour la TUI - **Image View** : vues triées et filtrées pour la TUI
- **Task** : moteur de tâches avec pause/reprise, annulation et séquences - **Task** : moteur de tâches avec pause/reprise, annulation et séquences
- **Task Checkpoint v1** : snapshot durable, fichier atomique et reprise sûre - **Task Checkpoint v1** : snapshot durable, fichier atomique et reprise sûre
- **Project Database v2** : identité, tâches typées/checkpoints et inventaire d'artefacts SQLite - **Project Database v3** : identité, tâches typées/checkpoints, paramètres d'import et inventaire d'artefacts SQLite
- **Task Kind Registry** : identité métier durable et reconstruction runtime explicite - **Task Kind Registry** : identité métier durable et reconstruction runtime explicite
- **Task Queue** : file FIFO avec sélection adaptative et backpressure - **Task Queue** : file FIFO avec sélection adaptative et backpressure
- **Hardware Profile** : détection des capacités matérielles - **Hardware Profile** : détection des capacités matérielles
@ -41,7 +40,7 @@ Lardon3D ne vise pas simplement "dossier de photos → objet 3D", mais "ensemble
### Briques prévues (PLANNED) ### Briques prévues (PLANNED)
- Premiers types métier reconstructibles et resoumission contrôlée des tâches récupérables - Resoumission contrôlée des tâches récupérables à l'ouverture
- DAG de dépendances - DAG de dépendances
- Pools de workers multiples (CPU/GPU/IO) - Pools de workers multiples (CPU/GPU/IO)
- Publication live validée - Publication live validée

View file

@ -108,7 +108,7 @@ par le `time_t` local avant conversion. Le format reste donc lisible entre
plateformes uniquement pour les valeurs communes à leurs domaines `size_t` et plateformes uniquement pour les valeurs communes à leurs domaines `size_t` et
`time_t`. `time_t`.
## Project Database v2 ## Project Database v3
SQLite contient l'état logique interrogable et les références aux fichiers ; SQLite contient l'état logique interrogable et les références aux fichiers ;
les checkpoints et artefacts volumineux restent externes. L'enregistrement du les checkpoints et artefacts volumineux restent externes. L'enregistrement du
@ -136,7 +136,7 @@ non durable, absent, invalide, version inconnue et erreur d'I/O. Aucune réparat
ou suppression silencieuse n'est effectuée. ou suppression silencieuse n'est effectuée.
Le format checkpoint reste en version 1 et ne contient pas de `task_kind`. Le Le format checkpoint reste en version 1 et ne contient pas de `task_kind`. Le
schéma SQLite v2 conserve `task_kind` et `task_kind_version` dans le résumé schéma SQLite v3 conserve `task_kind` et `task_kind_version` dans le résumé
logique interrogable. La migration v1→v2 laisse ces deux colonnes à `NULL` : une logique interrogable. La migration v1→v2 laisse ces deux colonnes à `NULL` : une
tâche legacy reste inspectable mais ne peut pas être reconstruite ou resoumise. tâche legacy reste inspectable mais ne peut pas être reconstruite ou resoumise.
Un kind inconnu ou une version non supportée est diagnostiqué sans exécuter de Un kind inconnu ou une version non supportée est diagnostiqué sans exécuter de
@ -147,7 +147,7 @@ code.
**IMPLEMENTED** — modèle durable, codec v1, lecture validée, publication **IMPLEMENTED** — modèle durable, codec v1, lecture validée, publication
atomique et restauration sûre d'une tâche isolée. atomique et restauration sûre d'une tâche isolée.
**IMPLEMENTED** — Project Database v2 pour identité, résumés de tâches typées, **IMPLEMENTED** — Project Database v3 pour identité, résumés de tâches typées,
références checkpoint et artefacts génériques. références checkpoint et artefacts génériques.
**IMPLEMENTED** — registry statique bornée et reconstruction explicite avec **IMPLEMENTED** — registry statique bornée et reconstruction explicite avec
@ -156,8 +156,24 @@ ownership du userdata.
**IMPLEMENTED** — API projet de sauvegarde fichier+DB et inventaire validé au **IMPLEMENTED** — API projet de sauvegarde fichier+DB et inventaire validé au
redémarrage. redémarrage.
**NOT_YET_WIRED** — autosave complet, premier type métier de production, **IMPLEMENTED** — `import.images` persiste son chemin source absolu dans une
resoumission scheduler et réconciliation des fichiers orphelins. table dédiée et publie un checkpoint après chaque lot validé. Le manifeste
publié rend le rejeu idempotent à la granularité d'une image.
**PLANNED** — catalogue d'artefacts photogrammétriques réels, migrations v3+ et Le chemin source absolu est l'intention durable v1 : il doit rester accessible
après redémarrage et un projet déplacé ne rend pas une source externe portable.
Une source absente ou devenue non-répertoire fait échouer proprement la
reconstruction. Une image déjà inscrite au manifeste est un résultat validé et
les modifications ultérieures de sa source sont ignorées. Pour la fenêtre
« copie publiée, manifeste non publié », la reprise n'adopte la destination
orpheline qu'après comparaison octet par octet avec la source ; une collision
différente est une erreur. Le manifeste est republié atomiquement avant le
checkpoint de fin de lot. Si ce checkpoint ou sa transaction DB échoue, le
manifeste demeure la frontière idempotente plus récente et un checkpoint
orphelin peut subsister selon le protocole filesystem puis SQLite.
**NOT_YET_WIRED** — autosave complet, resoumission scheduler et réconciliation
des fichiers orphelins.
**PLANNED** — catalogue d'artefacts photogrammétriques réels, migrations v4+ et
reprise globale. reprise globale.

View file

@ -167,9 +167,10 @@ référence vers le fichier checkpoint. Le fichier checkpoint validé reste la
source complète pour `lardon3d_task_restore()` ; la DB seule ne reconstruit source complète pour `lardon3d_task_restore()` ; la DB seule ne reconstruit
jamais une tâche. Un écart ou un fichier invalide interdit la reprise. jamais une tâche. Un écart ou un fichier invalide interdit la reprise.
## Schéma v2 implémenté ## Schéma v3 implémenté
- `metadata(key PRIMARY KEY, value)` contient uniquement `schema_version=2`. - `metadata(key PRIMARY KEY, value)` contient `schema_version=3` et
`next_task_id`, prochain ID durable allouable.
- `project(singleton=1, stable_id UNIQUE, name, created_at, updated_at)` décrit - `project(singleton=1, stable_id UNIQUE, name, created_at, updated_at)` décrit
l'unique identité logique de la DB. l'unique identité logique de la DB.
- `tasks(task_id PRIMARY KEY, name, task_kind, task_kind_version, saved_state, recovery_state, progress, - `tasks(task_id PRIMARY KEY, name, task_kind, task_kind_version, saved_state, recovery_state, progress,
@ -181,20 +182,22 @@ jamais une tâche. Un écart ou un fichier invalide interdit la reprise.
- `artifacts(artifact_id PRIMARY KEY, kind, path, state, size_bytes, - `artifacts(artifact_id PRIMARY KEY, kind, path, state, size_bytes,
producer_task_id REFERENCES tasks, created_at, updated_at)` inventorie des producer_task_id REFERENCES tasks, created_at, updated_at)` inventorie des
fichiers externes. Les états v1 sont `STAGED` et `READY`. fichiers externes. Les états v1 sont `STAGED` et `READY`.
- `image_import_tasks(task_id PRIMARY KEY REFERENCES tasks ON DELETE CASCADE,
source_path)` conserve l'unique paramètre métier v1 de `import.images`.
Les indexes portent uniquement sur `tasks(recovery_state, task_id)`, Les indexes portent uniquement sur `tasks(recovery_state, task_id)`,
`artifacts(state, artifact_id)` et `artifacts(producer_task_id)`. `artifacts(state, artifact_id)` et `artifacts(producer_task_id)`.
## Ouverture et migrations ## Ouverture et migrations
Une DB vide reçoit directement le schéma v2 dans une transaction Une DB vide reçoit directement le schéma v3 dans une transaction
`BEGIN IMMEDIATE`. Une DB v1 reçoit transactionnellement les colonnes nullable `BEGIN IMMEDIATE`. Une DB v1 reçoit transactionnellement les colonnes nullable
`task_kind` et `task_kind_version`, puis passe à v2. Les anciennes lignes restent `task_kind` et `task_kind_version`, puis les migrations v2→v3. Les anciennes lignes restent
`NULL/NULL`, sans type inventé et sans perte des projets, tâches, checkpoints ou `NULL/NULL`, sans type inventé et sans perte des projets, tâches, checkpoints ou
artefacts. Une interruption ou erreur provoque un rollback complet. Une DB v2 artefacts. Une interruption ou erreur provoque un rollback complet. Une DB v2
est validée puis ouverte. Une version future est refusée et une DB contenant est migrée vers v3 ; une DB v3 est validée puis ouverte. Une version future est refusée et une DB contenant
des tables sans métadonnée de version est considérée corrompue. La fonction des tables sans métadonnée de version est considérée corrompue. La fonction
interne de migration ne connaît que `0 → 2` et `1 → 2`. interne de migration ne connaît que `0 → 3`, `1 → 2 → 3` et `2 → 3`.
Migration v1→v2 exacte, exécutée entre `BEGIN IMMEDIATE` et `COMMIT` : Migration v1→v2 exacte, exécutée entre `BEGIN IMMEDIATE` et `COMMIT` :
@ -206,7 +209,25 @@ UPDATE metadata SET value=2
WHERE key='schema_version' AND value=1; WHERE key='schema_version' AND value=1;
``` ```
Configuration v2 : `foreign_keys=ON`, `journal_mode=DELETE`, Migration v2→v3 exacte, exécutée entre `BEGIN IMMEDIATE` et `COMMIT` :
```sql
CREATE TABLE image_import_tasks(
task_id INTEGER PRIMARY KEY REFERENCES tasks(task_id) ON DELETE CASCADE,
source_path TEXT NOT NULL
);
INSERT INTO metadata(key,value)
VALUES('next_task_id',(
SELECT CASE
WHEN COALESCE(MAX(task_id),0)>=9223372036854775807 THEN 0
ELSE COALESCE(MAX(task_id),0)+1
END FROM tasks
));
UPDATE metadata SET value=3
WHERE key='schema_version' AND value=2;
```
Configuration v3 : `foreign_keys=ON`, `journal_mode=DELETE`,
`synchronous=FULL`, `busy_timeout=5000`. Le mode DELETE convient au propriétaire `synchronous=FULL`, `busy_timeout=5000`. Le mode DELETE convient au propriétaire
unique actuel, évite les fichiers WAL/SHM durables et conserve la synchronisation unique actuel, évite les fichiers WAL/SHM durables et conserve la synchronisation
forte. Le timeout borne l'attente d'un verrou externe à cinq secondes. forte. Le timeout borne l'attente d'un verrou externe à cinq secondes.
@ -255,8 +276,9 @@ le mutex DB. Un upsert ne peut pas changer le couple kind/version d'un task ID.
## Statut ## Statut
**IMPLEMENTED** — SQLite système, schéma v2 et migration v1→v2, identité projet, **IMPLEMENTED** — SQLite système, schéma v3 et migrations v1→v2→v3, identité
transactions tâche+checkpoint, pagination de reprise et artefacts génériques. projet, transactions tâche+checkpoint, pagination de reprise et artefacts
génériques.
**IMPLEMENTED** — ouverture/fermeture avec le projet, identité INI/DB cohérente, **IMPLEMENTED** — ouverture/fermeture avec le projet, identité INI/DB cohérente,
publication de checkpoints par le projet et inventaire de reprise validé. publication de checkpoints par le projet et inventaire de reprise validé.
@ -264,9 +286,12 @@ publication de checkpoints par le projet et inventaire de reprise validé.
**IMPLEMENTED** — kinds persistants, classification par registry et **IMPLEMENTED** — kinds persistants, classification par registry et
reconstruction explicite testée hors scheduler. reconstruction explicite testée hors scheduler.
**NOT_YET_WIRED** — type métier de production, resoumission scheduler, autosave **IMPLEMENTED** — allocation transactionnelle de task IDs, paramètres
immuables de `import.images` et reconstruction production explicite.
**NOT_YET_WIRED** — resoumission automatique, autosave
à toutes les transitions et réconciliation des checkpoints orphelins, ScanSet à toutes les transitions et réconciliation des checkpoints orphelins, ScanSet
et catalogue image persistants, Feature Store et Visual Index. et catalogue image persistants, Feature Store et Visual Index.
**PLANNED** — migrations v3+, dépendances d'artefacts, graphe géométrique et **PLANNED** — migrations v4+, dépendances d'artefacts, graphe géométrique et
reconstruction incrémentale. reconstruction incrémentale.

View file

@ -38,7 +38,9 @@ pas encore structuré avec UUID et vague d'import.
| **État de traitement** | Pour chaque image, le catalogue maintient un état : RAW, FEATURES_EXTRACTED, MATCHED, REGISTERED. Cet état est lu par le pipeline pour décider de l'étape suivante. | | **État de traitement** | Pour chaque image, le catalogue maintient un état : RAW, FEATURES_EXTRACTED, MATCHED, REGISTERED. Cet état est lu par le pipeline pour décider de l'étape suivante. |
**Statut :** PARTIELLEMENT IMPLEMENTÉ — le `image_catalog` gère les **Statut :** PARTIELLEMENT IMPLEMENTÉ — le `image_catalog` gère les
métadonnées de base mais pas encore l'état de traitement ni l'UUID. métadonnées de base mais pas encore l'état de traitement ni l'UUID. L'import
`import.images` est exécuté par le scheduler générique, copie au plus 32 images
par lot et reprend à partir du manifeste publié sans dupliquer une entrée.
--- ---

View file

@ -64,6 +64,13 @@ Le Resource Governor est l'unique propriétaire des budgets (RAM, GPU, CPU, IO).
- Sous-estimation temporaire possible avec des estimations statiques - Sous-estimation temporaire possible avec des estimations statiques
- Pas d'adaptation basée sur le débit (duration_ns non encore utilisé) - Pas d'adaptation basée sur le débit (duration_ns non encore utilisé)
- L'import `import.images` est admis avec 128 Kio fixes, un coût borné par item,
un thread CPU, un slot I/O et des lots de 1 à 32. Il enregistre le nombre
d'images nouvellement publiées dans le manifeste et la durée réelle du lot.
Cela inclut une copie orpheline identique adoptée, même si aucun octet n'est
recopié. `peak_memory_bytes == 0` signifie explicitement « mesure inconnue » :
l'échantillon peut conserver taille/durée mais n'alimente jamais l'adaptation
mémoire.
- Pas de communication inter-classes de tâches - Pas de communication inter-classes de tâches
## Limites actuelles ## Limites actuelles

View file

@ -75,8 +75,11 @@ qu'à la terminaison de cette tentative.
**IMPLEMENTED** — snapshot, codec v1 et restauration isolée. **IMPLEMENTED** — snapshot, codec v1 et restauration isolée.
**NOT_YET_WIRED** — sauvegarde périodique, reconstruction métier des tâches et **IMPLEMENTED** — l'import `import.images` se sauvegarde à chaque fin de lot et
resoumission automatique. se reconstruit explicitement avec un userdata neuf lié au projet rouvert.
**NOT_YET_WIRED** — resoumission automatique à l'ouverture et autosave
générique des autres kinds.
**PLANNED** — reprise globale du scheduler via la Project Database. **PLANNED** — reprise globale du scheduler via la Project Database.
@ -92,9 +95,11 @@ fermeture finale du projet.
**IMPLEMENTED** — la registry reconstruit explicitement callback/userdata hors **IMPLEMENTED** — la registry reconstruit explicitement callback/userdata hors
mutex DB pour un kind connu ; elle ne soumet aucune tâche. mutex DB pour un kind connu ; elle ne soumet aucune tâche.
**NOT_YET_WIRED** — aucun type métier de production ni resoumission **IMPLEMENTED** — la queue accepte un identifiant restauré préassigné s'il
automatique. Une future resoumission devra aussi permettre à la queue de n'entre en collision avec aucune tâche connue. L'import production peut donc
conserver l'identifiant stable restauré au lieu d'en assigner un nouveau. être reconstruit puis soumis explicitement.
**NOT_YET_WIRED** — aucune resoumission automatique à `project_open()`.
## Invariants ## Invariants

View file

@ -26,7 +26,7 @@ fin de l'exécution. Le constructeur métier n'est jamais appelé sous mutex DB.
## Persistance et legacy ## Persistance et legacy
Le checkpoint générique reste en version 1. Project Database v2 conserve le Le checkpoint générique reste en version 1. Project Database v3 conserve le
kind/version ; les lignes migrées depuis v1 restent `NULL/NULL` et sont classées kind/version ; les lignes migrées depuis v1 restent `NULL/NULL` et sont classées
`LEGACY_UNTYPED`. Un kind inconnu ou une version non supportée reste inspectable `LEGACY_UNTYPED`. Un kind inconnu ou une version non supportée reste inspectable
mais inexécutable. Aucun type n'est inventé et aucun code n'est sélectionné par mais inexécutable. Aucun type n'est inventé et aucun code n'est sélectionné par
@ -37,8 +37,12 @@ adresse persistée.
**IMPLEMENTED** — identité typée immutable, registry statique, lookup, **IMPLEMENTED** — identité typée immutable, registry statique, lookup,
migration DB v1→v2, classification recovery et restauration explicite testée. migration DB v1→v2, classification recovery et restauration explicite testée.
**NOT_YET_WIRED** — type métier de production, paramètres métier persistants, **IMPLEMENTED** — le descriptor production `import.images`, version 1, charge
autosave complet, resoumission automatique et réconciliation orpheline. le chemin source borné depuis la table dédiée et reconstruit callback et
userdata sans `AppState *` ancien.
**NOT_YET_WIRED** — autosave complet, resoumission automatique et
réconciliation orpheline.
**PLANNED** — kinds des tâches ScanSet, Image Catalog, Feature Store, Visual **PLANNED** — kinds des tâches ScanSet, Image Catalog, Feature Store, Visual
Index et reconstruction lorsque ces traitements existeront réellement. Index et reconstruction lorsque ces traitements existeront réellement.

View file

@ -79,10 +79,26 @@ adaptatives et fondation de checkpoints persistants isolés.
callback et le userdata d'un kind connu. Le destructeur du userdata est détenu callback et le userdata d'un kind connu. Le destructeur du userdata est détenu
par la tâche restaurée et exécuté après arrêt de son exécution. par la tâche restaurée et exécuté après arrêt de son exécution.
**NOT_YET_WIRED** — sauvegarde automatique, types métier de production et **IMPLEMENTED** — `import.images` utilise la pause, l'annulation, les ruptures
restauration par la file. de séquence et les checkpoints génériques ; une tâche restaurée conserve son ID
lors de sa soumission explicite à la file.
La Project Database v2 peut enregistrer transactionnellement un résumé **NOT_YET_WIRED** — autosave générique et resoumission automatique.
Le chemin de production de l'import ne possède plus de thread ni de drapeau
d'annulation privés. Son wrapper TUI ne fait qu'enqueue/cancel/observer la
tâche générique. Chaque callback traite un lot borné, checkpoint hors mutex de
tâche, puis effectue une rupture de séquence afin d'obtenir un nouveau contrat
et une nouvelle réservation.
Le callback terminal optionnel est notifié exactement une fois pour
`COMPLETED`, `FAILED` ou `CANCELLED`, jamais pour une pause ou une rupture de
séquence. L'état est fixé sous mutex, puis la réservation terminale est libérée
avant l'appel hors mutex. `join()` attend la fin du callback ; le userdata reste
donc valide pendant celui-ci et son destructeur n'est appelé qu'ensuite par la
destruction de la tâche.
La Project Database v3 peut enregistrer transactionnellement un résumé
`Lardon3DTaskDurableSnapshot` et la référence de son checkpoint. Elle ne stocke `Lardon3DTaskDurableSnapshot` et la référence de son checkpoint. Elle ne stocke
ni estimation sérialisée complète, ni callback, ni réservation, et ne remplace ni estimation sérialisée complète, ni callback, ni réservation, et ne remplace
pas la validation du fichier checkpoint avant `task_restore()`. pas la validation du fichier checkpoint avant `task_restore()`.

View file

@ -9,6 +9,8 @@ Une **acquisition** désigne le processus physique de capture des images : posit
## Statut ## Statut
**PLANNED** — Concept définissant la structure d'entrée, pas encore implémenté comme module distinct. Actuellement, le module `import` et `image_catalog` gèrent les images individuellement sans regroupement en scan sets. **PLANNED** — Concept définissant la structure d'entrée, pas encore implémenté comme module distinct. Actuellement, le module `import` et `image_catalog` gèrent les images individuellement sans regroupement en scan sets.
Le chemin source durable de `import.images` ne constitue donc pas encore une
identité de ScanSet.
## Place dans le pipeline ## Place dans le pipeline

View file

@ -9,7 +9,7 @@ Lardon3D suit une feuille de route ordonnée qui privilégie la stabilité et la
### Phase 1 : Fondations ### Phase 1 : Fondations
- ✅ TUI modulaire avec ncursesw - ✅ TUI modulaire avec ncursesw
- ✅ Gestion persistante des projets - ✅ Gestion persistante des projets
- ✅ Import asynchrone et annulable - ✅ Import d'images migré vers le scheduler générique, borné et reconstructible
- ✅ Catalogue d'images et vues - ✅ Catalogue d'images et vues
- ✅ Moteur de tâches avec pause/reprise, annulation, checkpoints - ✅ Moteur de tâches avec pause/reprise, annulation, checkpoints
- ✅ File FIFO avec sélection adaptative et backpressure - ✅ File FIFO avec sélection adaptative et backpressure
@ -29,10 +29,11 @@ Lardon3D suit une feuille de route ordonnée qui privilégie la stabilité et la
### Phase 3 : Persistance ### Phase 3 : Persistance
- ✅ Fondation versionnée des checkpoints de tâches - ✅ Fondation versionnée des checkpoints de tâches
- ✅ Project Database v2 (SQLite, migration v1 et tâches typées) - ✅ Project Database v3 (paramètres d'import et allocation durable des task IDs)
- ✅ Branchement Project Database au cycle de vie projet et inventaire de reprise - ✅ Branchement Project Database au cycle de vie projet et inventaire de reprise
- ✅ Registry durable des types métier de tâches - ✅ Registry durable des types métier de tâches
- 📋 Premier type métier reconstructible et resoumission des tâches récupérables - ✅ Premier type métier reconstructible (`import.images`)
- 📋 Resoumission automatique contrôlée des tâches récupérables
- 📋 ScanSet et Image Catalog persistants - 📋 ScanSet et Image Catalog persistants
### Phase 4 : Pipeline avancé ### Phase 4 : Pipeline avancé

View file

@ -8,6 +8,8 @@
typedef struct { typedef struct {
size_t admissible_found; size_t admissible_found;
size_t processed;
size_t newly_manifested;
size_t copied; size_t copied;
size_t already_present; size_t already_present;
size_t ignored; size_t ignored;
@ -53,4 +55,13 @@ Lardon3DImportOutcome lardon3d_import_directory_controlled(
const Lardon3DImportControl *control const Lardon3DImportControl *control
); );
Lardon3DImportOutcome lardon3d_import_directory_batch(
Lardon3DAppState *state,
const char *source_directory,
size_t batch_size,
Lardon3DImportResult *result,
const Lardon3DImportControl *control,
bool *complete
);
#endif #endif

View file

@ -3,8 +3,13 @@
#include <stdbool.h> #include <stdbool.h>
#include <stddef.h> #include <stddef.h>
#include <stdint.h>
#include <lardon3d/app_state.h> #include <lardon3d/app_state.h>
#include <lardon3d/task_kind_registry.h>
#define LARDON3D_IMAGE_IMPORT_TASK_KIND "import.images"
enum { LARDON3D_IMAGE_IMPORT_TASK_KIND_VERSION = 1 };
typedef enum { typedef enum {
LARDON3D_IMPORT_TASK_IDLE = 0, LARDON3D_IMPORT_TASK_IDLE = 0,
@ -26,10 +31,33 @@ typedef struct {
typedef struct Lardon3DImportTask Lardon3DImportTask; typedef struct Lardon3DImportTask Lardon3DImportTask;
typedef struct {
const char *project_path;
Lardon3DProjectDb *project_db;
Lardon3DResourceGovernor *resource_governor;
} Lardon3DImageImportReconstructionContext;
Lardon3DTask *lardon3d_project_create_image_import_task(
Lardon3DAppState *state,
const char *source_directory,
uint64_t *task_id
);
bool lardon3d_project_enqueue_image_import(
Lardon3DAppState *state,
const char *source_directory,
uint64_t *task_id
);
bool lardon3d_image_import_reconstruct(
const Lardon3DTaskDurableSnapshot *snapshot,
void *context,
Lardon3DTaskKindBinding *binding
);
/* Compatibilité TUI : handle léger sur une tâche de la queue, sans thread. */
Lardon3DImportTask *lardon3d_import_task_create(void); Lardon3DImportTask *lardon3d_import_task_create(void);
bool lardon3d_import_task_start( bool lardon3d_import_task_start(
Lardon3DImportTask *task, Lardon3DImportTask *task,
const Lardon3DAppState *state, Lardon3DAppState *state,
const char *source_directory const char *source_directory
); );
void lardon3d_import_task_request_cancel(Lardon3DImportTask *task); void lardon3d_import_task_request_cancel(Lardon3DImportTask *task);

View file

@ -54,6 +54,12 @@ Lardon3DProjectTaskCheckpointResult lardon3d_project_checkpoint_task(
Lardon3DAppState *state, Lardon3DAppState *state,
const Lardon3DTask *task const Lardon3DTask *task
); );
Lardon3DProjectTaskCheckpointResult
lardon3d_project_checkpoint_image_import_task(
Lardon3DAppState *state,
const Lardon3DTask *task,
const char *source_path
);
Lardon3DProjectDbResult lardon3d_project_list_recoverable( Lardon3DProjectDbResult lardon3d_project_list_recoverable(
Lardon3DAppState *state, Lardon3DAppState *state,
const Lardon3DTaskKindRegistry *registry, const Lardon3DTaskKindRegistry *registry,

View file

@ -9,7 +9,7 @@
#include <lardon3d/task.h> #include <lardon3d/task.h>
enum { enum {
LARDON3D_PROJECT_DB_SCHEMA_VERSION = 2, LARDON3D_PROJECT_DB_SCHEMA_VERSION = 3,
LARDON3D_PROJECT_DB_ID_CAPACITY = 65, LARDON3D_PROJECT_DB_ID_CAPACITY = 65,
LARDON3D_PROJECT_DB_KIND_CAPACITY = 65, LARDON3D_PROJECT_DB_KIND_CAPACITY = 65,
LARDON3D_PROJECT_DB_PATH_CAPACITY = 4096, LARDON3D_PROJECT_DB_PATH_CAPACITY = 4096,
@ -83,6 +83,11 @@ typedef struct {
int64_t updated_at; int64_t updated_at;
} Lardon3DProjectDbArtifact; } Lardon3DProjectDbArtifact;
typedef struct {
uint64_t task_id;
char source_path[LARDON3D_PROJECT_DB_PATH_CAPACITY];
} Lardon3DProjectDbImageImport;
Lardon3DProjectDbResult lardon3d_project_db_open( Lardon3DProjectDbResult lardon3d_project_db_open(
const char *path, const char *path,
Lardon3DProjectDb **database, Lardon3DProjectDb **database,
@ -111,6 +116,24 @@ Lardon3DProjectDbResult lardon3d_project_db_record_task(
const Lardon3DProjectDbCheckpoint *checkpoint, const Lardon3DProjectDbCheckpoint *checkpoint,
int64_t updated_at int64_t updated_at
); );
Lardon3DProjectDbResult lardon3d_project_db_record_image_import_task(
Lardon3DProjectDb *database,
const Lardon3DTaskDurableSnapshot *snapshot,
const char *task_kind,
uint32_t task_kind_version,
const Lardon3DProjectDbCheckpoint *checkpoint,
const char *source_path,
int64_t updated_at
);
Lardon3DProjectDbResult lardon3d_project_db_load_image_import(
Lardon3DProjectDb *database,
uint64_t task_id,
Lardon3DProjectDbImageImport *parameters
);
Lardon3DProjectDbResult lardon3d_project_db_allocate_task_id(
Lardon3DProjectDb *database,
uint64_t *task_id
);
Lardon3DProjectDbResult lardon3d_project_db_load_task( Lardon3DProjectDbResult lardon3d_project_db_load_task(
Lardon3DProjectDb *database, Lardon3DProjectDb *database,
uint64_t task_id, uint64_t task_id,

View file

@ -188,8 +188,10 @@ const char *lardon3d_resource_decision_name(
Lardon3DResourceDecisionKind kind Lardon3DResourceDecisionKind kind
); );
/* Enregistre les métriques d'un lot terminé pour l'adaptation dynamique /* Enregistre les métriques d'un lot terminé pour l'adaptation dynamique
* de la taille des lots futurs. Le buffer est borné (8 entrées par classe * de la taille des lots futurs. batch_size est le nombre d'éléments dont le
* de tâche). Thread-safe. */ * traitement a é validé dans ce lot. peak_memory_bytes == 0 signifie que
* la mesure est inconnue et n'alimente jamais l'adaptation mémoire. Le buffer
* est borné (8 entrées par classe de tâche). Thread-safe. */
bool lardon3d_resource_governor_record_batch( bool lardon3d_resource_governor_record_batch(
Lardon3DResourceGovernor *governor, Lardon3DResourceGovernor *governor,
Lardon3DResourceTaskClass task_class, Lardon3DResourceTaskClass task_class,

View file

@ -25,6 +25,10 @@ typedef enum {
typedef struct Lardon3DTask Lardon3DTask; typedef struct Lardon3DTask Lardon3DTask;
typedef bool (*Lardon3DTaskCallback)(Lardon3DTask *task, void *userdata); typedef bool (*Lardon3DTaskCallback)(Lardon3DTask *task, void *userdata);
typedef void (*Lardon3DTaskUserdataDestroy)(void *userdata); typedef void (*Lardon3DTaskUserdataDestroy)(void *userdata);
typedef void (*Lardon3DTaskFinishedCallback)(
const Lardon3DTask *task,
void *userdata
);
typedef struct { typedef struct {
uint64_t id; uint64_t id;
@ -120,6 +124,13 @@ bool lardon3d_task_kind(
char task_kind[LARDON3D_TASK_KIND_CAPACITY], char task_kind[LARDON3D_TASK_KIND_CAPACITY],
uint32_t *task_kind_version uint32_t *task_kind_version
); );
/* Appelé au plus une fois, hors mutex de tâche et après libération de la
* réservation terminale. Le userdata de tâche reste vivant jusqu'au retour. */
bool lardon3d_task_set_finished_callback(
Lardon3DTask *task,
Lardon3DTaskFinishedCallback callback,
void *userdata
);
uint64_t lardon3d_task_id(const Lardon3DTask *task); uint64_t lardon3d_task_id(const Lardon3DTask *task);
bool lardon3d_task_assign_id(Lardon3DTask *task, uint64_t id); bool lardon3d_task_assign_id(Lardon3DTask *task, uint64_t id);
bool lardon3d_task_resource_estimate( bool lardon3d_task_resource_estimate(

View file

@ -14,6 +14,8 @@ typedef struct {
Lardon3DTaskCallback callback; Lardon3DTaskCallback callback;
void *userdata; void *userdata;
Lardon3DTaskUserdataDestroy userdata_destroy; Lardon3DTaskUserdataDestroy userdata_destroy;
Lardon3DTaskFinishedCallback finished_callback;
void *finished_userdata;
} Lardon3DTaskKindBinding; } Lardon3DTaskKindBinding;
typedef bool (*Lardon3DTaskKindReconstruct)( typedef bool (*Lardon3DTaskKindReconstruct)(
@ -61,5 +63,6 @@ Lardon3DTaskKindResult lardon3d_task_kind_registry_restore(
void *context, void *context,
Lardon3DTask **task Lardon3DTask **task
); );
const Lardon3DTaskKindRegistry *lardon3d_task_kind_registry_production(void);
#endif #endif

View file

@ -39,6 +39,7 @@ executable(
'src/task.c', 'src/task.c',
'src/task_checkpoint.c', 'src/task_checkpoint.c',
'src/task_kind_registry.c', 'src/task_kind_registry.c',
'src/task_kinds.c',
'src/task_queue.c', 'src/task_queue.c',
'src/resource_governor.c', 'src/resource_governor.c',
'src/resource_snapshot.c', 'src/resource_snapshot.c',
@ -67,9 +68,25 @@ import_task_test = executable(
'src/app_state.c', 'src/app_state.c',
'src/import.c', 'src/import.c',
'src/import_task.c', 'src/import_task.c',
'src/project.c',
'src/project_db.c',
'src/task.c',
'src/task_checkpoint.c',
'src/task_kind_registry.c',
'src/task_kinds.c',
'src/task_queue.c',
'src/resource_governor.c',
'src/resource_snapshot.c',
'src/image_catalog.c',
'src/image_view.c',
],
c_args: [
'-DLARDON3D_IMPORT_TASK_TESTING',
'-DLARDON3D_PROJECT_DB_TESTING',
'-DLARDON3D_CHECKPOINT_TESTING',
], ],
include_directories: include_directories('include'), include_directories: include_directories('include'),
dependencies: [threads], dependencies: [threads, sqlite3],
) )
test('import-task', import_task_test, timeout: 30) test('import-task', import_task_test, timeout: 30)

View file

@ -450,6 +450,16 @@ manifest_commit(Lardon3DAppState *state, ManifestWriter *writer)
return success; return success;
} }
static ssize_t
read_retry(int descriptor, void *buffer, size_t capacity)
{
ssize_t count;
do {
count = read(descriptor, buffer, capacity);
} while (count < 0 && errno == EINTR);
return count;
}
static int static int
copy_image( copy_image(
const char *source_path, const char *source_path,
@ -479,8 +489,38 @@ copy_image(
mode mode
); );
if (destination < 0) { if (destination < 0) {
if (errno != EEXIST) {
(void)close(source); (void)close(source);
return errno == EEXIST ? 0 : -1; return COPY_IMAGE_ERROR;
}
destination = open(destination_path, O_RDONLY | O_NOFOLLOW);
struct stat destination_info;
bool identical = destination >= 0
&& fstat(destination, &destination_info) == 0
&& S_ISREG(destination_info.st_mode)
&& destination_info.st_size == source_info.st_size;
char source_buffer[64 * 1024], destination_buffer[64 * 1024];
while (identical) {
ssize_t source_count = read_retry(source, source_buffer,
sizeof(source_buffer));
ssize_t destination_count = read_retry(destination,
destination_buffer, sizeof(destination_buffer));
if (source_count < 0 || destination_count < 0
|| source_count != destination_count) {
identical = false;
break;
}
if (source_count == 0) break;
if (memcmp(source_buffer, destination_buffer,
(size_t)source_count) != 0) {
identical = false;
}
}
if (destination >= 0 && close(destination) != 0) identical = false;
if (close(source) != 0) identical = false;
if (!identical) return COPY_IMAGE_ERROR;
*destination_size = source_info.st_size;
return COPY_IMAGE_COLLISION;
} }
bool success = true; bool success = true;
@ -722,6 +762,147 @@ cancel_import(
return LARDON3D_IMPORT_CANCELLED; return LARDON3D_IMPORT_CANCELLED;
} }
Lardon3DImportOutcome
lardon3d_import_directory_batch(
Lardon3DAppState *state,
const char *source_directory,
size_t batch_size,
Lardon3DImportResult *result,
const Lardon3DImportControl *control,
bool *complete
)
{
if (complete) *complete = false;
if (!state || !result || !complete || batch_size == 0
|| !state->project_loaded) {
return LARDON3D_IMPORT_FAILED;
}
*result = (Lardon3DImportResult) {0};
char trimmed_source[PATH_MAX], absolute_source[PATH_MAX];
if (!trim_source_path(state, source_directory, trimmed_source)
|| !resolve_source_path(state, trimmed_source, absolute_source)
|| has_forbidden_manifest_character(absolute_source)) {
return LARDON3D_IMPORT_FAILED;
}
struct stat source_info;
if (lstat(absolute_source, &source_info) != 0
|| !S_ISDIR(source_info.st_mode)) {
set_status(state, "Erreur : dossier source absent ou invalide.");
return LARDON3D_IMPORT_FAILED;
}
char images_path[PATH_MAX], originals_path[PATH_MAX];
if (!join_path(images_path, state->project_path, "images")
|| !join_path(originals_path, images_path, "originals")
|| !ensure_originals_directory(state, images_path, originals_path)) {
return LARDON3D_IMPORT_FAILED;
}
ManifestWriter manifest;
if (!manifest_begin(state, images_path, &manifest)) {
return LARDON3D_IMPORT_FAILED;
}
DIR *directory = opendir(absolute_source);
if (!directory) {
manifest_abort(&manifest);
set_status(state, "Erreur : impossible d'ouvrir le dossier source.");
return LARDON3D_IMPORT_FAILED;
}
CandidateList created = {0};
bool success = true, cancelled = false;
size_t added = 0;
for (;;) {
if (import_is_cancelled(control)) {
cancelled = true;
break;
}
errno = 0;
struct dirent *entry = readdir(directory);
if (!entry) {
if (errno != 0) success = false;
break;
}
if (strcmp(entry->d_name, ".") == 0
|| strcmp(entry->d_name, "..") == 0) continue;
char source_path[PATH_MAX], destination_path[PATH_MAX];
struct stat item_info;
if (!join_path(source_path, absolute_source, entry->d_name)) {
success = false; break;
}
if (lstat(source_path, &item_info) != 0
|| !S_ISREG(item_info.st_mode)
|| !has_supported_extension(entry->d_name)) {
++result->ignored;
continue;
}
++result->admissible_found;
if (has_forbidden_manifest_character(entry->d_name)) {
success = false; break;
}
int listed = manifest_contains(&manifest, entry->d_name);
if (listed < 0) {
success = false; break;
}
if (listed > 0) {
++result->processed;
++result->already_present;
continue;
}
if (added >= batch_size
|| !join_path(destination_path, originals_path, entry->d_name)) {
if (added < batch_size) success = false;
continue;
}
off_t size = 0;
int copied = copy_image(source_path, destination_path, &size, control);
if (copied == COPY_IMAGE_CANCELLED) {
cancelled = true; break;
}
if (copied == COPY_IMAGE_ERROR) {
success = false; break;
}
if (copied == COPY_IMAGE_COLLISION) {
struct stat destination_info;
if (lstat(destination_path, &destination_info) != 0
|| !S_ISREG(destination_info.st_mode)) {
success = false; break;
}
size = destination_info.st_size;
++result->already_present;
} else {
if (!candidate_list_append(state, &created, entry->d_name)) {
(void)unlink(destination_path);
success = false; break;
}
created.items[created.count - 1].created = true;
++result->copied;
}
if (!manifest_append(&manifest, entry->d_name, size, source_path)) {
success = false; break;
}
++added;
++result->newly_manifested;
++result->processed;
}
if (closedir(directory) != 0) success = false;
if (!success || cancelled) {
manifest_abort(&manifest);
(void)rollback_created_files(&created, originals_path);
free(created.items);
set_status(state, cancelled ? "Import annulé à une frontière sûre."
: "Erreur pendant un lot d'import.");
return cancelled ? LARDON3D_IMPORT_CANCELLED : LARDON3D_IMPORT_FAILED;
}
if (!manifest_commit(state, &manifest)) {
(void)rollback_created_files(&created, originals_path);
free(created.items);
return LARDON3D_IMPORT_FAILED;
}
free(created.items);
*complete = result->processed == result->admissible_found;
set_status(state, *complete ? "Import terminé." : "Lot d'import publié.");
publish_progress(control, result, result->processed, state->status_message);
return LARDON3D_IMPORT_SUCCEEDED;
}
Lardon3DImportOutcome Lardon3DImportOutcome
lardon3d_import_directory_controlled( lardon3d_import_directory_controlled(
Lardon3DAppState *state, Lardon3DAppState *state,

View file

@ -1,229 +1,316 @@
#include <pthread.h> #include <errno.h>
#include <stdbool.h> #include <stdbool.h>
#include <stdint.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>
#include <lardon3d/import.h> #include <lardon3d/import.h>
#include <lardon3d/import_task.h> #include <lardon3d/import_task.h>
#include <lardon3d/project.h>
#include <lardon3d/task_queue.h>
struct Lardon3DImportTask { enum {
pthread_mutex_t mutex; IMAGE_IMPORT_MINIMUM_BATCH = 1,
pthread_t thread; IMAGE_IMPORT_MAXIMUM_BATCH = 32,
bool thread_started; IMAGE_IMPORT_FIXED_MEMORY = 128 * 1024,
bool joined; IMAGE_IMPORT_MEMORY_PER_ITEM = NAME_MAX + 64,
bool cancel_requested;
Lardon3DAppState worker_state;
char source_directory[PATH_MAX];
Lardon3DImportTaskSnapshot snapshot;
}; };
static void typedef struct {
copy_message(char destination[256], const char *message) char project_path[PATH_MAX];
char source_path[PATH_MAX];
Lardon3DProjectDb *project_db;
Lardon3DResourceGovernor *governor;
} Lardon3DImageImportContext;
struct Lardon3DImportTask {
Lardon3DAppState *state;
uint64_t task_id;
};
static bool
canonical_source(const char *source, char output[PATH_MAX])
{ {
(void)snprintf(destination, 256, "%s", message ? message : ""); if (!source || !source[0] || strnlen(source, PATH_MAX) >= PATH_MAX) return false;
char absolute[PATH_MAX];
int written;
if (source[0] == '/') {
written = snprintf(absolute, sizeof(absolute), "%s", source);
} else {
char current[PATH_MAX];
if (!getcwd(current, sizeof(current))) return false;
written = snprintf(absolute, sizeof(absolute), "%s/%s", current, source);
}
if (written <= 0 || (size_t)written >= sizeof(absolute)) return false;
struct stat information;
if (lstat(absolute, &information) != 0 || S_ISLNK(information.st_mode)
|| !S_ISDIR(information.st_mode)) return false;
written = snprintf(output, PATH_MAX, "%s", absolute);
return written > 0 && (size_t)written < PATH_MAX;
}
static void
destroy_context(void *userdata)
{
free(userdata);
}
static void
runtime_project(const Lardon3DImageImportContext *context, Lardon3DAppState *state)
{
lardon3d_app_state_init(state);
state->project_loaded = true;
state->project_db = context->project_db;
state->resource_governor = context->governor;
(void)snprintf(state->project_path, sizeof(state->project_path), "%s",
context->project_path);
} }
static bool static bool
task_is_cancelled(void *context) cooperative_continue(void *userdata)
{ {
Lardon3DImportTask *task = context; return !lardon3d_task_checkpoint(userdata);
bool cancelled;
(void)pthread_mutex_lock(&task->mutex);
cancelled = task->cancel_requested;
(void)pthread_mutex_unlock(&task->mutex);
return cancelled;
} }
static void static void
task_progressed(void *context, const Lardon3DImportProgress *progress) import_finished(const Lardon3DTask *task, void *userdata)
{ {
Lardon3DImportTask *task = context; #ifdef LARDON3D_IMPORT_TASK_TESTING
(void)pthread_mutex_lock(&task->mutex); const char *skip = getenv("LARDON3D_TEST_IMPORT_SKIP_FINISHED_CHECKPOINT");
task->snapshot.total = progress->total; if (skip && strcmp(skip, "1") == 0) return;
task->snapshot.processed = progress->processed; #endif
task->snapshot.copied = progress->copied; Lardon3DImageImportContext *context = userdata;
task->snapshot.already_present = progress->already_present; Lardon3DAppState state;
task->snapshot.ignored = progress->ignored; runtime_project(context, &state);
copy_message(task->snapshot.message, progress->message); (void)lardon3d_project_checkpoint_task(&state, task);
(void)pthread_mutex_unlock(&task->mutex);
} }
static void * static uint64_t
run_import(void *argument) elapsed_ns(struct timespec begin, struct timespec end)
{ {
Lardon3DImportTask *task = argument; uint64_t seconds = end.tv_sec >= begin.tv_sec
Lardon3DImportResult result; ? (uint64_t)(end.tv_sec - begin.tv_sec) : 0;
const Lardon3DImportControl control = { long nanoseconds = end.tv_nsec - begin.tv_nsec;
if (nanoseconds < 0 && seconds > 0) {
--seconds;
nanoseconds += 1000000000L;
}
return seconds <= UINT64_MAX / UINT64_C(1000000000)
? seconds * UINT64_C(1000000000) + (uint64_t)nanoseconds : UINT64_MAX;
}
static bool
run_image_import(Lardon3DTask *task, void *userdata)
{
Lardon3DImageImportContext *context = userdata;
Lardon3DAppState state;
runtime_project(context, &state);
for (;;) {
Lardon3DTaskExecutionContract contract;
if (!lardon3d_task_execution_contract(task, &contract)
|| contract.batch_size < IMAGE_IMPORT_MINIMUM_BATCH
|| contract.batch_size > IMAGE_IMPORT_MAXIMUM_BATCH) {
return lardon3d_task_fail(task, "Contrat de lot import invalide.");
}
Lardon3DImportControl control = {
.context = task, .context = task,
.is_cancelled = task_is_cancelled, .is_cancelled = cooperative_continue,
.progressed = task_progressed,
}; };
Lardon3DImportOutcome outcome = lardon3d_import_directory_controlled( Lardon3DImportResult result;
&task->worker_state, bool complete = false;
task->source_directory, struct timespec begin, end;
&result, (void)clock_gettime(CLOCK_MONOTONIC, &begin);
&control Lardon3DImportOutcome outcome = lardon3d_import_directory_batch(
); &state, context->source_path, contract.batch_size, &result,
&control, &complete);
(void)pthread_mutex_lock(&task->mutex); (void)clock_gettime(CLOCK_MONOTONIC, &end);
task->snapshot.status = outcome == LARDON3D_IMPORT_SUCCEEDED if (outcome == LARDON3D_IMPORT_CANCELLED) return false;
? LARDON3D_IMPORT_TASK_SUCCEEDED if (outcome != LARDON3D_IMPORT_SUCCEEDED) {
: outcome == LARDON3D_IMPORT_CANCELLED return lardon3d_task_fail(task, state.status_message);
? LARDON3D_IMPORT_TASK_CANCELLED }
: LARDON3D_IMPORT_TASK_FAILED; unsigned int progress = complete ? 100U : result.admissible_found == 0
task->snapshot.total = result.admissible_found; ? 99U : (unsigned int)((result.processed * 99U)
task->snapshot.copied = result.copied; / result.admissible_found);
task->snapshot.already_present = result.already_present; if (!lardon3d_task_set_progress(task, progress, state.status_message)) {
copy_message(task->snapshot.message, task->worker_state.status_message); return false;
(void)pthread_mutex_unlock(&task->mutex); }
return NULL; (void)lardon3d_resource_governor_record_batch(
context->governor, LARDON3D_RESOURCE_TASK_IMPORT,
result.newly_manifested,
elapsed_ns(begin, end), 0);
if (lardon3d_project_checkpoint_image_import_task(
&state, task, context->source_path)
!= LARDON3D_PROJECT_TASK_CHECKPOINT_OK) {
return lardon3d_task_fail(task, "Checkpoint import impossible.");
}
if (complete) return true;
#ifdef LARDON3D_IMPORT_TASK_TESTING
const char *pause = getenv("LARDON3D_TEST_IMPORT_PAUSE_AFTER_BATCH");
if (pause && strcmp(pause, "1") == 0) {
(void)lardon3d_task_pause(task);
}
#endif
Lardon3DResourceReservation *reservation = NULL;
if (!lardon3d_task_sequence_break(task, context->governor,
&reservation, &contract)) return false;
}
} }
Lardon3DImportTask * static Lardon3DImageImportContext *
lardon3d_import_task_create(void) create_context(const char *project_path, const char *source_path,
Lardon3DProjectDb *database, Lardon3DResourceGovernor *governor)
{ {
Lardon3DImportTask *task = calloc(1, sizeof(*task)); if (!project_path || !source_path || !database || !governor) return NULL;
if (!task) { Lardon3DImageImportContext *context = calloc(1, sizeof(*context));
if (!context) return NULL;
int project_written = snprintf(context->project_path,
sizeof(context->project_path), "%s", project_path);
int source_written = snprintf(context->source_path,
sizeof(context->source_path), "%s", source_path);
if (project_written <= 0 || (size_t)project_written >= sizeof(context->project_path)
|| source_written <= 0 || (size_t)source_written >= sizeof(context->source_path)) {
free(context); return NULL;
}
context->project_db = database;
context->governor = governor;
return context;
}
bool
lardon3d_image_import_reconstruct(
const Lardon3DTaskDurableSnapshot *snapshot,
void *userdata,
Lardon3DTaskKindBinding *binding
)
{
Lardon3DImageImportReconstructionContext *runtime = userdata;
if (!snapshot || !runtime || !runtime->project_path || !runtime->project_db
|| !runtime->resource_governor || !binding) return false;
Lardon3DProjectDbImageImport parameters;
if (lardon3d_project_db_load_image_import(runtime->project_db, snapshot->id,
&parameters) != LARDON3D_PROJECT_DB_OK) return false;
char source[PATH_MAX];
if (!canonical_source(parameters.source_path, source)) return false;
Lardon3DImageImportContext *context = create_context(runtime->project_path,
source, runtime->project_db, runtime->resource_governor);
if (!context) return false;
*binding = (Lardon3DTaskKindBinding) {
.callback = run_image_import,
.userdata = context,
.userdata_destroy = destroy_context,
.finished_callback = import_finished,
.finished_userdata = context,
};
return true;
}
Lardon3DTask *
lardon3d_project_create_image_import_task(
Lardon3DAppState *state,
const char *source_directory,
uint64_t *task_id
)
{
if (task_id) *task_id = 0;
if (!state || !state->project_loaded || !state->project_db
|| !state->resource_governor || !task_id) return NULL;
char source[PATH_MAX];
if (!canonical_source(source_directory, source)) return NULL;
uint64_t id = 0;
if (lardon3d_project_db_allocate_task_id(state->project_db, &id)
!= LARDON3D_PROJECT_DB_OK) return NULL;
Lardon3DImageImportContext *context = create_context(state->project_path,
source, state->project_db, state->resource_governor);
if (!context) return NULL;
const Lardon3DResourceEstimate estimate = {
.memory_fixed_bytes = IMAGE_IMPORT_FIXED_MEMORY,
.memory_bytes_per_item = IMAGE_IMPORT_MEMORY_PER_ITEM,
.minimum_batch_size = IMAGE_IMPORT_MINIMUM_BATCH,
.maximum_batch_size = IMAGE_IMPORT_MAXIMUM_BATCH,
.desired_cpu_threads = 1,
.desired_io_slots = 1,
.task_class = LARDON3D_RESOURCE_TASK_IMPORT,
};
Lardon3DTask *task = lardon3d_task_create_typed("Import d'images",
&estimate, LARDON3D_IMAGE_IMPORT_TASK_KIND,
LARDON3D_IMAGE_IMPORT_TASK_KIND_VERSION, run_image_import, context,
destroy_context);
if (!task || !lardon3d_task_assign_id(task, id)
|| !lardon3d_task_set_finished_callback(task, import_finished, context)
|| lardon3d_project_checkpoint_image_import_task(state, task, source)
!= LARDON3D_PROJECT_TASK_CHECKPOINT_OK) {
lardon3d_task_destroy(task);
return NULL; return NULL;
} }
if (pthread_mutex_init(&task->mutex, NULL) != 0) { *task_id = id;
free(task);
return NULL;
}
task->snapshot.status = LARDON3D_IMPORT_TASK_IDLE;
return task; return task;
} }
bool bool
lardon3d_import_task_start( lardon3d_project_enqueue_image_import(Lardon3DAppState *state,
Lardon3DImportTask *task, const char *source_directory, uint64_t *task_id)
const Lardon3DAppState *state,
const char *source_directory
)
{ {
if (!task || !state || !source_directory) { if (!state || !state->task_queue) return false;
Lardon3DTask *task = lardon3d_project_create_image_import_task(
state, source_directory, task_id);
if (!task) return false;
if (!lardon3d_task_queue_add(state->task_queue, task, NULL)) {
lardon3d_task_destroy(task);
return false; return false;
} }
return true;
}
(void)pthread_mutex_lock(&task->mutex); Lardon3DImportTask *lardon3d_import_task_create(void) { return calloc(1, sizeof(Lardon3DImportTask)); }
if (task->thread_started) {
(void)pthread_mutex_unlock(&task->mutex); bool
return false; lardon3d_import_task_start(Lardon3DImportTask *task, Lardon3DAppState *state,
const char *source_directory)
{
if (!task || task->task_id != 0 || !state) return false;
task->state = state;
return lardon3d_project_enqueue_image_import(state, source_directory,
&task->task_id);
} }
int written = snprintf(
task->source_directory, void lardon3d_import_task_request_cancel(Lardon3DImportTask *task)
sizeof(task->source_directory), { if (task && task->state) (void)lardon3d_task_queue_cancel(task->state->task_queue, task->task_id); }
"%s",
source_directory bool
); lardon3d_import_task_snapshot(Lardon3DImportTask *task,
if (written < 0 || (size_t)written >= sizeof(task->source_directory)) { Lardon3DImportTaskSnapshot *snapshot)
copy_message(task->snapshot.message, "Erreur : chemin source trop long."); {
task->snapshot.status = LARDON3D_IMPORT_TASK_FAILED; if (!task || !task->state || !snapshot) return false;
(void)pthread_mutex_unlock(&task->mutex); Lardon3DTaskSnapshot generic;
return false; if (!lardon3d_task_queue_get(task->state->task_queue, task->task_id,
} &generic)) return false;
task->worker_state = *state; *snapshot = (Lardon3DImportTaskSnapshot) {
task->cancel_requested = false; .status = generic.state == TASK_COMPLETED ? LARDON3D_IMPORT_TASK_SUCCEEDED
task->joined = false; : generic.state == TASK_CANCELLED ? LARDON3D_IMPORT_TASK_CANCELLED
task->snapshot = (Lardon3DImportTaskSnapshot) { : generic.state == TASK_FAILED ? LARDON3D_IMPORT_TASK_FAILED
.status = LARDON3D_IMPORT_TASK_RUNNING, : LARDON3D_IMPORT_TASK_RUNNING,
.message = "Analyse du dossier source...", .total = 100,
.processed = generic.progress,
}; };
task->thread_started = true; (void)snprintf(snapshot->message, sizeof(snapshot->message), "%s",
int error = pthread_create(&task->thread, NULL, run_import, task); generic.message);
if (error != 0) {
task->thread_started = false;
task->snapshot.status = LARDON3D_IMPORT_TASK_FAILED;
copy_message(
task->snapshot.message,
"Erreur : impossible de lancer la tâche d'import."
);
(void)pthread_mutex_unlock(&task->mutex);
return false;
}
(void)pthread_mutex_unlock(&task->mutex);
return true;
}
void
lardon3d_import_task_request_cancel(Lardon3DImportTask *task)
{
if (!task) {
return;
}
(void)pthread_mutex_lock(&task->mutex);
if (task->snapshot.status == LARDON3D_IMPORT_TASK_RUNNING) {
task->cancel_requested = true;
copy_message(task->snapshot.message, "Annulation demandée...");
}
(void)pthread_mutex_unlock(&task->mutex);
}
bool
lardon3d_import_task_snapshot(
Lardon3DImportTask *task,
Lardon3DImportTaskSnapshot *snapshot
)
{
if (!task || !snapshot) {
return false;
}
(void)pthread_mutex_lock(&task->mutex);
*snapshot = task->snapshot;
(void)pthread_mutex_unlock(&task->mutex);
return true;
}
bool
lardon3d_import_task_is_finished(Lardon3DImportTask *task)
{
Lardon3DImportTaskSnapshot snapshot;
if (!lardon3d_import_task_snapshot(task, &snapshot)) {
return false;
}
return snapshot.status == LARDON3D_IMPORT_TASK_SUCCEEDED
|| snapshot.status == LARDON3D_IMPORT_TASK_CANCELLED
|| snapshot.status == LARDON3D_IMPORT_TASK_FAILED;
}
bool
lardon3d_import_task_join(Lardon3DImportTask *task)
{
if (!task) {
return false;
}
(void)pthread_mutex_lock(&task->mutex);
bool started = task->thread_started;
bool joined = task->joined;
bool should_join = started && !joined;
pthread_t thread = task->thread;
(void)pthread_mutex_unlock(&task->mutex);
if (!should_join) {
return started && joined;
}
if (pthread_join(thread, NULL) != 0) {
return false;
}
(void)pthread_mutex_lock(&task->mutex);
task->joined = true;
(void)pthread_mutex_unlock(&task->mutex);
return true; return true;
} }
bool lardon3d_import_task_is_finished(Lardon3DImportTask *task)
{ Lardon3DImportTaskSnapshot snapshot; return lardon3d_import_task_snapshot(task, &snapshot)
&& snapshot.status != LARDON3D_IMPORT_TASK_RUNNING; }
bool lardon3d_import_task_join(Lardon3DImportTask *task)
{ return task && lardon3d_import_task_is_finished(task); }
void void
lardon3d_import_task_destroy(Lardon3DImportTask *task) lardon3d_import_task_destroy(Lardon3DImportTask *task)
{ {
if (!task) { if (task && lardon3d_import_task_is_finished(task)) {
return; (void)lardon3d_task_queue_remove(task->state->task_queue, task->task_id);
} }
if (task->thread_started && !task->joined) {
lardon3d_import_task_request_cancel(task);
if (!lardon3d_import_task_join(task)) {
return;
}
}
(void)pthread_mutex_destroy(&task->mutex);
free(task); free(task);
} }

View file

@ -848,10 +848,11 @@ checkpoint_paths(
&& join_path(absolute, PATH_MAX, state->project_path, relative); && join_path(absolute, PATH_MAX, state->project_path, relative);
} }
Lardon3DProjectTaskCheckpointResult static Lardon3DProjectTaskCheckpointResult
lardon3d_project_checkpoint_task( checkpoint_task_internal(
Lardon3DAppState *state, Lardon3DAppState *state,
const Lardon3DTask *task const Lardon3DTask *task,
const char *image_import_source
) )
{ {
if (!state || !state->project_loaded || !state->project_db) { if (!state || !state->project_loaded || !state->project_db) {
@ -894,14 +895,13 @@ lardon3d_project_checkpoint_task(
.updated_at = now.tv_sec, .updated_at = now.tv_sec,
}; };
(void)snprintf(checkpoint.path, sizeof(checkpoint.path), "%s", relative); (void)snprintf(checkpoint.path, sizeof(checkpoint.path), "%s", relative);
Lardon3DProjectDbResult recorded = lardon3d_project_db_record_task( Lardon3DProjectDbResult recorded = image_import_source
state->project_db, ? lardon3d_project_db_record_image_import_task(
&snapshot, state->project_db, &snapshot, task_kind, task_kind_version,
task_kind, &checkpoint, image_import_source, now.tv_sec)
task_kind_version, : lardon3d_project_db_record_task(
&checkpoint, state->project_db, &snapshot, task_kind, task_kind_version,
now.tv_sec &checkpoint, now.tv_sec);
);
if (recorded == LARDON3D_PROJECT_DB_BUSY) { if (recorded == LARDON3D_PROJECT_DB_BUSY) {
return LARDON3D_PROJECT_TASK_CHECKPOINT_DB_BUSY; return LARDON3D_PROJECT_TASK_CHECKPOINT_DB_BUSY;
} }
@ -913,6 +913,28 @@ lardon3d_project_checkpoint_task(
: LARDON3D_PROJECT_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE; : LARDON3D_PROJECT_TASK_CHECKPOINT_PUBLISHED_NOT_DURABLE;
} }
Lardon3DProjectTaskCheckpointResult
lardon3d_project_checkpoint_task(
Lardon3DAppState *state,
const Lardon3DTask *task
)
{
return checkpoint_task_internal(state, task, NULL);
}
Lardon3DProjectTaskCheckpointResult
lardon3d_project_checkpoint_image_import_task(
Lardon3DAppState *state,
const Lardon3DTask *task,
const char *source_path
)
{
if (!source_path || !source_path[0]) {
return LARDON3D_PROJECT_TASK_CHECKPOINT_INVALID_TASK;
}
return checkpoint_task_internal(state, task, source_path);
}
static bool static bool
coherent_recovery( coherent_recovery(
const Lardon3DProjectDbTask *database_task, const Lardon3DProjectDbTask *database_task,

View file

@ -17,9 +17,10 @@ struct Lardon3DProjectDb {
char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY]; char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY];
}; };
static const char schema_v2[] = static const char schema_v3[] =
"CREATE TABLE metadata(key TEXT PRIMARY KEY,value INTEGER NOT NULL);" "CREATE TABLE metadata(key TEXT PRIMARY KEY,value INTEGER NOT NULL);"
"INSERT INTO metadata(key,value) VALUES('schema_version',2);" "INSERT INTO metadata(key,value) VALUES('schema_version',3);"
"INSERT INTO metadata(key,value) VALUES('next_task_id',1);"
"CREATE TABLE project(singleton INTEGER PRIMARY KEY CHECK(singleton=1)," "CREATE TABLE project(singleton INTEGER PRIMARY KEY CHECK(singleton=1),"
"stable_id TEXT NOT NULL UNIQUE,name TEXT NOT NULL,created_at INTEGER NOT NULL,updated_at INTEGER NOT NULL);" "stable_id TEXT NOT NULL UNIQUE,name TEXT NOT NULL,created_at INTEGER NOT NULL,updated_at INTEGER NOT NULL);"
"CREATE TABLE tasks(task_id INTEGER PRIMARY KEY CHECK(task_id>0),name TEXT NOT NULL," "CREATE TABLE tasks(task_id INTEGER PRIMARY KEY CHECK(task_id>0),name TEXT NOT NULL,"
@ -38,7 +39,9 @@ static const char schema_v2[] =
"state INTEGER NOT NULL CHECK(state BETWEEN 0 AND 1),size_bytes INTEGER NOT NULL CHECK(size_bytes>=0)," "state INTEGER NOT NULL CHECK(state BETWEEN 0 AND 1),size_bytes INTEGER NOT NULL CHECK(size_bytes>=0),"
"producer_task_id INTEGER REFERENCES tasks(task_id),created_at INTEGER NOT NULL,updated_at INTEGER NOT NULL);" "producer_task_id INTEGER REFERENCES tasks(task_id),created_at INTEGER NOT NULL,updated_at INTEGER NOT NULL);"
"CREATE INDEX artifacts_state_idx ON artifacts(state,artifact_id);" "CREATE INDEX artifacts_state_idx ON artifacts(state,artifact_id);"
"CREATE INDEX artifacts_producer_idx ON artifacts(producer_task_id);"; "CREATE INDEX artifacts_producer_idx ON artifacts(producer_task_id);"
"CREATE TABLE image_import_tasks(task_id INTEGER PRIMARY KEY REFERENCES tasks(task_id) ON DELETE CASCADE,"
"source_path TEXT NOT NULL);";
static void static void
copy_error(char destination[LARDON3D_PROJECT_DB_ERROR_CAPACITY], const char *text) copy_error(char destination[LARDON3D_PROJECT_DB_ERROR_CAPACITY], const char *text)
@ -127,12 +130,12 @@ migrate(Lardon3DProjectDb *database, unsigned int from_version)
if (from_version == LARDON3D_PROJECT_DB_SCHEMA_VERSION) { if (from_version == LARDON3D_PROJECT_DB_SCHEMA_VERSION) {
return LARDON3D_PROJECT_DB_OK; return LARDON3D_PROJECT_DB_OK;
} }
if (from_version != 0 && from_version != 1) { if (from_version != 0 && from_version != 1 && from_version != 2) {
return LARDON3D_PROJECT_DB_CORRUPT; return LARDON3D_PROJECT_DB_CORRUPT;
} }
Lardon3DProjectDbResult result = execute(database, "BEGIN IMMEDIATE", "begin migration"); Lardon3DProjectDbResult result = execute(database, "BEGIN IMMEDIATE", "begin migration");
if (result == LARDON3D_PROJECT_DB_OK && from_version == 0) { if (result == LARDON3D_PROJECT_DB_OK && from_version == 0) {
result = execute(database, schema_v2, "create schema v2"); result = execute(database, schema_v3, "create schema v3");
} }
if (result == LARDON3D_PROJECT_DB_OK && from_version == 1) { if (result == LARDON3D_PROJECT_DB_OK && from_version == 1) {
result = execute(database, result = execute(database,
@ -153,6 +156,25 @@ migrate(Lardon3DProjectDb *database, unsigned int from_version)
"finish schema v2 migration"); "finish schema v2 migration");
} }
} }
if (result == LARDON3D_PROJECT_DB_OK && from_version != 0) {
result = execute(database,
"CREATE TABLE image_import_tasks(task_id INTEGER PRIMARY KEY REFERENCES tasks(task_id) ON DELETE CASCADE,source_path TEXT NOT NULL);"
"INSERT INTO metadata(key,value) VALUES('next_task_id',(SELECT CASE WHEN COALESCE(MAX(task_id),0)>=9223372036854775807 THEN 0 ELSE COALESCE(MAX(task_id),0)+1 END FROM tasks))",
"migrate schema v2 to v3");
#ifdef LARDON3D_PROJECT_DB_TESTING
const char *forced_failure = getenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V3");
if (result == LARDON3D_PROJECT_DB_OK && forced_failure
&& strcmp(forced_failure, "1") == 0) {
result = execute(database, "INSERT INTO missing_test_table VALUES(1)",
"forced migration failure");
}
#endif
if (result == LARDON3D_PROJECT_DB_OK) {
result = execute(database,
"UPDATE metadata SET value=3 WHERE key='schema_version' AND value=2",
"finish schema v3 migration");
}
}
if (result == LARDON3D_PROJECT_DB_OK) result = execute(database, "COMMIT", "commit migration"); if (result == LARDON3D_PROJECT_DB_OK) result = execute(database, "COMMIT", "commit migration");
if (result != LARDON3D_PROJECT_DB_OK) (void)execute(database, "ROLLBACK", "rollback migration"); if (result != LARDON3D_PROJECT_DB_OK) (void)execute(database, "ROLLBACK", "rollback migration");
return result; return result;
@ -215,8 +237,8 @@ lardon3d_project_db_open(const char *path, Lardon3DProjectDb **output, char erro
if (result == LARDON3D_PROJECT_DB_OK) result = read_schema_version(database, &version); if (result == LARDON3D_PROJECT_DB_OK) result = read_schema_version(database, &version);
if (result == LARDON3D_PROJECT_DB_OK) result = migrate(database, version); if (result == LARDON3D_PROJECT_DB_OK) result = migrate(database, version);
if (result == LARDON3D_PROJECT_DB_OK) { if (result == LARDON3D_PROJECT_DB_OK) {
const char *required[] = {"project", "tasks", "checkpoints", "artifacts"}; const char *required[] = {"project", "tasks", "checkpoints", "artifacts", "image_import_tasks"};
for (size_t index = 0; index < 4 && result == LARDON3D_PROJECT_DB_OK; ++index) { for (size_t index = 0; index < 5 && result == LARDON3D_PROJECT_DB_OK; ++index) {
if (!table_exists(database->connection, required[index])) { if (!table_exists(database->connection, required[index])) {
copy_error(database->error, "Schéma v1 incomplet."); copy_error(database->error, "Schéma v1 incomplet.");
result = LARDON3D_PROJECT_DB_CORRUPT; result = LARDON3D_PROJECT_DB_CORRUPT;
@ -348,16 +370,19 @@ valid_durable_task(const Lardon3DTaskDurableSnapshot *snapshot, int64_t updated_
&& updated_at >= 0; && updated_at >= 0;
} }
Lardon3DProjectDbResult static Lardon3DProjectDbResult
lardon3d_project_db_record_task(Lardon3DProjectDb *database, const Lardon3DTaskDurableSnapshot *snapshot, record_task_internal(Lardon3DProjectDb *database, const Lardon3DTaskDurableSnapshot *snapshot,
const char *task_kind, uint32_t task_kind_version, const char *task_kind, uint32_t task_kind_version,
const Lardon3DProjectDbCheckpoint *checkpoint, int64_t updated_at) const Lardon3DProjectDbCheckpoint *checkpoint, const char *source_path,
int64_t updated_at)
{ {
bool typed = task_kind != NULL; bool typed = task_kind != NULL;
if (!database || !valid_durable_task(snapshot, updated_at) if (!database || !valid_durable_task(snapshot, updated_at)
|| (typed && (!lardon3d_task_kind_is_valid(task_kind) || (typed && (!lardon3d_task_kind_is_valid(task_kind)
|| task_kind_version == 0)) || task_kind_version == 0))
|| (!typed && task_kind_version != 0) || (!typed && task_kind_version != 0)
|| (source_path && !bounded_text(source_path,
LARDON3D_PROJECT_DB_PATH_CAPACITY, false))
|| (checkpoint && !valid_checkpoint(checkpoint))) return LARDON3D_PROJECT_DB_INVALID_ARGUMENT; || (checkpoint && !valid_checkpoint(checkpoint))) return LARDON3D_PROJECT_DB_INVALID_ARGUMENT;
#ifdef LARDON3D_PROJECT_DB_TESTING #ifdef LARDON3D_PROJECT_DB_TESTING
const char *forced_busy = getenv("LARDON3D_TEST_PROJECT_DB_BUSY_CHECKPOINT"); const char *forced_busy = getenv("LARDON3D_TEST_PROJECT_DB_BUSY_CHECKPOINT");
@ -402,6 +427,35 @@ lardon3d_project_db_record_task(Lardon3DProjectDb *database, const Lardon3DTaskD
} }
#endif #endif
} }
if (result == LARDON3D_PROJECT_DB_OK) {
result = prepare(database,
"UPDATE metadata SET value=CASE WHEN ?1=9223372036854775807 THEN 0 ELSE ?1+1 END "
"WHERE key='next_task_id' AND value>0 AND value<=?1",
&statement);
if (result == LARDON3D_PROJECT_DB_OK) {
(void)sqlite3_bind_int64(statement, 1,
(sqlite3_int64)snapshot->id);
result = step_done(database, statement, "advance recorded task id");
}
}
if (result == LARDON3D_PROJECT_DB_OK && source_path) {
result = prepare(database,
"INSERT INTO image_import_tasks(task_id,source_path) VALUES(?1,?2) "
"ON CONFLICT(task_id) DO UPDATE SET source_path=excluded.source_path "
"WHERE image_import_tasks.source_path=excluded.source_path",
&statement);
if (result == LARDON3D_PROJECT_DB_OK) {
(void)sqlite3_bind_int64(statement, 1, (sqlite3_int64)snapshot->id);
(void)sqlite3_bind_text(statement, 2, source_path, -1,
SQLITE_TRANSIENT);
result = step_done(database, statement, "upsert image import");
if (result == LARDON3D_PROJECT_DB_OK
&& sqlite3_changes(database->connection) != 1) {
copy_error(database->error, "Source d'import immuable.");
result = LARDON3D_PROJECT_DB_CONSTRAINT;
}
}
}
if (result == LARDON3D_PROJECT_DB_OK && checkpoint) { if (result == LARDON3D_PROJECT_DB_OK && checkpoint) {
result = prepare(database, "INSERT INTO checkpoints(task_id,path,format_version,durability,updated_at) VALUES(?1,?2,?3,?4,?5) " result = prepare(database, "INSERT INTO checkpoints(task_id,path,format_version,durability,updated_at) VALUES(?1,?2,?3,?4,?5) "
"ON CONFLICT(task_id) DO UPDATE SET path=excluded.path,format_version=excluded.format_version,durability=excluded.durability,updated_at=excluded.updated_at", &statement); "ON CONFLICT(task_id) DO UPDATE SET path=excluded.path,format_version=excluded.format_version,durability=excluded.durability,updated_at=excluded.updated_at", &statement);
@ -417,6 +471,36 @@ lardon3d_project_db_record_task(Lardon3DProjectDb *database, const Lardon3DTaskD
return result; return result;
} }
Lardon3DProjectDbResult
lardon3d_project_db_record_task(
Lardon3DProjectDb *database,
const Lardon3DTaskDurableSnapshot *snapshot,
const char *task_kind,
uint32_t task_kind_version,
const Lardon3DProjectDbCheckpoint *checkpoint,
int64_t updated_at
)
{
return record_task_internal(database, snapshot, task_kind,
task_kind_version, checkpoint, NULL, updated_at);
}
Lardon3DProjectDbResult
lardon3d_project_db_record_image_import_task(
Lardon3DProjectDb *database,
const Lardon3DTaskDurableSnapshot *snapshot,
const char *task_kind,
uint32_t task_kind_version,
const Lardon3DProjectDbCheckpoint *checkpoint,
const char *source_path,
int64_t updated_at
)
{
if (!source_path) return LARDON3D_PROJECT_DB_INVALID_ARGUMENT;
return record_task_internal(database, snapshot, task_kind,
task_kind_version, checkpoint, source_path, updated_at);
}
static bool static bool
read_task(sqlite3_stmt *statement, Lardon3DProjectDbTask *task) read_task(sqlite3_stmt *statement, Lardon3DProjectDbTask *task)
{ {
@ -493,6 +577,91 @@ lardon3d_project_db_list_recoverable(Lardon3DProjectDb *database, uint64_t after
(void)pthread_mutex_unlock(&database->mutex); return result; (void)pthread_mutex_unlock(&database->mutex); return result;
} }
Lardon3DProjectDbResult
lardon3d_project_db_allocate_task_id(
Lardon3DProjectDb *database,
uint64_t *task_id
)
{
if (task_id) *task_id = 0;
if (!database || !task_id) return LARDON3D_PROJECT_DB_INVALID_ARGUMENT;
(void)pthread_mutex_lock(&database->mutex);
Lardon3DProjectDbResult result = execute(database, "BEGIN IMMEDIATE",
"begin task id allocation");
sqlite3_stmt *statement = NULL;
sqlite3_int64 next = 0;
if (result == LARDON3D_PROJECT_DB_OK) {
result = prepare(database,
"SELECT value FROM metadata WHERE key='next_task_id'", &statement);
}
if (result == LARDON3D_PROJECT_DB_OK) {
int code = sqlite3_step(statement);
if (code != SQLITE_ROW || sqlite3_column_type(statement, 0) != SQLITE_INTEGER) {
result = LARDON3D_PROJECT_DB_CORRUPT;
} else {
next = sqlite3_column_int64(statement, 0);
if (next <= 0 || next > INT64_MAX) result = LARDON3D_PROJECT_DB_CONSTRAINT;
}
(void)sqlite3_finalize(statement); statement = NULL;
}
if (result == LARDON3D_PROJECT_DB_OK) {
result = prepare(database,
"UPDATE metadata SET value=CASE WHEN value=9223372036854775807 THEN 0 ELSE value+1 END WHERE key='next_task_id' AND value=?1",
&statement);
}
if (result == LARDON3D_PROJECT_DB_OK) {
(void)sqlite3_bind_int64(statement, 1, next);
result = step_done(database, statement, "advance task id");
statement = NULL;
if (result == LARDON3D_PROJECT_DB_OK
&& sqlite3_changes(database->connection) != 1) {
result = LARDON3D_PROJECT_DB_CONSTRAINT;
}
}
if (result == LARDON3D_PROJECT_DB_OK) {
result = execute(database, "COMMIT", "commit task id allocation");
}
if (result != LARDON3D_PROJECT_DB_OK) {
(void)execute(database, "ROLLBACK", "rollback task id allocation");
} else {
*task_id = (uint64_t)next;
}
(void)pthread_mutex_unlock(&database->mutex);
return result;
}
Lardon3DProjectDbResult
lardon3d_project_db_load_image_import(
Lardon3DProjectDb *database,
uint64_t task_id,
Lardon3DProjectDbImageImport *parameters
)
{
if (!database || !valid_task_id(task_id) || !parameters) {
return LARDON3D_PROJECT_DB_INVALID_ARGUMENT;
}
memset(parameters, 0, sizeof(*parameters));
(void)pthread_mutex_lock(&database->mutex);
sqlite3_stmt *statement = NULL;
Lardon3DProjectDbResult result = prepare(database,
"SELECT source_path FROM image_import_tasks WHERE task_id=?1",
&statement);
if (result == LARDON3D_PROJECT_DB_OK) {
(void)sqlite3_bind_int64(statement, 1, (sqlite3_int64)task_id);
int code = sqlite3_step(statement);
if (code == SQLITE_DONE) result = LARDON3D_PROJECT_DB_NOT_FOUND;
else if (code != SQLITE_ROW || !copy_column(statement, 0,
parameters->source_path, sizeof(parameters->source_path))) {
result = LARDON3D_PROJECT_DB_CORRUPT;
} else {
parameters->task_id = task_id;
}
(void)sqlite3_finalize(statement);
}
(void)pthread_mutex_unlock(&database->mutex);
return result;
}
Lardon3DProjectDbResult Lardon3DProjectDbResult
lardon3d_project_db_create_artifact(Lardon3DProjectDb *database, const Lardon3DProjectDbArtifact *artifact) lardon3d_project_db_create_artifact(Lardon3DProjectDb *database, const Lardon3DProjectDbArtifact *artifact)
{ {

View file

@ -180,6 +180,8 @@ record_batch_locked(
metrics[head] = (Lardon3DBatchMetrics) { metrics[head] = (Lardon3DBatchMetrics) {
.batch_size = batch_size, .batch_size = batch_size,
.duration_ns = duration_ns, .duration_ns = duration_ns,
/* Zéro est le marqueur persistant « mesure inconnue ». La boucle
* d'adaptation ignore explicitement ces échantillons. */
.peak_memory_bytes = peak_memory_bytes, .peak_memory_bytes = peak_memory_bytes,
}; };
head = (head + 1) % LARDON3D_BATCH_METRICS_CAPACITY; head = (head + 1) % LARDON3D_BATCH_METRICS_CAPACITY;

View file

@ -19,6 +19,10 @@ struct Lardon3DTask {
Lardon3DTaskCallback callback; Lardon3DTaskCallback callback;
void *userdata; void *userdata;
Lardon3DTaskUserdataDestroy userdata_destroy; Lardon3DTaskUserdataDestroy userdata_destroy;
Lardon3DTaskFinishedCallback finished_callback;
void *finished_userdata;
bool finished_notified;
bool finished_callback_running;
char task_kind[LARDON3D_TASK_KIND_CAPACITY]; char task_kind[LARDON3D_TASK_KIND_CAPACITY];
uint32_t task_kind_version; uint32_t task_kind_version;
Lardon3DResourceEstimate estimate; Lardon3DResourceEstimate estimate;
@ -98,6 +102,28 @@ finish_locked(
(void)pthread_cond_broadcast(&task->condition); (void)pthread_cond_broadcast(&task->condition);
} }
static void
notify_finished(Lardon3DTask *task)
{
Lardon3DTaskFinishedCallback callback = NULL;
void *userdata = NULL;
(void)pthread_mutex_lock(&task->mutex);
if (is_terminal(task->state) && !task->finished_notified) {
task->finished_notified = true;
callback = task->finished_callback;
userdata = task->finished_userdata;
task->finished_callback_running = callback != NULL;
}
(void)pthread_mutex_unlock(&task->mutex);
if (callback) {
callback(task, userdata);
(void)pthread_mutex_lock(&task->mutex);
task->finished_callback_running = false;
(void)pthread_cond_broadcast(&task->condition);
(void)pthread_mutex_unlock(&task->mutex);
}
}
Lardon3DTask * Lardon3DTask *
lardon3d_task_create( lardon3d_task_create(
const char *name, const char *name,
@ -218,6 +244,7 @@ lardon3d_task_start(
reservation_copy reservation_copy
); );
} }
notify_finished(task);
return true; return true;
} }
while (task->pause_requested && !task->cancel_requested) { while (task->pause_requested && !task->cancel_requested) {
@ -237,6 +264,7 @@ lardon3d_task_start(
reservation_copy reservation_copy
); );
} }
notify_finished(task);
return true; return true;
} }
task->state = TASK_RUNNING; task->state = TASK_RUNNING;
@ -257,33 +285,57 @@ lardon3d_task_start(
} else { } else {
finish_locked(task, TASK_COMPLETED, "Tâche terminée."); finish_locked(task, TASK_COMPLETED, "Tâche terminée.");
} }
Lardon3DResourceReservation *reservation_copy = task->current_reservation;
task->current_reservation = NULL;
(void)pthread_mutex_unlock(&task->mutex); (void)pthread_mutex_unlock(&task->mutex);
if (task->current_reservation) { if (reservation_copy) {
(void)lardon3d_resource_governor_release( (void)lardon3d_resource_governor_release(
governor, governor,
task->current_reservation reservation_copy
); );
task->current_reservation = NULL;
} }
notify_finished(task);
return true; return true;
} }
bool
lardon3d_task_set_finished_callback(
Lardon3DTask *task,
Lardon3DTaskFinishedCallback callback,
void *userdata
)
{
if (!task || !callback) return false;
(void)pthread_mutex_lock(&task->mutex);
bool accepted = !task->executing && task->state == TASK_PENDING
&& !task->finished_callback;
if (accepted) {
task->finished_callback = callback;
task->finished_userdata = userdata;
}
(void)pthread_mutex_unlock(&task->mutex);
return accepted;
}
void void
lardon3d_task_request_cancel(Lardon3DTask *task) lardon3d_task_request_cancel(Lardon3DTask *task)
{ {
if (!task) { if (!task) {
return; return;
} }
bool finished_here = false;
(void)pthread_mutex_lock(&task->mutex); (void)pthread_mutex_lock(&task->mutex);
if (!is_terminal(task->state)) { if (!is_terminal(task->state)) {
task->cancel_requested = true; task->cancel_requested = true;
copy_text(task->message, sizeof(task->message), "Annulation demandée."); copy_text(task->message, sizeof(task->message), "Annulation demandée.");
if (!task->executing) { if (!task->executing) {
finish_locked(task, TASK_CANCELLED, "Tâche annulée."); finish_locked(task, TASK_CANCELLED, "Tâche annulée.");
finished_here = true;
} }
(void)pthread_cond_broadcast(&task->condition); (void)pthread_cond_broadcast(&task->condition);
} }
(void)pthread_mutex_unlock(&task->mutex); (void)pthread_mutex_unlock(&task->mutex);
if (finished_here) notify_finished(task);
} }
bool bool
@ -332,7 +384,9 @@ lardon3d_task_join(Lardon3DTask *task)
return false; return false;
} }
(void)pthread_mutex_lock(&task->mutex); (void)pthread_mutex_lock(&task->mutex);
while (task->executing) { while (task->executing
|| (is_terminal(task->state) && task->finished_callback
&& (!task->finished_notified || task->finished_callback_running))) {
(void)pthread_cond_wait(&task->condition, &task->mutex); (void)pthread_cond_wait(&task->condition, &task->mutex);
} }
bool terminal = is_terminal(task->state); bool terminal = is_terminal(task->state);
@ -817,6 +871,7 @@ lardon3d_task_reject(Lardon3DTask *task, const char *message)
); );
} }
(void)pthread_mutex_unlock(&task->mutex); (void)pthread_mutex_unlock(&task->mutex);
if (accepted) notify_finished(task);
return accepted; return accepted;
} }

View file

@ -112,5 +112,12 @@ lardon3d_task_kind_registry_restore(
} }
return LARDON3D_TASK_KIND_RESTORE_FAILED; return LARDON3D_TASK_KIND_RESTORE_FAILED;
} }
if (binding.finished_callback
&& !lardon3d_task_set_finished_callback(*task,
binding.finished_callback, binding.finished_userdata)) {
lardon3d_task_destroy(*task);
*task = NULL;
return LARDON3D_TASK_KIND_RESTORE_FAILED;
}
return LARDON3D_TASK_KIND_OK; return LARDON3D_TASK_KIND_OK;
} }

17
src/task_kinds.c Normal file
View file

@ -0,0 +1,17 @@
#include <lardon3d/import_task.h>
#include <lardon3d/task_kind_registry.h>
const Lardon3DTaskKindRegistry *
lardon3d_task_kind_registry_production(void)
{
static const Lardon3DTaskKindDescriptor descriptors[] = {{
.kind = LARDON3D_IMAGE_IMPORT_TASK_KIND,
.kind_version = LARDON3D_IMAGE_IMPORT_TASK_KIND_VERSION,
.reconstruct = lardon3d_image_import_reconstruct,
}};
static const Lardon3DTaskKindRegistry registry = {
.descriptors = descriptors,
.count = sizeof(descriptors) / sizeof(descriptors[0]),
};
return &registry;
}

View file

@ -281,11 +281,25 @@ enqueue_locked(
uint64_t *task_id uint64_t *task_id
) )
{ {
if (queue->stopping || queue->next_id == 0 uint64_t id = lardon3d_task_id(task);
if (queue->stopping) {
return false;
}
for (TaskNode *existing = queue->all_head; existing;
existing = existing->next_all) {
if (id != 0 && lardon3d_task_id(existing->task) == id) {
return false;
}
}
if (id == 0) {
if (queue->next_id == 0
|| !lardon3d_task_assign_id(task, queue->next_id)) { || !lardon3d_task_assign_id(task, queue->next_id)) {
return false; return false;
} }
uint64_t id = queue->next_id++; id = queue->next_id++;
} else if (id >= queue->next_id) {
queue->next_id = id == UINT64_MAX ? 0 : id + 1;
}
node->task = task; node->task = task;
if (queue->all_tail) { if (queue->all_tail) {
queue->all_tail->next_all = node; queue->all_tail->next_all = node;

View file

@ -189,6 +189,9 @@ run_test(void)
CHECK(create_file(path, "one")); CHECK(create_file(path, "one"));
CHECK(join_path(path, valid_source, "two.PNG")); CHECK(join_path(path, valid_source, "two.PNG"));
CHECK(create_file(path, "two-two")); CHECK(create_file(path, "two-two"));
CHECK(create_directory(originals));
CHECK(join_path(path, originals, "one.jpg"));
CHECK(create_file(path, "one"));
Lardon3DAppState state; Lardon3DAppState state;
lardon3d_app_state_init(&state); lardon3d_app_state_init(&state);
@ -201,10 +204,14 @@ run_test(void)
) > 0); ) > 0);
Lardon3DImportResult result; Lardon3DImportResult result;
CHECK(lardon3d_import_directory(&state, valid_source, &result)); bool batch_complete = false;
CHECK(lardon3d_import_directory_batch(&state, valid_source, 8, &result,
NULL, &batch_complete) == LARDON3D_IMPORT_SUCCEEDED);
CHECK(batch_complete);
CHECK(result.admissible_found == 2); CHECK(result.admissible_found == 2);
CHECK(result.copied == 2); CHECK(result.newly_manifested == 2);
CHECK(result.already_present == 0); CHECK(result.copied == 1);
CHECK(result.already_present == 1);
char manifest[PATH_MAX]; char manifest[PATH_MAX];
CHECK(join_path(manifest, images, "manifest.tsv")); CHECK(join_path(manifest, images, "manifest.tsv"));
@ -234,6 +241,18 @@ run_test(void)
CHECK(strcmp(manifest_before, manifest_after) == 0); CHECK(strcmp(manifest_before, manifest_after) == 0);
CHECK(!has_manifest_temporary(images)); CHECK(!has_manifest_temporary(images));
char collision_source[PATH_MAX], collision_destination[PATH_MAX];
CHECK(join_path(collision_source, valid_source, "collision.jpg"));
CHECK(join_path(collision_destination, originals, "collision.jpg"));
CHECK(create_file(collision_source, "new-source"));
CHECK(create_file(collision_destination, "different-existing-output"));
batch_complete = false;
CHECK(lardon3d_import_directory_batch(&state, valid_source, 8, &result,
NULL, &batch_complete) == LARDON3D_IMPORT_FAILED);
CHECK(read_file(manifest, manifest_after, sizeof(manifest_after)));
CHECK(strcmp(manifest_before, manifest_after) == 0);
CHECK(unlink(collision_source) == 0 && unlink(collision_destination) == 0);
CHECK(join_path(path, valid_source, "three.tiff")); CHECK(join_path(path, valid_source, "three.tiff"));
CHECK(create_file(path, "three-three-three")); CHECK(create_file(path, "three-three-three"));
CHECK(lardon3d_import_directory(&state, valid_source, &result)); CHECK(lardon3d_import_directory(&state, valid_source, &result));

View file

@ -1,324 +1,175 @@
#include <dirent.h> #include <dirent.h>
#include <errno.h> #include <errno.h>
#include <fcntl.h> #include <fcntl.h>
#include <sched.h>
#include <stdbool.h> #include <stdbool.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/types.h>
#include <time.h>
#include <unistd.h> #include <unistd.h>
#include <lardon3d/app_state.h> #include <lardon3d/image_catalog.h>
#include <lardon3d/import_task.h> #include <lardon3d/import_task.h>
#include <lardon3d/project.h>
#include <lardon3d/task_queue.h>
#define CHECK(condition) \ #define CHECK(condition) do { if (!(condition)) { \
do { \ (void)fprintf(stderr, "Échec ligne %d : %s\n", __LINE__, #condition); return false; \
if (!(condition)) { \ } } while (0)
(void)fprintf(stderr, "Échec ligne %d : %s\n", __LINE__, #condition); \
return false; \
} \
} while (0)
static bool static bool
join_path(char destination[PATH_MAX], const char *parent, const char *child) join_path(char output[PATH_MAX], const char *parent, const char *child)
{ {
int written = snprintf(destination, PATH_MAX, "%s/%s", parent, child); int written = snprintf(output, PATH_MAX, "%s/%s", parent, child);
return written >= 0 && (size_t)written < PATH_MAX; return written > 0 && (size_t)written < PATH_MAX;
} }
static bool static bool
write_all(int descriptor, const void *data, size_t size) write_fixture(const char *path, unsigned int value)
{ {
const char *bytes = data; int descriptor = open(path, O_WRONLY | O_CREAT | O_EXCL, 0600);
size_t total = 0; if (descriptor < 0) return false;
while (total < size) { unsigned char data[64];
ssize_t written = write(descriptor, bytes + total, size - total); memset(data, (int)(value & 0xffU), sizeof(data));
if (written < 0 && errno == EINTR) { bool ok = write(descriptor, data, sizeof(data)) == (ssize_t)sizeof(data);
continue; return close(descriptor) == 0 && ok;
}
if (written <= 0) {
return false;
}
total += (size_t)written;
}
return true;
}
static bool
create_file(const char *path, size_t size)
{
int descriptor = open(path, O_WRONLY | O_CREAT | O_EXCL, 0644);
if (descriptor < 0) {
return false;
}
char block[64 * 1024];
(void)memset(block, 'L', sizeof(block));
bool success = true;
while (size > 0) {
size_t chunk = size < sizeof(block) ? size : sizeof(block);
if (!write_all(descriptor, block, chunk)) {
success = false;
break;
}
size -= chunk;
}
if (close(descriptor) != 0) {
success = false;
}
return success;
}
static bool
read_file(const char *path, char *content, size_t capacity)
{
int descriptor = open(path, O_RDONLY);
if (descriptor < 0 || capacity == 0) {
return false;
}
size_t total = 0;
while (total + 1 < capacity) {
ssize_t count = read(descriptor, content + total, capacity - total - 1);
if (count == 0) {
break;
}
if (count < 0 && errno == EINTR) {
continue;
}
if (count < 0) {
(void)close(descriptor);
return false;
}
total += (size_t)count;
}
content[total] = '\0';
return close(descriptor) == 0;
} }
static bool static bool
remove_tree(const char *path) remove_tree(const char *path)
{ {
struct stat info; struct stat info;
if (lstat(path, &info) != 0) { if (lstat(path, &info) != 0) return errno == ENOENT;
return errno == ENOENT; if (!S_ISDIR(info.st_mode)) return unlink(path) == 0;
}
if (!S_ISDIR(info.st_mode)) {
return unlink(path) == 0;
}
DIR *directory = opendir(path); DIR *directory = opendir(path);
if (!directory) { if (!directory) return false;
return false; bool ok = true;
} for (struct dirent *entry = readdir(directory); entry; entry = readdir(directory)) {
bool success = true; if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) continue;
for (struct dirent *entry = readdir(directory);
entry;
entry = readdir(directory)) {
if (strcmp(entry->d_name, ".") == 0
|| strcmp(entry->d_name, "..") == 0) {
continue;
}
char child[PATH_MAX]; char child[PATH_MAX];
if (!join_path(child, path, entry->d_name) || !remove_tree(child)) { if (!join_path(child, path, entry->d_name) || !remove_tree(child)) ok = false;
success = false;
} }
} if (closedir(directory) != 0 || rmdir(path) != 0) ok = false;
if (closedir(directory) != 0 || rmdir(path) != 0) { return ok;
success = false;
}
return success;
} }
static bool static bool
has_temporary_file(const char *images_path) setup_runtime(Lardon3DAppState *state)
{ {
DIR *directory = opendir(images_path); state->hardware_profile = (Lardon3DHardwareProfile) {
if (!directory) { .logical_cpu_count = 1024,
return true; .page_size_bytes = 4096,
} .memory_total_bytes = UINT64_MAX,
bool found = false; .cpu_architecture = "test",
for (struct dirent *entry = readdir(directory); };
entry; Lardon3DResourcePolicy policy = {
entry = readdir(directory)) { .maximum_cpu_load_ratio = 1.0,
if (strncmp(entry->d_name, ".manifest.tsv.tmp.", 18) == 0) { .maximum_io_pressure_avg10 = 100.0,
found = true; .io_slot_capacity = 1,
} };
} state->resource_governor = lardon3d_resource_governor_create(
if (closedir(directory) != 0) { &state->hardware_profile, &policy);
found = true; state->task_queue = state->resource_governor
} ? lardon3d_task_queue_create(state->resource_governor, 4) : NULL;
return found; return state->task_queue != NULL;
} }
static bool static bool
wait_until_finished( wait_for_state(Lardon3DTaskQueue *queue, uint64_t id,
Lardon3DImportTask *task, Lardon3DTaskState expected, Lardon3DTaskSnapshot *snapshot)
Lardon3DImportTaskSnapshot *snapshot
)
{ {
const struct timespec pause = {.tv_sec = 0, .tv_nsec = 1000000}; for (size_t attempt = 0; attempt < 1000000; ++attempt) {
for (size_t attempt = 0; attempt < 10000; ++attempt) { if (lardon3d_task_queue_get(queue, id, snapshot)
if (!lardon3d_import_task_snapshot(task, snapshot) && snapshot->state == expected) return true;
|| snapshot->processed > snapshot->total) { (void)sched_yield();
return false;
}
if (snapshot->status != LARDON3D_IMPORT_TASK_RUNNING) {
return true;
}
(void)nanosleep(&pause, NULL);
} }
return false; return false;
} }
static bool
run_success_tests(
Lardon3DAppState *state,
const char *source,
const char *manifest
)
{
Lardon3DImportTask *unused = lardon3d_import_task_create();
CHECK(unused);
CHECK(!lardon3d_import_task_start(NULL, state, source));
CHECK(!lardon3d_import_task_start(unused, NULL, source));
CHECK(!lardon3d_import_task_start(unused, state, NULL));
lardon3d_import_task_destroy(unused);
Lardon3DImportTask *task = lardon3d_import_task_create();
CHECK(task);
CHECK(lardon3d_import_task_start(task, state, source));
CHECK(!lardon3d_import_task_start(task, state, source));
Lardon3DImportTaskSnapshot snapshot;
CHECK(wait_until_finished(task, &snapshot));
CHECK(snapshot.status == LARDON3D_IMPORT_TASK_SUCCEEDED);
CHECK(snapshot.total == 2);
CHECK(snapshot.processed == 2);
CHECK(snapshot.copied == 2);
CHECK(snapshot.already_present == 0);
CHECK(lardon3d_import_task_join(task));
CHECK(lardon3d_import_task_join(task));
lardon3d_import_task_destroy(task);
CHECK(access(manifest, F_OK) == 0);
task = lardon3d_import_task_create();
CHECK(task && lardon3d_import_task_start(task, state, source));
CHECK(wait_until_finished(task, &snapshot));
CHECK(snapshot.status == LARDON3D_IMPORT_TASK_SUCCEEDED);
CHECK(snapshot.copied == 0);
CHECK(snapshot.already_present == 2);
CHECK(lardon3d_import_task_join(task));
lardon3d_import_task_destroy(task);
return true;
}
static bool
run_cancellation_test(
Lardon3DAppState *state,
const char *source,
const char *images,
const char *originals,
const char *manifest
)
{
char manifest_before[8192];
CHECK(read_file(manifest, manifest_before, sizeof(manifest_before)));
Lardon3DImportTask *task = lardon3d_import_task_create();
CHECK(task && lardon3d_import_task_start(task, state, source));
Lardon3DImportTaskSnapshot snapshot;
const struct timespec pause = {.tv_sec = 0, .tv_nsec = 1000000};
bool copy_observed = false;
for (size_t attempt = 0; attempt < 10000; ++attempt) {
CHECK(lardon3d_import_task_snapshot(task, &snapshot));
CHECK(snapshot.processed <= snapshot.total);
if (snapshot.status != LARDON3D_IMPORT_TASK_RUNNING) {
break;
}
if (snapshot.copied > 0) {
copy_observed = true;
break;
}
(void)nanosleep(&pause, NULL);
}
CHECK(copy_observed);
lardon3d_import_task_request_cancel(task);
CHECK(wait_until_finished(task, &snapshot));
CHECK(snapshot.status == LARDON3D_IMPORT_TASK_CANCELLED);
CHECK(lardon3d_import_task_join(task));
lardon3d_import_task_destroy(task);
char manifest_after[8192];
CHECK(read_file(manifest, manifest_after, sizeof(manifest_after)));
CHECK(strcmp(manifest_before, manifest_after) == 0);
CHECK(!has_temporary_file(images));
for (size_t index = 0; index < 8; ++index) {
char filename[64];
CHECK(snprintf(filename, sizeof(filename), "large-%zu.jpg", index) > 0);
char destination[PATH_MAX];
CHECK(join_path(destination, originals, filename));
CHECK(access(destination, F_OK) != 0);
}
return true;
}
static bool static bool
run_test(void) run_test(void)
{ {
char base[] = "/tmp/lardon3d-import-task-test.XXXXXX"; char root[] = "/tmp/lardon3d-import-generic-XXXXXX";
CHECK(mkdtemp(base)); CHECK(mkdtemp(root));
char project[PATH_MAX]; char source[PATH_MAX]; CHECK(join_path(source, root, "source"));
char images[PATH_MAX]; CHECK(mkdir(source, 0700) == 0);
char originals[PATH_MAX]; for (unsigned int index = 0; index < 80; ++index) {
char source[PATH_MAX]; char name[32], path[PATH_MAX];
char large_source[PATH_MAX]; CHECK(snprintf(name, sizeof(name), "image-%03u.jpg", index) > 0);
char manifest[PATH_MAX]; CHECK(join_path(path, source, name) && write_fixture(path, index));
CHECK(join_path(project, base, "project"));
CHECK(join_path(images, project, "images"));
CHECK(join_path(originals, images, "originals"));
CHECK(join_path(source, base, "source"));
CHECK(join_path(large_source, base, "large-source"));
CHECK(join_path(manifest, images, "manifest.tsv"));
CHECK(mkdir(project, 0755) == 0);
CHECK(mkdir(images, 0755) == 0);
CHECK(mkdir(source, 0755) == 0);
CHECK(mkdir(large_source, 0755) == 0);
char path[PATH_MAX];
CHECK(join_path(path, source, "one.jpg"));
CHECK(create_file(path, 3));
CHECK(join_path(path, source, "two.png"));
CHECK(create_file(path, 7));
for (size_t index = 0; index < 8; ++index) {
char filename[64];
CHECK(snprintf(filename, sizeof(filename), "large-%zu.jpg", index) > 0);
CHECK(join_path(path, large_source, filename));
CHECK(create_file(path, 16 * 1024 * 1024));
} }
CHECK(setenv("LARDON3D_PROJECTS_ROOT", root, 1) == 0);
Lardon3DAppState state; lardon3d_app_state_init(&state);
CHECK(setup_runtime(&state));
CHECK(lardon3d_project_create(&state, "Persistent Import"));
Lardon3DAppState state; CHECK(setenv("LARDON3D_TEST_IMPORT_PAUSE_AFTER_BATCH", "1", 1) == 0);
lardon3d_app_state_init(&state); CHECK(setenv("LARDON3D_TEST_IMPORT_SKIP_FINISHED_CHECKPOINT", "1", 1) == 0);
state.project_loaded = true; uint64_t task_id = 0;
CHECK(snprintf( CHECK(lardon3d_project_enqueue_image_import(&state, source, &task_id));
state.project_path, CHECK(task_id > 0 && task_id <= INT64_MAX);
sizeof(state.project_path), Lardon3DTaskSnapshot runtime;
"%s", CHECK(wait_for_state(state.task_queue, task_id, TASK_PAUSED, &runtime));
project CHECK(runtime.progress > 0 && runtime.progress < 100);
) > 0); lardon3d_task_queue_destroy(state.task_queue); state.task_queue = NULL;
CHECK(run_success_tests(&state, source, manifest)); lardon3d_project_close(&state);
CHECK(run_cancellation_test( CHECK(unsetenv("LARDON3D_TEST_IMPORT_PAUSE_AFTER_BATCH") == 0);
&state, CHECK(unsetenv("LARDON3D_TEST_IMPORT_SKIP_FINISHED_CHECKPOINT") == 0);
large_source,
images, state.task_queue = lardon3d_task_queue_create(state.resource_governor, 4);
originals, CHECK(state.task_queue && lardon3d_project_open(&state, "Persistent Import"));
manifest Lardon3DProjectRecoveryEntry entry; size_t count = 0;
)); const Lardon3DTaskKindRegistry *registry =
CHECK(remove_tree(base)); lardon3d_task_kind_registry_production();
CHECK(lardon3d_project_list_recoverable(&state, registry, 0, &entry, 1,
&count) == LARDON3D_PROJECT_DB_OK && count == 1);
CHECK(entry.task_id == task_id
&& strcmp(entry.task_kind, LARDON3D_IMAGE_IMPORT_TASK_KIND) == 0
&& entry.task_kind_version == LARDON3D_IMAGE_IMPORT_TASK_KIND_VERSION);
Lardon3DProjectDbImageImport persisted_parameters;
CHECK(lardon3d_project_db_load_image_import(state.project_db, task_id,
&persisted_parameters) == LARDON3D_PROJECT_DB_OK);
CHECK(strcmp(persisted_parameters.source_path, source) == 0);
Lardon3DImageImportReconstructionContext reconstruction = {
.project_path = state.project_path,
.project_db = state.project_db,
.resource_governor = state.resource_governor,
};
Lardon3DTask *restored = NULL;
char unavailable_source[PATH_MAX];
CHECK(join_path(unavailable_source, root, "source-unavailable"));
CHECK(rename(source, unavailable_source) == 0);
CHECK(lardon3d_task_kind_registry_restore(registry, entry.task_kind,
entry.task_kind_version, &entry.snapshot, &reconstruction, &restored)
== LARDON3D_TASK_KIND_RECONSTRUCTION_FAILED);
CHECK(!restored && rename(unavailable_source, source) == 0);
CHECK(lardon3d_task_kind_registry_restore(registry, entry.task_kind,
entry.task_kind_version, &entry.snapshot, &reconstruction, &restored)
== LARDON3D_TASK_KIND_OK);
CHECK(restored && lardon3d_task_id(restored) == task_id);
CHECK(lardon3d_task_queue_add(state.task_queue, restored, NULL));
CHECK(wait_for_state(state.task_queue, task_id, TASK_COMPLETED, &runtime));
CHECK(runtime.progress == 100);
char error[256];
Lardon3DImageCatalog *catalog = lardon3d_image_catalog_load(
&state, error, sizeof(error));
CHECK(catalog && lardon3d_image_catalog_count(catalog) == 80);
lardon3d_image_catalog_destroy(catalog);
lardon3d_task_queue_destroy(state.task_queue); state.task_queue = NULL;
lardon3d_project_close(&state);
state.task_queue = lardon3d_task_queue_create(state.resource_governor, 4);
CHECK(state.task_queue && lardon3d_project_open(&state, "Persistent Import"));
CHECK(lardon3d_project_list_recoverable(&state, registry, 0, &entry, 1,
&count) == LARDON3D_PROJECT_DB_OK && count == 0);
lardon3d_task_queue_destroy(state.task_queue); state.task_queue = NULL;
lardon3d_project_close(&state);
lardon3d_resource_governor_destroy(state.resource_governor);
CHECK(unsetenv("LARDON3D_PROJECTS_ROOT") == 0);
CHECK(remove_tree(root));
return true; return true;
} }
int int main(void) { return run_test() ? EXIT_SUCCESS : EXIT_FAILURE; }
main(void)
{
return run_test() ? EXIT_SUCCESS : EXIT_FAILURE;
}

View file

@ -35,6 +35,7 @@ task_snapshot(uint64_t id, Lardon3DTaskState saved)
} }
typedef struct { Lardon3DProjectDb *database; bool success; } ThreadContext; typedef struct { Lardon3DProjectDb *database; bool success; } ThreadContext;
typedef struct { Lardon3DProjectDb *database; uint64_t id; bool success; } IdThreadContext;
static void * static void *
read_thread(void *userdata) read_thread(void *userdata)
@ -51,13 +52,22 @@ read_thread(void *userdata)
return NULL; return NULL;
} }
static void *
allocate_id_thread(void *userdata)
{
IdThreadContext *context = userdata;
context->success = lardon3d_project_db_allocate_task_id(
context->database, &context->id) == LARDON3D_PROJECT_DB_OK;
return NULL;
}
static bool static bool
create_future_database(const char *path) create_future_database(const char *path)
{ {
sqlite3 *connection = NULL; sqlite3 *connection = NULL;
if (sqlite3_open(path, &connection) != SQLITE_OK) return false; if (sqlite3_open(path, &connection) != SQLITE_OK) return false;
bool ok = sqlite3_exec(connection, "CREATE TABLE metadata(key TEXT PRIMARY KEY,value INTEGER NOT NULL);" bool ok = sqlite3_exec(connection, "CREATE TABLE metadata(key TEXT PRIMARY KEY,value INTEGER NOT NULL);"
"INSERT INTO metadata VALUES('schema_version',3);", NULL, NULL, NULL) == SQLITE_OK; "INSERT INTO metadata VALUES('schema_version',4);", NULL, NULL, NULL) == SQLITE_OK;
return sqlite3_close(connection) == SQLITE_OK && ok; return sqlite3_close(connection) == SQLITE_OK && ok;
} }
@ -85,6 +95,21 @@ create_v1_database(const char *path)
return sqlite3_close(connection) == SQLITE_OK && ok; return sqlite3_close(connection) == SQLITE_OK && ok;
} }
static bool
create_v2_database(const char *path)
{
if (!create_v1_database(path)) return false;
sqlite3 *connection = NULL;
if (sqlite3_open(path, &connection) != SQLITE_OK) return false;
bool ok = sqlite3_exec(connection,
"ALTER TABLE tasks ADD COLUMN task_kind TEXT;"
"ALTER TABLE tasks ADD COLUMN task_kind_version INTEGER CHECK(task_kind_version IS NULL OR task_kind_version>0);"
"UPDATE tasks SET task_kind='test.work',task_kind_version=1 WHERE task_id=9;"
"UPDATE metadata SET value=2 WHERE key='schema_version'",
NULL, NULL, NULL) == SQLITE_OK;
return sqlite3_close(connection) == SQLITE_OK && ok;
}
static bool static bool
query_integer(const char *path, const char *sql, sqlite3_int64 expected) query_integer(const char *path, const char *sql, sqlite3_int64 expected)
{ {
@ -109,18 +134,22 @@ run_test(void)
char directory[] = "/tmp/lardon3d-project-db-XXXXXX"; char directory[] = "/tmp/lardon3d-project-db-XXXXXX";
CHECK(mkdtemp(directory)); CHECK(mkdtemp(directory));
char database_path[512], artifact_path[512], future_path[512], corrupt_path[512]; char database_path[512], artifact_path[512], future_path[512], corrupt_path[512];
char legacy_path[512], failed_migration_path[512]; char legacy_path[512], failed_migration_path[512], v2_path[512];
char failed_v3_migration_path[512];
CHECK(snprintf(database_path, sizeof(database_path), "%s/project.db", directory) > 0); CHECK(snprintf(database_path, sizeof(database_path), "%s/project.db", directory) > 0);
CHECK(snprintf(artifact_path, sizeof(artifact_path), "%s/artifact.bin", directory) > 0); CHECK(snprintf(artifact_path, sizeof(artifact_path), "%s/artifact.bin", directory) > 0);
CHECK(snprintf(future_path, sizeof(future_path), "%s/future.db", directory) > 0); CHECK(snprintf(future_path, sizeof(future_path), "%s/future.db", directory) > 0);
CHECK(snprintf(corrupt_path, sizeof(corrupt_path), "%s/corrupt.db", directory) > 0); CHECK(snprintf(corrupt_path, sizeof(corrupt_path), "%s/corrupt.db", directory) > 0);
CHECK(snprintf(legacy_path, sizeof(legacy_path), "%s/legacy.db", directory) > 0); CHECK(snprintf(legacy_path, sizeof(legacy_path), "%s/legacy.db", directory) > 0);
CHECK(snprintf(failed_migration_path, sizeof(failed_migration_path), "%s/failed-migration.db", directory) > 0); CHECK(snprintf(failed_migration_path, sizeof(failed_migration_path), "%s/failed-migration.db", directory) > 0);
CHECK(snprintf(v2_path, sizeof(v2_path), "%s/v2.db", directory) > 0);
CHECK(snprintf(failed_v3_migration_path, sizeof(failed_v3_migration_path),
"%s/failed-v3-migration.db", directory) > 0);
char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY]; char error[LARDON3D_PROJECT_DB_ERROR_CAPACITY];
Lardon3DProjectDb *database = NULL; Lardon3DProjectDb *database = NULL;
CHECK(lardon3d_project_db_open(database_path, &database, error) == LARDON3D_PROJECT_DB_OK); CHECK(lardon3d_project_db_open(database_path, &database, error) == LARDON3D_PROJECT_DB_OK);
CHECK(database && lardon3d_project_db_schema_version(database) == 2); CHECK(database && lardon3d_project_db_schema_version(database) == 3);
Lardon3DProjectDbProject project = {.created_at = 100, .updated_at = 100}; Lardon3DProjectDbProject project = {.created_at = 100, .updated_at = 100};
(void)snprintf(project.stable_id, sizeof(project.stable_id), "project-0001"); (void)snprintf(project.stable_id, sizeof(project.stable_id), "project-0001");
@ -161,6 +190,9 @@ run_test(void)
&checkpoint, 202) == LARDON3D_PROJECT_DB_INVALID_ARGUMENT); &checkpoint, 202) == LARDON3D_PROJECT_DB_INVALID_ARGUMENT);
CHECK(lardon3d_project_db_record_task(database, &running, "test.work", 0, CHECK(lardon3d_project_db_record_task(database, &running, "test.work", 0,
&checkpoint, 202) == LARDON3D_PROJECT_DB_INVALID_ARGUMENT); &checkpoint, 202) == LARDON3D_PROJECT_DB_INVALID_ARGUMENT);
CHECK(lardon3d_project_db_record_image_import_task(database, &running,
"import.images", 1, &checkpoint, "/tmp/source-a", 202)
== LARDON3D_PROJECT_DB_CONSTRAINT);
Lardon3DTaskDurableSnapshot completed = task_snapshot(2, TASK_COMPLETED); Lardon3DTaskDurableSnapshot completed = task_snapshot(2, TASK_COMPLETED);
CHECK(lardon3d_project_db_record_task(database, &completed, "test.work", 1, NULL, 202) == LARDON3D_PROJECT_DB_OK); CHECK(lardon3d_project_db_record_task(database, &completed, "test.work", 1, NULL, 202) == LARDON3D_PROJECT_DB_OK);
@ -172,6 +204,19 @@ run_test(void)
CHECK(lardon3d_project_db_list_recoverable(database, 1, page, 1, &count) == LARDON3D_PROJECT_DB_OK && count == 0); CHECK(lardon3d_project_db_list_recoverable(database, 1, page, 1, &count) == LARDON3D_PROJECT_DB_OK && count == 0);
CHECK(lardon3d_project_db_list_recoverable(database, 0, page, LARDON3D_PROJECT_DB_RECOVERY_PAGE_MAX + 1, &count) == LARDON3D_PROJECT_DB_INVALID_ARGUMENT); CHECK(lardon3d_project_db_list_recoverable(database, 0, page, LARDON3D_PROJECT_DB_RECOVERY_PAGE_MAX + 1, &count) == LARDON3D_PROJECT_DB_INVALID_ARGUMENT);
Lardon3DTaskDurableSnapshot image_import = task_snapshot(5, TASK_PENDING);
CHECK(lardon3d_project_db_record_image_import_task(database, &image_import,
"import.images", 1, &checkpoint, "/tmp/source-a", 202)
== LARDON3D_PROJECT_DB_OK);
Lardon3DProjectDbImageImport import_parameters;
CHECK(lardon3d_project_db_load_image_import(database, 5,
&import_parameters) == LARDON3D_PROJECT_DB_OK);
CHECK(import_parameters.task_id == 5
&& strcmp(import_parameters.source_path, "/tmp/source-a") == 0);
CHECK(lardon3d_project_db_record_image_import_task(database, &image_import,
"import.images", 1, &checkpoint, "/tmp/source-b", 203)
== LARDON3D_PROJECT_DB_CONSTRAINT);
Lardon3DTaskDurableSnapshot rollback_task = task_snapshot(3, TASK_PENDING); Lardon3DTaskDurableSnapshot rollback_task = task_snapshot(3, TASK_PENDING);
CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_CHECKPOINT", "1", 1) == 0); CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_CHECKPOINT", "1", 1) == 0);
CHECK(lardon3d_project_db_record_task(database, &rollback_task, "test.work", 1, &checkpoint, 203) == LARDON3D_PROJECT_DB_IO_ERROR); CHECK(lardon3d_project_db_record_task(database, &rollback_task, "test.work", 1, &checkpoint, 203) == LARDON3D_PROJECT_DB_IO_ERROR);
@ -206,11 +251,21 @@ run_test(void)
CHECK(pthread_create(&threads[1], NULL, read_thread, &contexts[1]) == 0); CHECK(pthread_create(&threads[1], NULL, read_thread, &contexts[1]) == 0);
CHECK(pthread_join(threads[0], NULL) == 0 && pthread_join(threads[1], NULL) == 0); CHECK(pthread_join(threads[0], NULL) == 0 && pthread_join(threads[1], NULL) == 0);
CHECK(contexts[0].success && contexts[1].success); CHECK(contexts[0].success && contexts[1].success);
IdThreadContext id_contexts[2] = {{.database = database}, {.database = database}};
CHECK(pthread_create(&threads[0], NULL, allocate_id_thread,
&id_contexts[0]) == 0);
CHECK(pthread_create(&threads[1], NULL, allocate_id_thread,
&id_contexts[1]) == 0);
CHECK(pthread_join(threads[0], NULL) == 0
&& pthread_join(threads[1], NULL) == 0);
CHECK(id_contexts[0].success && id_contexts[1].success
&& id_contexts[0].id != id_contexts[1].id
&& id_contexts[0].id > 5 && id_contexts[1].id > 5);
char too_long[LARDON3D_PROJECT_DB_PATH_CAPACITY + 1]; memset(too_long, 'x', sizeof(too_long)); too_long[sizeof(too_long) - 1] = '\0'; char too_long[LARDON3D_PROJECT_DB_PATH_CAPACITY + 1]; memset(too_long, 'x', sizeof(too_long)); too_long[sizeof(too_long) - 1] = '\0';
CHECK(lardon3d_project_db_open(too_long, &database, error) == LARDON3D_PROJECT_DB_INVALID_ARGUMENT); CHECK(lardon3d_project_db_open(too_long, &database, error) == LARDON3D_PROJECT_DB_INVALID_ARGUMENT);
lardon3d_project_db_close(contexts[0].database); database = NULL; lardon3d_project_db_close(contexts[0].database); database = NULL;
CHECK(query_integer(database_path, "SELECT value FROM metadata WHERE key='schema_version'", 2)); CHECK(query_integer(database_path, "SELECT value FROM metadata WHERE key='schema_version'", 3));
CHECK(query_integer(database_path, "SELECT count(*) FROM tasks WHERE task_id=1", 1)); CHECK(query_integer(database_path, "SELECT count(*) FROM tasks WHERE task_id=1", 1));
CHECK(lardon3d_project_db_open(database_path, &database, error) == LARDON3D_PROJECT_DB_OK); CHECK(lardon3d_project_db_open(database_path, &database, error) == LARDON3D_PROJECT_DB_OK);
CHECK(lardon3d_project_db_load_task(database, 1, &task) == LARDON3D_PROJECT_DB_OK); CHECK(lardon3d_project_db_load_task(database, 1, &task) == LARDON3D_PROJECT_DB_OK);
@ -225,7 +280,7 @@ run_test(void)
CHECK(create_v1_database(legacy_path)); CHECK(create_v1_database(legacy_path));
CHECK(lardon3d_project_db_open(legacy_path, &database, error) == LARDON3D_PROJECT_DB_OK); CHECK(lardon3d_project_db_open(legacy_path, &database, error) == LARDON3D_PROJECT_DB_OK);
CHECK(lardon3d_project_db_schema_version(database) == 2); CHECK(lardon3d_project_db_schema_version(database) == 3);
CHECK(lardon3d_project_db_get_project(database, &loaded_project) == LARDON3D_PROJECT_DB_OK CHECK(lardon3d_project_db_get_project(database, &loaded_project) == LARDON3D_PROJECT_DB_OK
&& strcmp(loaded_project.stable_id, "legacy-project") == 0); && strcmp(loaded_project.stable_id, "legacy-project") == 0);
CHECK(lardon3d_project_db_load_task(database, 9, &task) == LARDON3D_PROJECT_DB_OK); CHECK(lardon3d_project_db_load_task(database, 9, &task) == LARDON3D_PROJECT_DB_OK);
@ -234,7 +289,7 @@ run_test(void)
CHECK(lardon3d_project_db_load_artifact(database, "legacy-artifact", CHECK(lardon3d_project_db_load_artifact(database, "legacy-artifact",
&loaded_artifact) == LARDON3D_PROJECT_DB_OK); &loaded_artifact) == LARDON3D_PROJECT_DB_OK);
lardon3d_project_db_close(database); database = NULL; lardon3d_project_db_close(database); database = NULL;
CHECK(query_integer(legacy_path, "SELECT value FROM metadata WHERE key='schema_version'", 2)); CHECK(query_integer(legacy_path, "SELECT value FROM metadata WHERE key='schema_version'", 3));
CHECK(create_v1_database(failed_migration_path)); CHECK(create_v1_database(failed_migration_path));
CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V2", "1", 1) == 0); CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V2", "1", 1) == 0);
@ -249,8 +304,33 @@ run_test(void)
&& !task.has_task_kind); && !task.has_task_kind);
lardon3d_project_db_close(database); database = NULL; lardon3d_project_db_close(database); database = NULL;
CHECK(create_v2_database(v2_path));
CHECK(lardon3d_project_db_open(v2_path, &database, error)
== LARDON3D_PROJECT_DB_OK);
CHECK(lardon3d_project_db_load_task(database, 9, &task)
== LARDON3D_PROJECT_DB_OK && task.has_task_kind
&& strcmp(task.task_kind, "test.work") == 0);
CHECK(task.has_checkpoint
&& strcmp(task.checkpoint.path, "legacy.chk") == 0);
CHECK(lardon3d_project_db_load_artifact(database, "legacy-artifact",
&loaded_artifact) == LARDON3D_PROJECT_DB_OK);
lardon3d_project_db_close(database); database = NULL;
CHECK(query_integer(v2_path,
"SELECT value FROM metadata WHERE key='schema_version'", 3));
CHECK(create_v2_database(failed_v3_migration_path));
CHECK(setenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V3", "1", 1) == 0);
CHECK(lardon3d_project_db_open(failed_v3_migration_path, &database, error)
== LARDON3D_PROJECT_DB_IO_ERROR);
CHECK(unsetenv("LARDON3D_TEST_PROJECT_DB_FAIL_MIGRATION_V3") == 0);
CHECK(query_integer(failed_v3_migration_path,
"SELECT value FROM metadata WHERE key='schema_version'", 2));
CHECK(query_integer(failed_v3_migration_path,
"SELECT count(*) FROM sqlite_master WHERE type='table' AND name='image_import_tasks'", 0));
CHECK(unlink(artifact_path) == 0); CHECK(unlink(database_path) == 0); CHECK(unlink(future_path) == 0); CHECK(unlink(corrupt_path) == 0); CHECK(unlink(artifact_path) == 0); CHECK(unlink(database_path) == 0); CHECK(unlink(future_path) == 0); CHECK(unlink(corrupt_path) == 0);
CHECK(unlink(legacy_path) == 0); CHECK(unlink(failed_migration_path) == 0); CHECK(unlink(legacy_path) == 0); CHECK(unlink(failed_migration_path) == 0);
CHECK(unlink(v2_path) == 0); CHECK(unlink(failed_v3_migration_path) == 0);
CHECK(rmdir(directory) == 0); CHECK(rmdir(directory) == 0);
return true; return true;
} }

View file

@ -430,6 +430,26 @@ run_adaptive_batch_test(void)
CHECK(decision.kind == LARDON3D_RESOURCE_START); CHECK(decision.kind == LARDON3D_RESOURCE_START);
CHECK(decision.batch_size == 8); CHECK(decision.batch_size == 8);
/* Une mesure mémoire à zéro est inconnue : elle ne doit pas réduire un
* lot, même si sa taille et sa durée sont enregistrées. */
CHECK(lardon3d_resource_governor_record_batch(
governor,
LARDON3D_RESOURCE_TASK_IO,
8,
1000000000ULL,
0
));
CHECK(lardon3d_resource_governor_decide(governor, &snapshot,
&(Lardon3DResourceRequest) {
.memory_bytes_per_item = MEBIBYTES(100),
.minimum_batch_size = 2,
.preferred_batch_size = 8,
.requested_cpu_threads = 4,
.io_intensive = true,
}, &decision));
CHECK(decision.kind == LARDON3D_RESOURCE_START);
CHECK(decision.batch_size == 8);
/* Test 2: Record batch matching estimate → no reduction */ /* Test 2: Record batch matching estimate → no reduction */
CHECK(lardon3d_resource_governor_record_batch( CHECK(lardon3d_resource_governor_record_batch(
governor, governor,

View file

@ -26,6 +26,26 @@ typedef struct {
Lardon3DResourceReservation *reservation; Lardon3DResourceReservation *reservation;
} StartContext; } StartContext;
typedef struct {
Lardon3DResourceGovernor *governor;
size_t count;
Lardon3DTaskState state;
bool snapshot_succeeded;
bool reservation_released;
} FinishProbe;
static void
finished_callback(const Lardon3DTask *task, void *userdata)
{
FinishProbe *probe = userdata;
Lardon3DTaskSnapshot snapshot;
++probe->count;
probe->snapshot_succeeded = lardon3d_task_snapshot(task, &snapshot);
if (probe->snapshot_succeeded) probe->state = snapshot.state;
probe->reservation_released = !probe->governor
|| lardon3d_resource_governor_reservation_count(probe->governor) == 0;
}
static void static void
short_pause(long nanoseconds) short_pause(long nanoseconds)
{ {
@ -128,6 +148,9 @@ run_test(void)
&work &work
); );
CHECK(task); CHECK(task);
FinishProbe completed_probe = {.governor = governor};
CHECK(lardon3d_task_set_finished_callback(task, finished_callback,
&completed_probe));
CHECK(lardon3d_task_assign_id(task, 42)); CHECK(lardon3d_task_assign_id(task, 42));
CHECK(!lardon3d_task_assign_id(task, 43)); CHECK(!lardon3d_task_assign_id(task, 43));
CHECK(lardon3d_task_id(task) == 42); CHECK(lardon3d_task_id(task) == 42);
@ -146,6 +169,7 @@ run_test(void)
CHECK(wait_for_state(task, TASK_RUNNING)); CHECK(wait_for_state(task, TASK_RUNNING));
CHECK(lardon3d_task_pause(task)); CHECK(lardon3d_task_pause(task));
CHECK(wait_for_state(task, TASK_PAUSED)); CHECK(wait_for_state(task, TASK_PAUSED));
CHECK(completed_probe.count == 0);
CHECK(lardon3d_task_snapshot(task, &snapshot)); CHECK(lardon3d_task_snapshot(task, &snapshot));
unsigned int paused_progress = snapshot.progress; unsigned int paused_progress = snapshot.progress;
short_pause(5000000); short_pause(5000000);
@ -163,12 +187,18 @@ run_test(void)
CHECK(snapshot.progress == 100); CHECK(snapshot.progress == 100);
CHECK(snapshot.started_at.tv_sec > 0); CHECK(snapshot.started_at.tv_sec > 0);
CHECK(snapshot.finished_at.tv_sec > 0); CHECK(snapshot.finished_at.tv_sec > 0);
CHECK(completed_probe.count == 1 && completed_probe.snapshot_succeeded
&& completed_probe.state == TASK_COMPLETED
&& completed_probe.reservation_released);
CHECK(!lardon3d_task_start(task, NULL, NULL)); CHECK(!lardon3d_task_start(task, NULL, NULL));
lardon3d_task_destroy(task); lardon3d_task_destroy(task);
work = (Work) {.steps = 1000, .pause_ns = 1000000}; work = (Work) {.steps = 1000, .pause_ns = 1000000};
task = lardon3d_task_create("Annulation", &estimate, work_callback, &work); task = lardon3d_task_create("Annulation", &estimate, work_callback, &work);
CHECK(task); CHECK(task);
FinishProbe cancelled_probe = {.governor = governor};
CHECK(lardon3d_task_set_finished_callback(task, finished_callback,
&cancelled_probe));
CHECK(lardon3d_resource_governor_reserve( CHECK(lardon3d_resource_governor_reserve(
governor, &resource_snapshot, &estimate, &decision, &reservation governor, &resource_snapshot, &estimate, &decision, &reservation
)); ));
@ -182,10 +212,16 @@ run_test(void)
CHECK(lardon3d_task_snapshot(task, &snapshot)); CHECK(lardon3d_task_snapshot(task, &snapshot));
CHECK(snapshot.state == TASK_CANCELLED); CHECK(snapshot.state == TASK_CANCELLED);
CHECK(snapshot.progress < 100); CHECK(snapshot.progress < 100);
CHECK(cancelled_probe.count == 1 && cancelled_probe.snapshot_succeeded
&& cancelled_probe.state == TASK_CANCELLED
&& cancelled_probe.reservation_released);
lardon3d_task_destroy(task); lardon3d_task_destroy(task);
task = lardon3d_task_create("Échec", &estimate, failure_callback, NULL); task = lardon3d_task_create("Échec", &estimate, failure_callback, NULL);
CHECK(task); CHECK(task);
FinishProbe failed_probe = {.governor = governor};
CHECK(lardon3d_task_set_finished_callback(task, finished_callback,
&failed_probe));
CHECK(lardon3d_resource_governor_reserve( CHECK(lardon3d_resource_governor_reserve(
governor, &resource_snapshot, &estimate, &decision, &reservation governor, &resource_snapshot, &estimate, &decision, &reservation
)); ));
@ -194,6 +230,9 @@ run_test(void)
CHECK(lardon3d_task_snapshot(task, &snapshot)); CHECK(lardon3d_task_snapshot(task, &snapshot));
CHECK(snapshot.state == TASK_FAILED); CHECK(snapshot.state == TASK_FAILED);
CHECK(strcmp(snapshot.message, "Erreur contrôlée.") == 0); CHECK(strcmp(snapshot.message, "Erreur contrôlée.") == 0);
CHECK(failed_probe.count == 1 && failed_probe.snapshot_succeeded
&& failed_probe.state == TASK_FAILED
&& failed_probe.reservation_released);
lardon3d_task_destroy(task); lardon3d_task_destroy(task);
task = lardon3d_task_create( task = lardon3d_task_create(
@ -206,8 +245,14 @@ run_test(void)
CHECK(lardon3d_task_snapshot(task, &snapshot)); CHECK(lardon3d_task_snapshot(task, &snapshot));
CHECK(snapshot.state == TASK_PAUSED); CHECK(snapshot.state == TASK_PAUSED);
CHECK(lardon3d_task_resume(task)); CHECK(lardon3d_task_resume(task));
FinishProbe pending_cancel_probe = {0};
CHECK(lardon3d_task_set_finished_callback(task, finished_callback,
&pending_cancel_probe));
lardon3d_task_request_cancel(task);
lardon3d_task_request_cancel(task); lardon3d_task_request_cancel(task);
CHECK(lardon3d_task_join(task)); CHECK(lardon3d_task_join(task));
CHECK(pending_cancel_probe.count == 1
&& pending_cancel_probe.state == TASK_CANCELLED);
lardon3d_task_destroy(task); lardon3d_task_destroy(task);
lardon3d_resource_governor_destroy(governor); lardon3d_resource_governor_destroy(governor);
return true; return true;

View file

@ -12,11 +12,13 @@
typedef struct { typedef struct {
int *destroyed; int *destroyed;
int *finished;
uint64_t expected_id; uint64_t expected_id;
} TestUserdata; } TestUserdata;
typedef struct { typedef struct {
int *destroyed; int *destroyed;
int *finished;
bool fail; bool fail;
} ReconstructContext; } ReconstructContext;
@ -37,6 +39,17 @@ destroy_userdata(void *userdata)
} }
} }
static void
finished_callback(const Lardon3DTask *task, void *userdata)
{
TestUserdata *data = userdata;
Lardon3DTaskSnapshot terminal;
if (data && lardon3d_task_snapshot(task, &terminal)
&& terminal.state == TASK_COMPLETED && *data->destroyed == 0) {
++*data->finished;
}
}
static bool static bool
reconstruct( reconstruct(
const Lardon3DTaskDurableSnapshot *snapshot, const Lardon3DTaskDurableSnapshot *snapshot,
@ -51,11 +64,14 @@ reconstruct(
} }
*data = (TestUserdata) { *data = (TestUserdata) {
.destroyed = settings->destroyed, .destroyed = settings->destroyed,
.finished = settings->finished,
.expected_id = snapshot->id, .expected_id = snapshot->id,
}; };
binding->callback = test_callback; binding->callback = test_callback;
binding->userdata = data; binding->userdata = data;
binding->userdata_destroy = destroy_userdata; binding->userdata_destroy = destroy_userdata;
binding->finished_callback = finished_callback;
binding->finished_userdata = data;
return !settings->fail; return !settings->fail;
} }
@ -113,8 +129,11 @@ run_test(void)
CHECK(!lardon3d_task_kind_registry_init(&registry, duplicate, 2)); CHECK(!lardon3d_task_kind_registry_init(&registry, duplicate, 2));
CHECK(lardon3d_task_kind_registry_init(&registry, descriptors, 2)); CHECK(lardon3d_task_kind_registry_init(&registry, descriptors, 2));
int destroyed = 0; int destroyed = 0, finished = 0;
ReconstructContext context = {.destroyed = &destroyed}; ReconstructContext context = {
.destroyed = &destroyed,
.finished = &finished,
};
Lardon3DTaskDurableSnapshot durable = snapshot(); Lardon3DTaskDurableSnapshot durable = snapshot();
Lardon3DTask *task = NULL; Lardon3DTask *task = NULL;
CHECK(lardon3d_task_kind_registry_restore(&registry, "test.recovery", 1, CHECK(lardon3d_task_kind_registry_restore(&registry, "test.recovery", 1,
@ -149,9 +168,10 @@ run_test(void)
Lardon3DTaskSnapshot runtime; Lardon3DTaskSnapshot runtime;
CHECK(lardon3d_task_snapshot(task, &runtime) CHECK(lardon3d_task_snapshot(task, &runtime)
&& runtime.state == TASK_COMPLETED); && runtime.state == TASK_COMPLETED);
CHECK(finished == 1 && destroyed == 0);
lardon3d_task_destroy(task); lardon3d_task_destroy(task);
lardon3d_resource_governor_destroy(governor); lardon3d_resource_governor_destroy(governor);
CHECK(destroyed == 1); CHECK(destroyed == 1 && finished == 1);
context.fail = true; context.fail = true;
CHECK(lardon3d_task_kind_registry_restore(&registry, "test.recovery", 1, CHECK(lardon3d_task_kind_registry_restore(&registry, "test.recovery", 1,

View file

@ -223,6 +223,21 @@ run_test(void)
queue = lardon3d_task_queue_create(governor, 1024); queue = lardon3d_task_queue_create(governor, 1024);
CHECK(queue); CHECK(queue);
OrderLog restored_log = {0};
CHECK(pthread_mutex_init(&restored_log.mutex, NULL) == 0);
QueueWork restored_work = {.log = &restored_log, .value = 0, .steps = 1};
Lardon3DTask *restored = lardon3d_task_create(
"Restaurée", &estimate, queue_callback, &restored_work);
CHECK(restored && lardon3d_task_assign_id(restored, 100));
CHECK(lardon3d_task_queue_add(queue, restored, NULL));
Lardon3DTask *duplicate = lardon3d_task_create(
"Doublon", &estimate, queue_callback, &restored_work);
CHECK(duplicate && lardon3d_task_assign_id(duplicate, 100));
CHECK(!lardon3d_task_queue_add(queue, duplicate, NULL));
lardon3d_task_destroy(duplicate);
CHECK(wait_terminal(queue, 100, &snapshot));
CHECK(lardon3d_task_queue_remove(queue, 100));
CHECK(pthread_mutex_destroy(&restored_log.mutex) == 0);
OrderLog control_log = {0}; OrderLog control_log = {0};
CHECK(pthread_mutex_init(&control_log.mutex, NULL) == 0); CHECK(pthread_mutex_init(&control_log.mutex, NULL) == 0);
QueueWork slow = {.log = &control_log, .value = 1, .steps = 500}; QueueWork slow = {.log = &control_log, .value = 1, .steps = 500};