From 015841a20c895ca436acc86da6c763856fa75115 Mon Sep 17 00:00:00 2001 From: grayTerminal-sh Date: Tue, 28 Jul 2026 07:28:04 +0200 Subject: [PATCH] fix: borner les sorties des outils documentaires --- Makefile | 12 +- include/core/document_analysis.h | 4 + include/core/document_tool_runner.h | 17 ++ include/core/eml_pipeline_task.h | 10 + include/core/exiftool_analysis.h | 8 + include/core/ocr_analysis.h | 9 + src/core/document_tool_runner.c | 343 ++++++++++++++++++++-------- src/core/eml_pipeline_task.c | 40 +++- src/core/exiftool_analysis.c | 32 ++- src/core/ocr_analysis.c | 21 +- src/core/pdf_analysis.c | 37 ++- tests/fake_document_tool.c | 75 +++++- tests/test_document_tool_runner.c | 160 +++++++++++++ tests/test_eml_pipeline_task.c | 96 ++++++++ tests/test_exiftool_analysis.c | 50 ++++ tests/test_ocr_analysis.c | 24 ++ tests/test_pdf_analysis.c | 60 +++++ 17 files changed, 897 insertions(+), 101 deletions(-) create mode 100644 tests/test_document_tool_runner.c diff --git a/Makefile b/Makefile index 0263d3e..0d5d02e 100644 --- a/Makefile +++ b/Makefile @@ -139,6 +139,7 @@ TEST_EML_MIME_EXTRACTOR := tests/test_eml_mime_extractor TEST_EXIFTOOL_ANALYSIS := tests/test_exiftool_analysis TEST_OCR_ANALYSIS := tests/test_ocr_analysis TEST_PDF_ANALYSIS := tests/test_pdf_analysis +TEST_DOCUMENT_TOOL_RUNNER := tests/test_document_tool_runner FAKE_DOCUMENT_TOOL := tests/fake_document_tool DOCUMENT_ANALYSIS_TEST_SOURCES := \ @@ -147,6 +148,12 @@ DOCUMENT_ANALYSIS_TEST_SOURCES := \ src/core/tool_process.c \ src/core/file_hash.c +$(TEST_DOCUMENT_TOOL_RUNNER): tests/test_document_tool_runner.c \ + $(DOCUMENT_ANALYSIS_TEST_SOURCES) $(FAKE_DOCUMENT_TOOL) + $(CC) $(TEST_CFLAGS) -Wpedantic \ + tests/test_document_tool_runner.c \ + $(DOCUMENT_ANALYSIS_TEST_SOURCES) -o $@ $(TEST_LDFLAGS) + all: $(TARGET) $(TEST_BANK_PROPOSAL): \ @@ -888,7 +895,8 @@ test: \ $(TEST_EML_MIME_EXTRACTOR) \ $(TEST_EXIFTOOL_ANALYSIS) \ $(TEST_OCR_ANALYSIS) \ - $(TEST_PDF_ANALYSIS) + $(TEST_PDF_ANALYSIS) \ + $(TEST_DOCUMENT_TOOL_RUNNER) @echo "Exécution des tests..." @./$(TEST_NODE) @./$(TEST_TREE_MODEL) @@ -961,6 +969,7 @@ test: \ @$(TEST_EXIFTOOL_ANALYSIS) @$(TEST_OCR_ANALYSIS) @$(TEST_PDF_ANALYSIS) + @$(TEST_DOCUMENT_TOOL_RUNNER) @echo "Tous les tests sont valides." %.o: %.c @@ -1037,6 +1046,7 @@ clean: $(TEST_EXIFTOOL_ANALYSIS) \ $(TEST_OCR_ANALYSIS) \ $(TEST_PDF_ANALYSIS) \ + $(TEST_DOCUMENT_TOOL_RUNNER) \ $(FAKE_DOCUMENT_TOOL) diff --git a/include/core/document_analysis.h b/include/core/document_analysis.h index f0122dc..8f36b5c 100644 --- a/include/core/document_analysis.h +++ b/include/core/document_analysis.h @@ -38,6 +38,10 @@ typedef struct char *raw_stdout; char *raw_stdout_sha256; char *raw_stderr; + gsize stdout_bytes_observed; + gsize stderr_bytes_observed; + gboolean stdout_truncated; + gboolean stderr_truncated; int exit_status; DocumentAnalysisState state; GPtrArray *warnings; diff --git a/include/core/document_tool_runner.h b/include/core/document_tool_runner.h index 32769f4..02fd093 100644 --- a/include/core/document_tool_runner.h +++ b/include/core/document_tool_runner.h @@ -9,6 +9,12 @@ G_BEGIN_DECLS +typedef struct +{ + gsize stdout_limit; + gsize stderr_limit; +} DocumentToolRunnerLimits; + gboolean document_tool_runner_run( const char *tool_id, const char *executable, @@ -19,6 +25,17 @@ gboolean document_tool_runner_run( GError **error ); +gboolean document_tool_runner_run_with_limits( + const char *tool_id, + const char *executable, + const char *const arguments[], + const char *source_path, + const DocumentToolRunnerLimits *limits, + GCancellable *cancellable, + DocumentToolExecution **out_execution, + GError **error +); + char *document_tool_runner_read_version( const char *executable, const char *const arguments[], diff --git a/include/core/eml_pipeline_task.h b/include/core/eml_pipeline_task.h index 65c55cf..e28b3f3 100644 --- a/include/core/eml_pipeline_task.h +++ b/include/core/eml_pipeline_task.h @@ -21,6 +21,8 @@ typedef struct EmlPipelineResult GPtrArray *bank_proposals; /**< Tableau de BankProposal* */ GPtrArray *document_analyses; /**< Tableau de DocumentFileAnalysis* */ GPtrArray *warnings; /**< Avertissements globaux */ + DocumentAnalysisState state; + guint skipped_document_analyses; } EmlPipelineResult; /** @brief Libère un résultat EmlPipelineResult. */ @@ -44,6 +46,14 @@ BackgroundTask *eml_pipeline_task_new_with_tools( const DocumentAnalysisTools *tools ); +BackgroundTask *eml_pipeline_task_new_with_tools_and_limit( + const char *eml_path, + const char *processed_evidence_dir, + const char *evidence_id, + const DocumentAnalysisTools *tools, + guint document_analysis_limit +); + G_END_DECLS #endif /* LABFY_INVESTIGATION_EML_PIPELINE_TASK_H */ diff --git a/include/core/exiftool_analysis.h b/include/core/exiftool_analysis.h index 13cdc9f..8672391 100644 --- a/include/core/exiftool_analysis.h +++ b/include/core/exiftool_analysis.h @@ -6,6 +6,7 @@ #define LABFY_INVESTIGATION_EXIFTOOL_ANALYSIS_H #include "core/document_analysis.h" +#include "core/document_tool_runner.h" G_BEGIN_DECLS @@ -21,6 +22,13 @@ ExiftoolAnalysisResult *exiftool_analysis_run( GCancellable *cancellable, GError **error ); +ExiftoolAnalysisResult *exiftool_analysis_run_with_limits( + const char *executable, + const char *file_path, + const DocumentToolRunnerLimits *limits, + GCancellable *cancellable, + GError **error +); ExiftoolAnalysisResult *exiftool_analysis_parse( const char *file_path, const char *json, diff --git a/include/core/ocr_analysis.h b/include/core/ocr_analysis.h index 6ae19ec..e068ac4 100644 --- a/include/core/ocr_analysis.h +++ b/include/core/ocr_analysis.h @@ -6,6 +6,7 @@ #define LABFY_INVESTIGATION_OCR_ANALYSIS_H #include "core/document_analysis.h" +#include "core/document_tool_runner.h" G_BEGIN_DECLS @@ -23,6 +24,14 @@ OcrAnalysisResult *ocr_analysis_run( GCancellable *cancellable, GError **error ); +OcrAnalysisResult *ocr_analysis_run_with_limits( + const char *executable, + const char *image_path, + const char *languages, + const DocumentToolRunnerLimits *limits, + GCancellable *cancellable, + GError **error +); void ocr_analysis_result_free(OcrAnalysisResult *result); gboolean ocr_analysis_mime_is_compatible(const char *mime_type); diff --git a/src/core/document_tool_runner.c b/src/core/document_tool_runner.c index 6834ac9..d6d690f 100644 --- a/src/core/document_tool_runner.c +++ b/src/core/document_tool_runner.c @@ -1,90 +1,232 @@ /****************************************************************************** * @file document_tool_runner.c - * @brief Exécution bornée et annulable des outils documentaires. + * @brief Exécution réellement bornée et annulable des outils documentaires. ******************************************************************************/ #include "core/document_tool_runner.h" #include "core/file_hash.h" -#include "core/tool_process.h" + #include +#define DOCUMENT_TOOL_RUNNER_READ_BLOCK 4096U +#define DOCUMENT_TOOL_RUNNER_VERSION_LIMIT 65536U + +typedef struct +{ + GInputStream *stream; + GCancellable *cancellable; + GByteArray *prefix; + gsize limit; + gsize bytes_observed; + gboolean truncated; + GError *error; +} DocumentToolStreamCapture; + +typedef struct +{ + GBytes *stdout_bytes; + GBytes *stderr_bytes; + gsize stdout_bytes_observed; + gsize stderr_bytes_observed; + gboolean stdout_truncated; + gboolean stderr_truncated; + gboolean exited_normally; + int exit_status; +} DocumentToolCaptureResult; + +static GPtrArray *document_tool_runner_build_argv( + const char *executable, + const char *const arguments[] +) +{ + GPtrArray *argv = g_ptr_array_new(); + g_ptr_array_add(argv, (gpointer) executable); + for (gsize index = 0; arguments != NULL && + arguments[index] != NULL; index++) + g_ptr_array_add(argv, (gpointer) arguments[index]); + g_ptr_array_add(argv, NULL); + return argv; +} + +static gpointer document_tool_runner_drain_stream(gpointer user_data) +{ + DocumentToolStreamCapture *capture = user_data; + guint8 block[DOCUMENT_TOOL_RUNNER_READ_BLOCK]; + + while (TRUE) + { + gssize bytes_read = g_input_stream_read( + capture->stream, block, sizeof(block), + capture->cancellable, &capture->error); + if (bytes_read <= 0) + break; + + gsize observed = (gsize) bytes_read; + if (G_MAXSIZE - capture->bytes_observed < observed) + capture->bytes_observed = G_MAXSIZE; + else + capture->bytes_observed += observed; + + gsize remaining = capture->prefix->len < capture->limit + ? capture->limit - capture->prefix->len : 0; + gsize retained = MIN(remaining, observed); + if (retained > 0) + g_byte_array_append(capture->prefix, block, retained); + if (retained < observed) + capture->truncated = TRUE; + } + return NULL; +} + +static void document_tool_capture_result_clear( + DocumentToolCaptureResult *result +) +{ + g_clear_pointer(&result->stdout_bytes, g_bytes_unref); + g_clear_pointer(&result->stderr_bytes, g_bytes_unref); +} + +static gboolean document_tool_runner_capture( + const char *executable, + const char *const arguments[], + const DocumentToolRunnerLimits *limits, + GCancellable *cancellable, + DocumentToolCaptureResult *out_result, + GError **error +) +{ + GPtrArray *argv = document_tool_runner_build_argv(executable, arguments); + GSubprocessLauncher *launcher = g_subprocess_launcher_new( + G_SUBPROCESS_FLAGS_STDOUT_PIPE | G_SUBPROCESS_FLAGS_STDERR_PIPE); + GError *local_error = NULL; + GSubprocess *process = g_subprocess_launcher_spawnv( + launcher, (const char *const *) argv->pdata, &local_error); + g_object_unref(launcher); + g_ptr_array_unref(argv); + if (process == NULL) + { + g_propagate_error(error, local_error); + return FALSE; + } + + DocumentToolStreamCapture stdout_capture = { + .stream = g_subprocess_get_stdout_pipe(process), + .cancellable = cancellable, + .prefix = g_byte_array_sized_new(MIN(limits->stdout_limit, 4096U)), + .limit = limits->stdout_limit + }; + DocumentToolStreamCapture stderr_capture = { + .stream = g_subprocess_get_stderr_pipe(process), + .cancellable = cancellable, + .prefix = g_byte_array_sized_new(MIN(limits->stderr_limit, 4096U)), + .limit = limits->stderr_limit + }; + GThread *stdout_thread = g_thread_new( + "document-stdout", document_tool_runner_drain_stream, &stdout_capture); + GThread *stderr_thread = g_thread_new( + "document-stderr", document_tool_runner_drain_stream, &stderr_capture); + + gboolean waited = g_subprocess_wait(process, cancellable, &local_error); + if (!waited) + { + g_subprocess_force_exit(process); + GError *final_wait_error = NULL; + (void) g_subprocess_wait(process, NULL, &final_wait_error); + g_clear_error(&final_wait_error); + } + g_thread_join(stdout_thread); + g_thread_join(stderr_thread); + + gboolean cancelled = + (cancellable != NULL && g_cancellable_is_cancelled(cancellable)) || + g_error_matches(local_error, G_IO_ERROR, G_IO_ERROR_CANCELLED) || + g_error_matches(stdout_capture.error, G_IO_ERROR, G_IO_ERROR_CANCELLED) || + g_error_matches(stderr_capture.error, G_IO_ERROR, G_IO_ERROR_CANCELLED); + + if (!waited || stdout_capture.error != NULL || + stderr_capture.error != NULL) + { + if (cancelled) + g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_CANCELLED, + "L'exécution de l'outil documentaire a été annulée."); + else if (local_error != NULL) + g_propagate_error(error, g_steal_pointer(&local_error)); + else + g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_FAILED, + "La lecture des sorties de l'outil documentaire a échoué."); + g_clear_error(&local_error); + g_clear_error(&stdout_capture.error); + g_clear_error(&stderr_capture.error); + g_byte_array_unref(stdout_capture.prefix); + g_byte_array_unref(stderr_capture.prefix); + g_object_unref(process); + return FALSE; + } + + out_result->stdout_bytes = g_byte_array_free_to_bytes( + stdout_capture.prefix); + out_result->stderr_bytes = g_byte_array_free_to_bytes( + stderr_capture.prefix); + out_result->stdout_bytes_observed = stdout_capture.bytes_observed; + out_result->stderr_bytes_observed = stderr_capture.bytes_observed; + out_result->stdout_truncated = stdout_capture.truncated; + out_result->stderr_truncated = stderr_capture.truncated; + out_result->exited_normally = g_subprocess_get_if_exited(process); + out_result->exit_status = out_result->exited_normally + ? g_subprocess_get_exit_status(process) : -1; + g_clear_error(&local_error); + g_object_unref(process); + return TRUE; +} + +static char *document_tool_runner_bytes_to_text(GBytes *bytes) +{ + gsize length = 0; + const char *data = g_bytes_get_data(bytes, &length); + return g_utf8_make_valid(data != NULL ? data : "", (gssize) length); +} + char *document_tool_runner_read_version( const char *executable, const char *const arguments[], GCancellable *cancellable ) { - ToolProcessResult *result = NULL; + DocumentToolRunnerLimits limits = { + DOCUMENT_TOOL_RUNNER_VERSION_LIMIT, + DOCUMENT_TOOL_RUNNER_VERSION_LIMIT + }; + DocumentToolCaptureResult capture = { 0 }; GError *error = NULL; - char *version = NULL; - if (!tool_process_run(executable, arguments, NULL, cancellable, - &result, &error)) + if (!document_tool_runner_capture(executable, arguments, &limits, + cancellable, &capture, &error)) { g_clear_error(&error); return NULL; } - GBytes *stdout_bytes = tool_process_result_ref_stdout(result); - GBytes *stderr_bytes = tool_process_result_ref_stderr(result); - gsize stdout_length = 0; - gsize stderr_length = 0; - const char *stdout_data = g_bytes_get_data( - stdout_bytes, &stdout_length); - const char *stderr_data = g_bytes_get_data( - stderr_bytes, &stderr_length); - if (stdout_data == NULL) - stdout_data = ""; - if (stderr_data == NULL) - stderr_data = ""; - if (stdout_length > 0) - version = g_utf8_make_valid(stdout_data, (gssize) stdout_length); - else if (stderr_length > 0) - version = g_utf8_make_valid(stderr_data, (gssize) stderr_length); - if (version != NULL) - g_strstrip(version); - g_bytes_unref(stdout_bytes); - g_bytes_unref(stderr_bytes); - tool_process_result_free(result); + char *version = document_tool_runner_bytes_to_text( + g_bytes_get_size(capture.stdout_bytes) > 0 + ? capture.stdout_bytes : capture.stderr_bytes); + g_strstrip(version); + document_tool_capture_result_clear(&capture); return version; } -static char *document_tool_runner_bytes_to_text( - GBytes *bytes, - gsize limit, - gboolean *truncated -) -{ - gsize length = 0; - const char *data = bytes != NULL - ? g_bytes_get_data(bytes, &length) - : ""; - if (data == NULL) - data = ""; - if (length > limit) - { - length = limit; - *truncated = TRUE; - } - return g_utf8_make_valid(data, (gssize) length); -} - -gboolean document_tool_runner_run( +gboolean document_tool_runner_run_with_limits( const char *tool_id, const char *executable, const char *const arguments[], const char *source_path, + const DocumentToolRunnerLimits *limits, GCancellable *cancellable, DocumentToolExecution **out_execution, GError **error ) { - ToolProcessResult *process_result = NULL; - DocumentToolExecution *execution = NULL; - GError *process_error = NULL; - gboolean stdout_truncated = FALSE; - gboolean stderr_truncated = FALSE; - g_return_val_if_fail(error == NULL || *error == NULL, FALSE); if (tool_id == NULL || executable == NULL || source_path == NULL || - out_execution == NULL || *out_execution != NULL) + limits == NULL || limits->stdout_limit == 0 || + limits->stderr_limit == 0 || out_execution == NULL || + *out_execution != NULL) { g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT, "Les paramètres de l'outil documentaire sont invalides."); @@ -98,31 +240,34 @@ gboolean document_tool_runner_run( "Le fichier dépasse la taille maximale d'analyse."); return FALSE; } - execution = document_tool_execution_new(tool_id, source_path); + + DocumentToolExecution *execution = + document_tool_execution_new(tool_id, source_path); for (gsize index = 0; arguments != NULL && arguments[index] != NULL; index++) document_tool_execution_add_argument(execution, arguments[index]); (void) file_hash_compute_sha256(source_path, cancellable, &execution->source_sha256, NULL, NULL); - if (!tool_process_run(executable, arguments, NULL, cancellable, - &process_result, &process_error)) + DocumentToolCaptureResult capture = { 0 }; + GError *capture_error = NULL; + if (!document_tool_runner_capture(executable, arguments, limits, + cancellable, &capture, &capture_error)) { - if (g_error_matches(process_error, TOOL_PROCESS_ERROR, - TOOL_PROCESS_ERROR_CANCELLED)) + if (g_error_matches(capture_error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) { execution->state = DOCUMENT_ANALYSIS_STATE_CANCELLED; - g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_CANCELLED, - "L'analyse documentaire a été annulée."); + g_propagate_error(error, capture_error); + capture_error = NULL; } else { execution->state = DOCUMENT_ANALYSIS_STATE_UNAVAILABLE; - g_ptr_array_add(execution->errors, - g_strdup(process_error != NULL ? process_error->message : - "Outil indisponible.")); + g_ptr_array_add(execution->errors, g_strdup( + capture_error != NULL ? capture_error->message : + "Outil indisponible.")); } - g_clear_error(&process_error); + g_clear_error(&capture_error); GDateTime *now = g_date_time_new_now_utc(); execution->finished_at_utc = g_date_time_format_iso8601(now); g_date_time_unref(now); @@ -130,33 +275,53 @@ gboolean document_tool_runner_run( return execution->state != DOCUMENT_ANALYSIS_STATE_CANCELLED; } - GBytes *stdout_bytes = tool_process_result_ref_stdout(process_result); - GBytes *stderr_bytes = tool_process_result_ref_stderr(process_result); - execution->raw_stdout = document_tool_runner_bytes_to_text(stdout_bytes, - DOCUMENT_ANALYSIS_MAX_STDOUT, &stdout_truncated); - execution->raw_stderr = document_tool_runner_bytes_to_text(stderr_bytes, - DOCUMENT_ANALYSIS_MAX_STDERR, &stderr_truncated); - execution->exit_status = - tool_process_result_get_exit_status(process_result); - if (execution->raw_stdout != NULL) - execution->raw_stdout_sha256 = g_compute_checksum_for_string( - G_CHECKSUM_SHA256, execution->raw_stdout, -1); - if (stdout_truncated || stderr_truncated) - { - execution->state = DOCUMENT_ANALYSIS_STATE_PARTIAL; + execution->raw_stdout = + document_tool_runner_bytes_to_text(capture.stdout_bytes); + execution->raw_stderr = + document_tool_runner_bytes_to_text(capture.stderr_bytes); + execution->stdout_bytes_observed = capture.stdout_bytes_observed; + execution->stderr_bytes_observed = capture.stderr_bytes_observed; + execution->stdout_truncated = capture.stdout_truncated; + execution->stderr_truncated = capture.stderr_truncated; + execution->exit_status = capture.exit_status; + execution->raw_stdout_sha256 = g_compute_checksum_for_string( + G_CHECKSUM_SHA256, execution->raw_stdout, -1); + + if (execution->stdout_truncated) g_ptr_array_add(execution->warnings, - g_strdup("La sortie de l'outil a été tronquée à la limite.")); - } - else - execution->state = tool_process_result_is_success(process_result) - ? DOCUMENT_ANALYSIS_STATE_SUCCESS - : DOCUMENT_ANALYSIS_STATE_FAILED; + g_strdup("La sortie standard de l'outil a été tronquée.")); + if (execution->stderr_truncated) + g_ptr_array_add(execution->warnings, + g_strdup("La sortie d'erreur de l'outil a été tronquée.")); + execution->state = + execution->stdout_truncated || execution->stderr_truncated + ? DOCUMENT_ANALYSIS_STATE_PARTIAL + : capture.exited_normally && capture.exit_status == 0 + ? DOCUMENT_ANALYSIS_STATE_SUCCESS + : DOCUMENT_ANALYSIS_STATE_FAILED; + GDateTime *now = g_date_time_new_now_utc(); execution->finished_at_utc = g_date_time_format_iso8601(now); g_date_time_unref(now); - g_clear_pointer(&stdout_bytes, g_bytes_unref); - g_clear_pointer(&stderr_bytes, g_bytes_unref); - tool_process_result_free(process_result); + document_tool_capture_result_clear(&capture); *out_execution = execution; return TRUE; } + +gboolean document_tool_runner_run( + const char *tool_id, + const char *executable, + const char *const arguments[], + const char *source_path, + GCancellable *cancellable, + DocumentToolExecution **out_execution, + GError **error +) +{ + const DocumentToolRunnerLimits limits = { + DOCUMENT_ANALYSIS_MAX_STDOUT, + DOCUMENT_ANALYSIS_MAX_STDERR + }; + return document_tool_runner_run_with_limits(tool_id, executable, + arguments, source_path, &limits, cancellable, out_execution, error); +} diff --git a/src/core/eml_pipeline_task.c b/src/core/eml_pipeline_task.c index 02f8bfb..8d74357 100644 --- a/src/core/eml_pipeline_task.c +++ b/src/core/eml_pipeline_task.c @@ -18,6 +18,7 @@ typedef struct char *pdfinfo; char *pdftotext; char *pdftoppm; + guint document_analysis_limit; } EmlPipelineTaskData; static void eml_pipeline_task_data_free(gpointer user_data) @@ -116,18 +117,19 @@ static gboolean eml_pipeline_task_worker(BackgroundTask *task, GPtrArray *bank_proposals = g_ptr_array_new_with_free_func((GDestroyNotify) bank_proposal_free); GPtrArray *document_analyses = g_ptr_array_new_with_free_func( (GDestroyNotify) document_file_analysis_free); + GPtrArray *pipeline_warnings = + g_ptr_array_new_with_free_func(g_free); + guint skipped_document_analyses = 0; for (guint i = 0; mime_res->attachments != NULL && i < mime_res->attachments->len; i++) { - if (document_analyses->len >= - DOCUMENT_ANALYSIS_MAX_PIPELINE_ITEMS) - break; if (g_cancellable_is_cancelled(cancellable)) { eml_analysis_free(analysis); eml_mime_result_free(mime_res); g_ptr_array_unref(bank_proposals); g_ptr_array_unref(document_analyses); + g_ptr_array_unref(pipeline_warnings); g_set_error_literal( error, G_IO_ERROR, @@ -160,6 +162,11 @@ static gboolean eml_pipeline_task_worker(BackgroundTask *task, g_strcmp0(att->detected_mime, "application/pdf") == 0 || g_strcmp0(att->content_type, "application/pdf") == 0) { + if (document_analyses->len >= data->document_analysis_limit) + { + skipped_document_analyses++; + continue; + } DocumentAnalysisTools tools = { .exiftool = data->exiftool, .tesseract = data->tesseract, @@ -182,6 +189,7 @@ static gboolean eml_pipeline_task_worker(BackgroundTask *task, eml_mime_result_free(mime_res); g_ptr_array_unref(bank_proposals); g_ptr_array_unref(document_analyses); + g_ptr_array_unref(pipeline_warnings); return FALSE; } g_clear_error(&analysis_error); @@ -210,7 +218,16 @@ static gboolean eml_pipeline_task_worker(BackgroundTask *task, res->mime_result = mime_res; res->bank_proposals = bank_proposals; res->document_analyses = document_analyses; - res->warnings = g_ptr_array_new_with_free_func(g_free); + res->warnings = pipeline_warnings; + res->skipped_document_analyses = skipped_document_analyses; + res->state = skipped_document_analyses > 0 + ? DOCUMENT_ANALYSIS_STATE_PARTIAL + : DOCUMENT_ANALYSIS_STATE_SUCCESS; + if (skipped_document_analyses > 0) + g_ptr_array_add(res->warnings, g_strdup_printf( + "La limite d'analyses documentaires est atteinte : " + "%u fichier(s) n'ont pas été analysés.", + skipped_document_analyses)); if (out_result != NULL) *out_result = res; @@ -238,9 +255,23 @@ BackgroundTask *eml_pipeline_task_new_with_tools( const char *processed_evidence_dir, const char *evidence_id, const DocumentAnalysisTools *tools) +{ + return eml_pipeline_task_new_with_tools_and_limit( + eml_path, processed_evidence_dir, evidence_id, tools, + DOCUMENT_ANALYSIS_MAX_PIPELINE_ITEMS); +} + +BackgroundTask *eml_pipeline_task_new_with_tools_and_limit( + const char *eml_path, + const char *processed_evidence_dir, + const char *evidence_id, + const DocumentAnalysisTools *tools, + guint document_analysis_limit) { if (eml_path == NULL || processed_evidence_dir == NULL || tools == NULL) return NULL; + if (document_analysis_limit == 0) + return NULL; EmlPipelineTaskData *data = g_new0(EmlPipelineTaskData, 1); data->eml_path = g_strdup(eml_path); @@ -251,6 +282,7 @@ BackgroundTask *eml_pipeline_task_new_with_tools( data->pdfinfo = g_strdup(tools->pdfinfo); data->pdftotext = g_strdup(tools->pdftotext); data->pdftoppm = g_strdup(tools->pdftoppm); + data->document_analysis_limit = document_analysis_limit; BackgroundTask *task = background_task_new( "Analyse du message EML et de ses pièces jointes"); diff --git a/src/core/exiftool_analysis.c b/src/core/exiftool_analysis.c index 3c2e0af..7f5da12 100644 --- a/src/core/exiftool_analysis.c +++ b/src/core/exiftool_analysis.c @@ -216,12 +216,28 @@ ExiftoolAnalysisResult *exiftool_analysis_run( GCancellable *cancellable, GError **error ) +{ + const DocumentToolRunnerLimits limits = { + DOCUMENT_ANALYSIS_MAX_STDOUT, + DOCUMENT_ANALYSIS_MAX_STDERR + }; + return exiftool_analysis_run_with_limits( + executable, file_path, &limits, cancellable, error); +} + +ExiftoolAnalysisResult *exiftool_analysis_run_with_limits( + const char *executable, + const char *file_path, + const DocumentToolRunnerLimits *limits, + GCancellable *cancellable, + GError **error +) { const char *arguments[] = { "-j", "-G1", "-n", "--", file_path, NULL }; const char *version_arguments[] = { "-ver", NULL }; DocumentToolExecution *execution = NULL; - if (!document_tool_runner_run("exiftool", executable, arguments, - file_path, cancellable, &execution, error)) + if (!document_tool_runner_run_with_limits("exiftool", executable, + arguments, file_path, limits, cancellable, &execution, error)) { document_tool_execution_free(execution); return NULL; @@ -237,6 +253,18 @@ ExiftoolAnalysisResult *exiftool_analysis_run( } execution->version = document_tool_runner_read_version( executable, version_arguments, cancellable); + if (execution->stdout_truncated) + { + ExiftoolAnalysisResult *truncated = + g_new0(ExiftoolAnalysisResult, 1); + truncated->execution = execution; + truncated->metadata = g_ptr_array_new_with_free_func( + exiftool_metadata_entry_free); + execution->state = DOCUMENT_ANALYSIS_STATE_FAILED; + g_ptr_array_add(execution->errors, g_strdup( + "Le JSON ExifTool tronqué n'a pas été interprété.")); + return truncated; + } ExiftoolAnalysisResult *result = exiftool_analysis_parse(file_path, execution->raw_stdout != NULL ? execution->raw_stdout : "", execution->raw_stderr, execution->exit_status, error); diff --git a/src/core/ocr_analysis.c b/src/core/ocr_analysis.c index 1b48f5c..e6e5d18 100644 --- a/src/core/ocr_analysis.c +++ b/src/core/ocr_analysis.c @@ -29,6 +29,23 @@ OcrAnalysisResult *ocr_analysis_run( GCancellable *cancellable, GError **error ) +{ + const DocumentToolRunnerLimits limits = { + DOCUMENT_ANALYSIS_MAX_TEXT, + DOCUMENT_ANALYSIS_MAX_STDERR + }; + return ocr_analysis_run_with_limits(executable, image_path, languages, + &limits, cancellable, error); +} + +OcrAnalysisResult *ocr_analysis_run_with_limits( + const char *executable, + const char *image_path, + const char *languages, + const DocumentToolRunnerLimits *limits, + GCancellable *cancellable, + GError **error +) { if (executable == NULL || image_path == NULL || languages == NULL || languages[0] == '\0' || @@ -45,8 +62,8 @@ OcrAnalysisResult *ocr_analysis_run( }; const char *version_arguments[] = { "--version", NULL }; DocumentToolExecution *execution = NULL; - if (!document_tool_runner_run("tesseract", executable, arguments, - image_path, cancellable, &execution, error)) + if (!document_tool_runner_run_with_limits("tesseract", executable, + arguments, image_path, limits, cancellable, &execution, error)) { document_tool_execution_free(execution); return NULL; diff --git a/src/core/pdf_analysis.c b/src/core/pdf_analysis.c index 189a415..c277f50 100644 --- a/src/core/pdf_analysis.c +++ b/src/core/pdf_analysis.c @@ -139,6 +139,7 @@ static gboolean pdf_analysis_render_and_ocr( } gboolean success = TRUE; + char *source_basename = g_path_get_basename(result->source_path); for (guint page_number = 1; page_number <= pages; page_number++) { if (cancellable != NULL && @@ -150,8 +151,8 @@ static gboolean pdf_analysis_render_and_ocr( success = FALSE; break; } - char *prefix = g_strdup_printf("%s/page-%u", - temporary_directory, page_number); + char *prefix = g_strdup_printf("%s/%s-page-%u", + temporary_directory, source_basename, page_number); char *page_text = g_strdup_printf("%u", page_number); const char *render_arguments[] = { "-f", page_text, "-singlefile", "-png", @@ -162,6 +163,9 @@ static gboolean pdf_analysis_render_and_ocr( render_arguments, result->source_path, cancellable, &render_execution, error)) { + result->state = result->pages->len > 0 + ? DOCUMENT_ANALYSIS_STATE_PARTIAL + : DOCUMENT_ANALYSIS_STATE_CANCELLED; document_tool_execution_free(render_execution); g_free(page_text); g_free(prefix); @@ -198,12 +202,23 @@ static gboolean pdf_analysis_render_and_ocr( g_ptr_array_add(result->pages, page); if (page->state != DOCUMENT_ANALYSIS_STATE_SUCCESS) result->state = DOCUMENT_ANALYSIS_STATE_PARTIAL; + if (ocr == NULL && error != NULL && *error != NULL && + g_error_matches(*error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) + { + result->state = result->pages->len > 1 + ? DOCUMENT_ANALYSIS_STATE_PARTIAL + : DOCUMENT_ANALYSIS_STATE_CANCELLED; + success = FALSE; + } } g_remove(image_path); g_free(image_path); g_free(page_text); g_free(prefix); + if (!success) + break; } + g_free(source_basename); g_rmdir(temporary_directory); g_free(temporary_directory); return success; @@ -233,7 +248,16 @@ PdfAnalysisResult *pdf_analysis_run( if (!document_tool_runner_run("pdfinfo", tools->pdfinfo, info_arguments, pdf_path, cancellable, &result->pdfinfo_execution, error)) + { + if (error != NULL && *error != NULL && + g_error_matches(*error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) + { + result->state = DOCUMENT_ANALYSIS_STATE_CANCELLED; + g_clear_error(error); + return result; + } goto failure; + } if (result->pdfinfo_execution->state == DOCUMENT_ANALYSIS_STATE_UNAVAILABLE) { @@ -262,7 +286,16 @@ PdfAnalysisResult *pdf_analysis_run( if (!document_tool_runner_run("pdftotext", tools->pdftotext, text_arguments, pdf_path, cancellable, &result->native_execution, error)) + { + if (error != NULL && *error != NULL && + g_error_matches(*error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) + { + result->state = DOCUMENT_ANALYSIS_STATE_CANCELLED; + g_clear_error(error); + return result; + } goto failure; + } result->native_execution->version = document_tool_runner_read_version( tools->pdftotext, version_arguments, cancellable); diff --git a/tests/fake_document_tool.c b/tests/fake_document_tool.c index 6d769ab..8297dc2 100644 --- a/tests/fake_document_tool.c +++ b/tests/fake_document_tool.c @@ -5,6 +5,7 @@ #include #include #include +#include #include static int has_argument(int argc, char **argv, const char *value) @@ -23,8 +24,56 @@ static const char *find_pdf_path(int argc, char **argv) return ""; } +static long argument_long( + int argc, + char **argv, + const char *name, + long fallback +) +{ + for (int index = 1; index + 1 < argc; index++) + if (strcmp(argv[index], name) == 0) + return strtol(argv[index + 1], NULL, 10); + return fallback; +} + +static void write_repeated(FILE *stream, char value, long count) +{ + char block[1024]; + memset(block, value, sizeof(block)); + while (count > 0) + { + size_t amount = (size_t) (count > (long) sizeof(block) + ? (long) sizeof(block) : count); + if (fwrite(block, 1, amount, stream) != amount) + return; + fflush(stream); + count -= (long) amount; + } +} + int main(int argc, char **argv) { + if (has_argument(argc, argv, "--emit")) + { + long stdout_size = argument_long( + argc, argv, "--stdout-size", 0); + long stderr_size = argument_long( + argc, argv, "--stderr-size", 0); + long chunks = argument_long(argc, argv, "--chunks", 1); + int exit_status = (int) argument_long( + argc, argv, "--exit-status", 0); + if (chunks < 1) + chunks = 1; + for (long index = 0; index < chunks; index++) + { + write_repeated(stdout, 'O', stdout_size / chunks); + write_repeated(stderr, 'E', stderr_size / chunks); + if (has_argument(argc, argv, "--slow")) + (void) poll(NULL, 0, 20); + } + return exit_status; + } if (has_argument(argc, argv, "-ver")) { puts("13.00"); @@ -37,6 +86,14 @@ int main(int argc, char **argv) } if (has_argument(argc, argv, "-j")) { + if (strstr(argv[argc - 1], "slow") != NULL) + sleep(2); + if (strstr(argv[argc - 1], "large") != NULL) + { + fputs("[{\"File:MIMEType\":\"image/png\",\"Padding\":\"", stdout); + write_repeated(stdout, 'X', 8192); + return 0; + } puts("[{\"File:MIMEType\":\"image/png\"," "\"File:FileSize\":42,\"EXIF:ImageWidth\":10," "\"EXIF:GPSLatitude\":48.5,\"EXIF:GPSLongitude\":2.2," @@ -46,6 +103,8 @@ int main(int argc, char **argv) if (has_argument(argc, argv, "-enc")) { const char *path = find_pdf_path(argc, argv); + if (strstr(path, "slow-text") != NULL) + sleep(2); if (strstr(path, "native") != NULL) { puts("Texte synthétique suffisamment long pour être considéré " @@ -55,6 +114,12 @@ int main(int argc, char **argv) } if (has_argument(argc, argv, "-singlefile")) { + const char *pdf_path = find_pdf_path(argc, argv); + if (strstr(pdf_path, "slow-render") != NULL) + sleep(2); + if (strstr(pdf_path, "slow-page-2") != NULL && + strcmp(argv[2], "2") == 0) + sleep(2); const char *prefix = argv[argc - 1]; char path[4096]; if (snprintf(path, sizeof(path), "%s.png", prefix) < 0) @@ -68,8 +133,14 @@ int main(int argc, char **argv) } if (has_argument(argc, argv, "stdout")) { - if (strstr(argv[1], "sleep") != NULL) + if (strstr(argv[1], "sleep") != NULL || + strstr(argv[1], "slow-ocr") != NULL) sleep(2); + if (strstr(argv[1], "large") != NULL) + { + write_repeated(stdout, 'T', 8192); + return 0; + } if (strstr(argv[1], "page-2") != NULL) puts("Texte OCR synthétique page deux."); else @@ -77,6 +148,8 @@ int main(int argc, char **argv) return 0; } const char *path = find_pdf_path(argc, argv); + if (strstr(path, "slow-info") != NULL) + sleep(2); if (strstr(path, "encrypted") != NULL) puts("Pages: 2\nEncrypted: yes"); else diff --git a/tests/test_document_tool_runner.c b/tests/test_document_tool_runner.c new file mode 100644 index 0000000..59dfa02 --- /dev/null +++ b/tests/test_document_tool_runner.c @@ -0,0 +1,160 @@ +/****************************************************************************** + * @file test_document_tool_runner.c + * @brief Tests synthétiques de la capture documentaire bornée. + ******************************************************************************/ +#include "core/document_tool_runner.h" + +#include +#include +#include + +static char *create_source(char **out_directory) +{ + GError *error = NULL; + *out_directory = g_dir_make_tmp("labfy-runner-XXXXXX", &error); + g_assert_no_error(error); + char *path = g_build_filename(*out_directory, "source.bin", NULL); + g_assert_true(g_file_set_contents(path, "synthetic", -1, &error)); + g_assert_no_error(error); + return path; +} + +static void remove_source(char *directory, char *path) +{ + g_remove(path); + g_rmdir(directory); + g_free(path); + g_free(directory); +} + +static void test_limits_and_concurrent_drain(void) +{ + char *directory = NULL; + char *path = create_source(&directory); + const char *arguments[] = { + "--emit", "--stdout-size", "5000", "--stderr-size", "7000", + "--chunks", "20", "--exit-status", "7", NULL + }; + DocumentToolRunnerLimits limits = { 1000, 1500 }; + DocumentToolExecution *execution = NULL; + GError *error = NULL; + g_assert_true(document_tool_runner_run_with_limits( + "synthetic", "tests/fake_document_tool", arguments, path, + &limits, NULL, &execution, &error)); + g_assert_no_error(error); + g_assert_cmpuint(strlen(execution->raw_stdout), ==, 1000); + g_assert_cmpuint(strlen(execution->raw_stderr), ==, 1500); + g_assert_cmpuint(execution->stdout_bytes_observed, ==, 5000); + g_assert_cmpuint(execution->stderr_bytes_observed, ==, 7000); + g_assert_true(execution->stdout_truncated); + g_assert_true(execution->stderr_truncated); + g_assert_cmpint(execution->exit_status, ==, 7); + g_assert_cmpint(execution->state, ==, + DOCUMENT_ANALYSIS_STATE_PARTIAL); + g_assert_cmpint(execution->raw_stdout[0], ==, 'O'); + g_assert_cmpint(execution->raw_stderr[0], ==, 'E'); + document_tool_execution_free(execution); + remove_source(directory, path); +} + +static void test_exact_and_below_limits(void) +{ + char *directory = NULL; + char *path = create_source(&directory); + const char *arguments[] = { + "--emit", "--stdout-size", "1000", "--stderr-size", "12", NULL + }; + DocumentToolRunnerLimits limits = { 1000, 20 }; + DocumentToolExecution *execution = NULL; + g_assert_true(document_tool_runner_run_with_limits( + "synthetic", "tests/fake_document_tool", arguments, path, + &limits, NULL, &execution, NULL)); + g_assert_false(execution->stdout_truncated); + g_assert_false(execution->stderr_truncated); + g_assert_cmpint(execution->state, ==, + DOCUMENT_ANALYSIS_STATE_SUCCESS); + document_tool_execution_free(execution); + remove_source(directory, path); +} + +static gpointer cancel_later(gpointer user_data) +{ + g_usleep(60000); + g_cancellable_cancel(user_data); + return NULL; +} + +static void test_cancellation_during_output(void) +{ + char *directory = NULL; + char *path = create_source(&directory); + const char *arguments[] = { + "--emit", "--stdout-size", "100000", "--stderr-size", "100000", + "--chunks", "100", "--slow", NULL + }; + DocumentToolRunnerLimits limits = { 128, 128 }; + DocumentToolExecution *execution = NULL; + GCancellable *cancellable = g_cancellable_new(); + GThread *thread = g_thread_new("cancel-runner", cancel_later, cancellable); + GError *error = NULL; + g_assert_false(document_tool_runner_run_with_limits( + "synthetic", "tests/fake_document_tool", arguments, path, + &limits, cancellable, &execution, &error)); + g_thread_join(thread); + g_assert_error(error, G_IO_ERROR, G_IO_ERROR_CANCELLED); + g_assert_nonnull(execution); + g_assert_cmpint(execution->state, ==, + DOCUMENT_ANALYSIS_STATE_CANCELLED); + g_clear_error(&error); + document_tool_execution_free(execution); + g_object_unref(cancellable); + + const char *success_arguments[] = { + "--emit", "--stdout-size", "1", NULL + }; + execution = NULL; + g_assert_true(document_tool_runner_run_with_limits( + "synthetic", "tests/fake_document_tool", success_arguments, path, + &limits, NULL, &execution, NULL)); + document_tool_execution_free(execution); + remove_source(directory, path); +} + +static void test_repeated_runs_do_not_exhaust_descriptors(void) +{ + char *directory = NULL; + char *path = create_source(&directory); + const char *arguments[] = { + "--emit", "--stdout-size", "8", "--stderr-size", "8", NULL + }; + DocumentToolRunnerLimits limits = { 16, 16 }; + struct rlimit original_limit; + g_assert_cmpint(getrlimit(RLIMIT_NOFILE, &original_limit), ==, 0); + struct rlimit test_limit = original_limit; + test_limit.rlim_cur = MIN(original_limit.rlim_cur, (rlim_t) 64); + g_assert_cmpint(setrlimit(RLIMIT_NOFILE, &test_limit), ==, 0); + for (guint index = 0; index < 96; index++) + { + DocumentToolExecution *execution = NULL; + g_assert_true(document_tool_runner_run_with_limits( + "synthetic", "tests/fake_document_tool", arguments, path, + &limits, NULL, &execution, NULL)); + document_tool_execution_free(execution); + } + g_assert_cmpint(setrlimit(RLIMIT_NOFILE, &original_limit), ==, 0); + remove_source(directory, path); +} + +int main(int argc, char **argv) +{ + g_test_init(&argc, &argv, NULL); + g_test_add_func("/document-tool-runner/limits-concurrent", + test_limits_and_concurrent_drain); + g_test_add_func("/document-tool-runner/exact-below", + test_exact_and_below_limits); + g_test_add_func("/document-tool-runner/cancellation", + test_cancellation_during_output); + g_test_add_func("/document-tool-runner/no-descriptor-exhaustion", + test_repeated_runs_do_not_exhaust_descriptors); + return g_test_run(); +} diff --git a/tests/test_eml_pipeline_task.c b/tests/test_eml_pipeline_task.c index 646dfba..5e2d1f3 100644 --- a/tests/test_eml_pipeline_task.c +++ b/tests/test_eml_pipeline_task.c @@ -136,11 +136,107 @@ static void test_eml_pipeline_document_analysis(void) g_free(tmp_dir); } +static void wait_for_task(BackgroundTask *task) +{ + while (background_task_get_state(task) == + BACKGROUND_TASK_STATE_RUNNING || + background_task_get_state(task) == + BACKGROUND_TASK_STATE_PENDING) + g_main_context_iteration(NULL, TRUE); +} + +static DocumentAnalysisTools synthetic_tools(void) +{ + DocumentAnalysisTools tools = { + .exiftool = "tests/fake_document_tool", + .tesseract = "tests/fake_document_tool", + .pdfinfo = "tests/fake_document_tool", + .pdftotext = "tests/fake_document_tool", + .pdftoppm = "tests/fake_document_tool" + }; + return tools; +} + +static void test_eml_pipeline_pdf_end_to_end_and_limit(void) +{ + GError *error = NULL; + char *tmp_dir = g_dir_make_tmp("labfy-eml-pdf-XXXXXX", &error); + g_assert_no_error(error); + char *eml_path = g_build_filename(tmp_dir, "pdf.eml", NULL); + char *processed_dir = g_build_filename(tmp_dir, "processed", NULL); + static const char eml[] = + "From: synthetic@example.test\r\n" + "MIME-Version: 1.0\r\n" + "Content-Type: multipart/mixed; boundary=pdf-boundary\r\n\r\n" + "--pdf-boundary\r\n" + "Content-Type: application/pdf; name=scan.pdf\r\n" + "Content-Disposition: attachment; filename=scan.pdf\r\n" + "Content-Transfer-Encoding: base64\r\n\r\n" + "JVBERi1zeW50aGV0aWM=\r\n" + "--pdf-boundary\r\n" + "Content-Type: application/pdf; name=second.pdf\r\n" + "Content-Disposition: attachment; filename=second.pdf\r\n" + "Content-Transfer-Encoding: base64\r\n\r\n" + "JVBERi1zeW50aGV0aWM=\r\n" + "--pdf-boundary--\r\n"; + g_assert_true(g_file_set_contents(eml_path, eml, -1, &error)); + g_assert_no_error(error); + char *source_before = NULL; + gsize source_before_length = 0; + g_assert_true(g_file_get_contents( + eml_path, &source_before, &source_before_length, &error)); + g_assert_no_error(error); + + DocumentAnalysisTools tools = synthetic_tools(); + BackgroundTask *task = eml_pipeline_task_new_with_tools_and_limit( + eml_path, processed_dir, "synthetic-evidence", &tools, 1); + g_assert_nonnull(task); + wait_for_task(task); + g_assert_cmpint(background_task_get_state(task), ==, + BACKGROUND_TASK_STATE_COMPLETED); + EmlPipelineResult *result = background_task_get_result(task); + g_assert_nonnull(result); + g_assert_cmpuint(result->mime_result->attachments->len, ==, 2); + g_assert_cmpuint(result->document_analyses->len, ==, 1); + g_assert_cmpuint(result->skipped_document_analyses, ==, 1); + g_assert_cmpint(result->state, ==, DOCUMENT_ANALYSIS_STATE_PARTIAL); + g_assert_cmpuint(result->warnings->len, ==, 1); + DocumentFileAnalysis *document = g_ptr_array_index( + result->document_analyses, 0); + g_assert_nonnull(document->pdf); + g_assert_false(document->pdf->native_text_usable); + g_assert_cmpuint(document->pdf->pages->len, ==, 2); + PdfPageAnalysis *first = g_ptr_array_index(document->pdf->pages, 0); + PdfPageAnalysis *second = g_ptr_array_index(document->pdf->pages, 1); + g_assert_cmpuint(first->page_number, ==, 1); + g_assert_cmpuint(second->page_number, ==, 2); + g_assert_nonnull(first->render_execution); + g_assert_nonnull(first->execution); + + char *source_after = NULL; + gsize source_after_length = 0; + g_assert_true(g_file_get_contents( + eml_path, &source_after, &source_after_length, &error)); + g_assert_no_error(error); + g_assert_cmpuint(source_after_length, ==, source_before_length); + g_assert_cmpmem(source_after, source_after_length, + source_before, source_before_length); + g_free(source_after); + g_free(source_before); + background_task_unref(task); + g_remove(eml_path); + g_free(processed_dir); + g_free(eml_path); + g_free(tmp_dir); +} + int main(int argc, char **argv) { g_test_init(&argc, &argv, NULL); g_test_add_func("/eml-pipeline-task/basic", test_eml_pipeline_basic); g_test_add_func("/eml-pipeline-task/document-analysis", test_eml_pipeline_document_analysis); + g_test_add_func("/eml-pipeline-task/pdf-end-to-end-limit", + test_eml_pipeline_pdf_end_to_end_and_limit); return g_test_run(); } diff --git a/tests/test_exiftool_analysis.c b/tests/test_exiftool_analysis.c index 9a2c310..7f41eed 100644 --- a/tests/test_exiftool_analysis.c +++ b/tests/test_exiftool_analysis.c @@ -68,6 +68,54 @@ static void test_run_and_unavailable(void) g_free(directory); } +static gpointer cancel_exiftool(gpointer user_data) +{ + g_usleep(50000); + g_cancellable_cancel(user_data); + return NULL; +} + +static void test_cancellation_and_truncated_json(void) +{ + GError *error = NULL; + char *directory = g_dir_make_tmp("labfy-exif-hardening-XXXXXX", &error); + char *slow_path = g_build_filename(directory, "slow.png", NULL); + char *large_path = g_build_filename(directory, "large.png", NULL); + g_assert_true(g_file_set_contents(slow_path, "PNG", 3, &error)); + g_assert_true(g_file_set_contents(large_path, "PNG", 3, &error)); + + GCancellable *cancellable = g_cancellable_new(); + GThread *thread = g_thread_new( + "exif-cancel", cancel_exiftool, cancellable); + ExiftoolAnalysisResult *result = exiftool_analysis_run( + "tests/fake_document_tool", slow_path, cancellable, &error); + g_thread_join(thread); + g_assert_null(result); + g_assert_error(error, G_IO_ERROR, G_IO_ERROR_CANCELLED); + g_clear_error(&error); + g_object_unref(cancellable); + + DocumentToolRunnerLimits limits = { 128, 128 }; + result = exiftool_analysis_run_with_limits( + "tests/fake_document_tool", large_path, &limits, NULL, &error); + g_assert_no_error(error); + g_assert_nonnull(result); + g_assert_true(result->execution->stdout_truncated); + g_assert_cmpuint(strlen(result->execution->raw_stdout), ==, 128); + g_assert_cmpuint(result->metadata->len, ==, 0); + g_assert_cmpint(result->execution->state, ==, + DOCUMENT_ANALYSIS_STATE_FAILED); + g_assert_cmpuint(result->execution->warnings->len, >, 0); + exiftool_analysis_result_free(result); + + g_remove(slow_path); + g_remove(large_path); + g_rmdir(directory); + g_free(slow_path); + g_free(large_path); + g_free(directory); +} + int main(int argc, char **argv) { g_test_init(&argc, &argv, NULL); @@ -76,5 +124,7 @@ int main(int argc, char **argv) g_test_add_func("/exiftool-analysis/invalid-json", test_invalid_json); g_test_add_func("/exiftool-analysis/run-unavailable", test_run_and_unavailable); + g_test_add_func("/exiftool-analysis/cancellation-truncated", + test_cancellation_and_truncated_json); return g_test_run(); } diff --git a/tests/test_ocr_analysis.c b/tests/test_ocr_analysis.c index 48899b2..d4546f7 100644 --- a/tests/test_ocr_analysis.c +++ b/tests/test_ocr_analysis.c @@ -75,6 +75,29 @@ static void test_cancellation(void) g_free(directory); } +static void test_truncated_text(void) +{ + GError *error = NULL; + char *directory = g_dir_make_tmp("labfy-ocr-limit-XXXXXX", &error); + char *path = g_build_filename(directory, "large.png", NULL); + g_assert_true(g_file_set_contents(path, "PNG", 3, &error)); + DocumentToolRunnerLimits limits = { 96, 64 }; + OcrAnalysisResult *result = ocr_analysis_run_with_limits( + "tests/fake_document_tool", path, "fra", &limits, + NULL, &error); + g_assert_no_error(error); + g_assert_nonnull(result); + g_assert_cmpuint(strlen(result->text), ==, 96); + g_assert_true(result->execution->stdout_truncated); + g_assert_cmpint(result->execution->state, ==, + DOCUMENT_ANALYSIS_STATE_PARTIAL); + ocr_analysis_result_free(result); + g_remove(path); + g_rmdir(directory); + g_free(path); + g_free(directory); +} + int main(int argc, char **argv) { g_test_init(&argc, &argv, NULL); @@ -83,5 +106,6 @@ int main(int argc, char **argv) g_test_add_func("/ocr-analysis/unavailable-compatible", test_unavailable_and_compatibility); g_test_add_func("/ocr-analysis/cancellation", test_cancellation); + g_test_add_func("/ocr-analysis/truncated-text", test_truncated_text); return g_test_run(); } diff --git a/tests/test_pdf_analysis.c b/tests/test_pdf_analysis.c index 29477b8..4bec023 100644 --- a/tests/test_pdf_analysis.c +++ b/tests/test_pdf_analysis.c @@ -84,6 +84,64 @@ static void test_heuristic(void) "Texte synthétique imprimable et suffisamment long pour le test.")); } +static gpointer cancel_pdf(gpointer user_data) +{ + g_usleep(70000); + g_cancellable_cancel(user_data); + return NULL; +} + +static void assert_cancelled_pdf( + const PdfAnalysisTools *tools, + const char *path, + guint expected_completed_pages +) +{ + GCancellable *cancellable = g_cancellable_new(); + GThread *thread = g_thread_new("pdf-cancel", cancel_pdf, cancellable); + GError *error = NULL; + PdfAnalysisResult *result = pdf_analysis_run( + tools, path, "fra", cancellable, &error); + g_thread_join(thread); + g_assert_no_error(error); + g_assert_nonnull(result); + g_assert_cmpuint(result->pages->len, ==, expected_completed_pages); + g_assert_true(result->state == DOCUMENT_ANALYSIS_STATE_CANCELLED || + result->state == DOCUMENT_ANALYSIS_STATE_PARTIAL); + pdf_analysis_result_free(result); + g_object_unref(cancellable); +} + +static void test_cancellation_stages_and_completed_pages(void) +{ + GError *error = NULL; + char *directory = g_dir_make_tmp("labfy-pdf-cancel-XXXXXX", &error); + g_assert_no_error(error); + PdfAnalysisTools tools = fake_tools(); + char *slow_info = create_pdf(directory, "slow-info.pdf"); + char *slow_text = create_pdf(directory, "slow-text.pdf"); + char *slow_render = create_pdf(directory, "slow-render.pdf"); + char *slow_ocr = create_pdf(directory, "slow-ocr.pdf"); + char *slow_second = create_pdf(directory, "slow-page-2.pdf"); + assert_cancelled_pdf(&tools, slow_info, 0); + assert_cancelled_pdf(&tools, slow_text, 0); + assert_cancelled_pdf(&tools, slow_render, 0); + assert_cancelled_pdf(&tools, slow_ocr, 1); + assert_cancelled_pdf(&tools, slow_second, 1); + g_remove(slow_info); + g_remove(slow_text); + g_remove(slow_render); + g_remove(slow_ocr); + g_remove(slow_second); + g_rmdir(directory); + g_free(slow_info); + g_free(slow_text); + g_free(slow_render); + g_free(slow_ocr); + g_free(slow_second); + g_free(directory); +} + int main(int argc, char **argv) { g_test_init(&argc, &argv, NULL); @@ -92,5 +150,7 @@ int main(int argc, char **argv) g_test_add_func("/pdf-analysis/ocr-fallback-cleanup", test_ocr_fallback_order_and_cleanup); g_test_add_func("/pdf-analysis/heuristic", test_heuristic); + g_test_add_func("/pdf-analysis/cancellation-stages", + test_cancellation_stages_and_completed_pages); return g_test_run(); }