fy59/labfy-investigation
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

fix: borner les sorties des outils documentaires

Browse source
This commit is contained in:
grayTerminal-sh 2026-07-28 07:28:04 +02:00
parent d96fc83fc0
commit 015841a20c
17 changed files with 897 additions and 101 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
Makefile
View file

@ -139,6 +139,7 @@ TEST_EML_MIME_EXTRACTOR := tests/test_eml_mime_extractor
TEST_EXIFTOOL_ANALYSIS := tests/test_exiftool_analysis
TEST_OCR_ANALYSIS := tests/test_ocr_analysis
TEST_PDF_ANALYSIS := tests/test_pdf_analysis
TEST_DOCUMENT_TOOL_RUNNER := tests/test_document_tool_runner
FAKE_DOCUMENT_TOOL := tests/fake_document_tool
DOCUMENT_ANALYSIS_TEST_SOURCES := \
@ -147,6 +148,12 @@ DOCUMENT_ANALYSIS_TEST_SOURCES := \
src/core/tool_process.c \
src/core/file_hash.c
$(TEST_DOCUMENT_TOOL_RUNNER): tests/test_document_tool_runner.c \
$(DOCUMENT_ANALYSIS_TEST_SOURCES) $(FAKE_DOCUMENT_TOOL)
$(CC) $(TEST_CFLAGS) -Wpedantic \
tests/test_document_tool_runner.c \
$(DOCUMENT_ANALYSIS_TEST_SOURCES) -o $@ $(TEST_LDFLAGS)
all: $(TARGET)
$(TEST_BANK_PROPOSAL): \
@ -888,7 +895,8 @@ test: \
$(TEST_EML_MIME_EXTRACTOR) \
$(TEST_EXIFTOOL_ANALYSIS) \
$(TEST_OCR_ANALYSIS) \
$(TEST_PDF_ANALYSIS)
$(TEST_PDF_ANALYSIS) \
$(TEST_DOCUMENT_TOOL_RUNNER)
@echo "Exécution des tests..."
@./$(TEST_NODE)
@./$(TEST_TREE_MODEL)
@ -961,6 +969,7 @@ test: \
@$(TEST_EXIFTOOL_ANALYSIS)
@$(TEST_OCR_ANALYSIS)
@$(TEST_PDF_ANALYSIS)
@$(TEST_DOCUMENT_TOOL_RUNNER)
@echo "Tous les tests sont valides."
%.o: %.c
@ -1037,6 +1046,7 @@ clean:
$(TEST_EXIFTOOL_ANALYSIS) \
$(TEST_OCR_ANALYSIS) \
$(TEST_PDF_ANALYSIS) \
$(TEST_DOCUMENT_TOOL_RUNNER) \
$(FAKE_DOCUMENT_TOOL)

4
include/core/document_analysis.h
View file

@ -38,6 +38,10 @@ typedef struct
char *raw_stdout;
char *raw_stdout_sha256;
char *raw_stderr;
gsize stdout_bytes_observed;
gsize stderr_bytes_observed;
gboolean stdout_truncated;
gboolean stderr_truncated;
int exit_status;
DocumentAnalysisState state;
GPtrArray *warnings;

17
include/core/document_tool_runner.h
View file

@ -9,6 +9,12 @@
G_BEGIN_DECLS
typedef struct
{
gsize stdout_limit;
gsize stderr_limit;
} DocumentToolRunnerLimits;
gboolean document_tool_runner_run(
const char *tool_id,
const char *executable,
@ -19,6 +25,17 @@ gboolean document_tool_runner_run(
GError **error
);
gboolean document_tool_runner_run_with_limits(
const char *tool_id,
const char *executable,
const char *const arguments[],
const char *source_path,
const DocumentToolRunnerLimits *limits,
GCancellable *cancellable,
DocumentToolExecution **out_execution,
GError **error
);
char *document_tool_runner_read_version(
const char *executable,
const char *const arguments[],

10
include/core/eml_pipeline_task.h
View file

@ -21,6 +21,8 @@ typedef struct EmlPipelineResult
GPtrArray *bank_proposals; /**< Tableau de BankProposal* */
GPtrArray *document_analyses; /**< Tableau de DocumentFileAnalysis* */
GPtrArray *warnings; /**< Avertissements globaux */
DocumentAnalysisState state;
guint skipped_document_analyses;
} EmlPipelineResult;
/** @brief Libère un résultat EmlPipelineResult. */
@ -44,6 +46,14 @@ BackgroundTask *eml_pipeline_task_new_with_tools(
const DocumentAnalysisTools *tools
);
BackgroundTask *eml_pipeline_task_new_with_tools_and_limit(
const char *eml_path,
const char *processed_evidence_dir,
const char *evidence_id,
const DocumentAnalysisTools *tools,
guint document_analysis_limit
);
G_END_DECLS
#endif /* LABFY_INVESTIGATION_EML_PIPELINE_TASK_H */

8
include/core/exiftool_analysis.h
View file

@ -6,6 +6,7 @@
#define LABFY_INVESTIGATION_EXIFTOOL_ANALYSIS_H
#include "core/document_analysis.h"
#include "core/document_tool_runner.h"
G_BEGIN_DECLS
@ -21,6 +22,13 @@ ExiftoolAnalysisResult *exiftool_analysis_run(
GCancellable *cancellable,
GError **error
);
ExiftoolAnalysisResult *exiftool_analysis_run_with_limits(
const char *executable,
const char *file_path,
const DocumentToolRunnerLimits *limits,
GCancellable *cancellable,
GError **error
);
ExiftoolAnalysisResult *exiftool_analysis_parse(
const char *file_path,
const char *json,

9
include/core/ocr_analysis.h
View file

@ -6,6 +6,7 @@
#define LABFY_INVESTIGATION_OCR_ANALYSIS_H
#include "core/document_analysis.h"
#include "core/document_tool_runner.h"
G_BEGIN_DECLS
@ -23,6 +24,14 @@ OcrAnalysisResult *ocr_analysis_run(
GCancellable *cancellable,
GError **error
);
OcrAnalysisResult *ocr_analysis_run_with_limits(
const char *executable,
const char *image_path,
const char *languages,
const DocumentToolRunnerLimits *limits,
GCancellable *cancellable,
GError **error
);
void ocr_analysis_result_free(OcrAnalysisResult *result);
gboolean ocr_analysis_mime_is_compatible(const char *mime_type);

343
src/core/document_tool_runner.c
View file

@ -1,90 +1,232 @@
/******************************************************************************
* @file document_tool_runner.c
* @brief Exécution bornée et annulable des outils documentaires.
* @brief Exécution réellement bornée et annulable des outils documentaires.
******************************************************************************/
#include "core/document_tool_runner.h"
#include "core/file_hash.h"
#include "core/tool_process.h"
#include <glib/gstdio.h>
#define DOCUMENT_TOOL_RUNNER_READ_BLOCK 4096U
#define DOCUMENT_TOOL_RUNNER_VERSION_LIMIT 65536U
typedef struct
{
GInputStream *stream;
GCancellable *cancellable;
GByteArray *prefix;
gsize limit;
gsize bytes_observed;
gboolean truncated;
GError *error;
} DocumentToolStreamCapture;
typedef struct
{
GBytes *stdout_bytes;
GBytes *stderr_bytes;
gsize stdout_bytes_observed;
gsize stderr_bytes_observed;
gboolean stdout_truncated;
gboolean stderr_truncated;
gboolean exited_normally;
int exit_status;
} DocumentToolCaptureResult;
static GPtrArray *document_tool_runner_build_argv(
const char *executable,
const char *const arguments[]
)
{
GPtrArray *argv = g_ptr_array_new();
g_ptr_array_add(argv, (gpointer) executable);
for (gsize index = 0; arguments != NULL &&
arguments[index] != NULL; index++)
g_ptr_array_add(argv, (gpointer) arguments[index]);
g_ptr_array_add(argv, NULL);
return argv;
}
static gpointer document_tool_runner_drain_stream(gpointer user_data)
{
DocumentToolStreamCapture *capture = user_data;
guint8 block[DOCUMENT_TOOL_RUNNER_READ_BLOCK];
while (TRUE)
{
gssize bytes_read = g_input_stream_read(
capture->stream, block, sizeof(block),
capture->cancellable, &capture->error);
if (bytes_read <= 0)
break;
gsize observed = (gsize) bytes_read;
if (G_MAXSIZE - capture->bytes_observed < observed)
capture->bytes_observed = G_MAXSIZE;
else
capture->bytes_observed += observed;
gsize remaining = capture->prefix->len < capture->limit
? capture->limit - capture->prefix->len : 0;
gsize retained = MIN(remaining, observed);
if (retained > 0)
g_byte_array_append(capture->prefix, block, retained);
if (retained < observed)
capture->truncated = TRUE;
}
return NULL;
}
static void document_tool_capture_result_clear(
DocumentToolCaptureResult *result
)
{
g_clear_pointer(&result->stdout_bytes, g_bytes_unref);
g_clear_pointer(&result->stderr_bytes, g_bytes_unref);
}
static gboolean document_tool_runner_capture(
const char *executable,
const char *const arguments[],
const DocumentToolRunnerLimits *limits,
GCancellable *cancellable,
DocumentToolCaptureResult *out_result,
GError **error
)
{
GPtrArray *argv = document_tool_runner_build_argv(executable, arguments);
GSubprocessLauncher *launcher = g_subprocess_launcher_new(
G_SUBPROCESS_FLAGS_STDOUT_PIPE | G_SUBPROCESS_FLAGS_STDERR_PIPE);
GError *local_error = NULL;
GSubprocess *process = g_subprocess_launcher_spawnv(
launcher, (const char *const *) argv->pdata, &local_error);
g_object_unref(launcher);
g_ptr_array_unref(argv);
if (process == NULL)
{
g_propagate_error(error, local_error);
return FALSE;
}
DocumentToolStreamCapture stdout_capture = {
.stream = g_subprocess_get_stdout_pipe(process),
.cancellable = cancellable,
.prefix = g_byte_array_sized_new(MIN(limits->stdout_limit, 4096U)),
.limit = limits->stdout_limit
};
DocumentToolStreamCapture stderr_capture = {
.stream = g_subprocess_get_stderr_pipe(process),
.cancellable = cancellable,
.prefix = g_byte_array_sized_new(MIN(limits->stderr_limit, 4096U)),
.limit = limits->stderr_limit
};
GThread *stdout_thread = g_thread_new(
"document-stdout", document_tool_runner_drain_stream, &stdout_capture);
GThread *stderr_thread = g_thread_new(
"document-stderr", document_tool_runner_drain_stream, &stderr_capture);
gboolean waited = g_subprocess_wait(process, cancellable, &local_error);
if (!waited)
{
g_subprocess_force_exit(process);
GError *final_wait_error = NULL;
(void) g_subprocess_wait(process, NULL, &final_wait_error);
g_clear_error(&final_wait_error);
}
g_thread_join(stdout_thread);
g_thread_join(stderr_thread);
gboolean cancelled =
(cancellable != NULL && g_cancellable_is_cancelled(cancellable)) ||
g_error_matches(local_error, G_IO_ERROR, G_IO_ERROR_CANCELLED) ||
g_error_matches(stdout_capture.error, G_IO_ERROR, G_IO_ERROR_CANCELLED) ||
g_error_matches(stderr_capture.error, G_IO_ERROR, G_IO_ERROR_CANCELLED);
if (!waited || stdout_capture.error != NULL ||
stderr_capture.error != NULL)
{
if (cancelled)
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_CANCELLED,
"L'exécution de l'outil documentaire a été annulée.");
else if (local_error != NULL)
g_propagate_error(error, g_steal_pointer(&local_error));
else
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_FAILED,
"La lecture des sorties de l'outil documentaire a échoué.");
g_clear_error(&local_error);
g_clear_error(&stdout_capture.error);
g_clear_error(&stderr_capture.error);
g_byte_array_unref(stdout_capture.prefix);
g_byte_array_unref(stderr_capture.prefix);
g_object_unref(process);
return FALSE;
}
out_result->stdout_bytes = g_byte_array_free_to_bytes(
stdout_capture.prefix);
out_result->stderr_bytes = g_byte_array_free_to_bytes(
stderr_capture.prefix);
out_result->stdout_bytes_observed = stdout_capture.bytes_observed;
out_result->stderr_bytes_observed = stderr_capture.bytes_observed;
out_result->stdout_truncated = stdout_capture.truncated;
out_result->stderr_truncated = stderr_capture.truncated;
out_result->exited_normally = g_subprocess_get_if_exited(process);
out_result->exit_status = out_result->exited_normally
? g_subprocess_get_exit_status(process) : -1;
g_clear_error(&local_error);
g_object_unref(process);
return TRUE;
}
static char *document_tool_runner_bytes_to_text(GBytes *bytes)
{
gsize length = 0;
const char *data = g_bytes_get_data(bytes, &length);
return g_utf8_make_valid(data != NULL ? data : "", (gssize) length);
}
char *document_tool_runner_read_version(
const char *executable,
const char *const arguments[],
GCancellable *cancellable
)
{
ToolProcessResult *result = NULL;
DocumentToolRunnerLimits limits = {
DOCUMENT_TOOL_RUNNER_VERSION_LIMIT,
DOCUMENT_TOOL_RUNNER_VERSION_LIMIT
};
DocumentToolCaptureResult capture = { 0 };
GError *error = NULL;
char *version = NULL;
if (!tool_process_run(executable, arguments, NULL, cancellable,
&result, &error))
if (!document_tool_runner_capture(executable, arguments, &limits,
cancellable, &capture, &error))
{
g_clear_error(&error);
return NULL;
}
GBytes *stdout_bytes = tool_process_result_ref_stdout(result);
GBytes *stderr_bytes = tool_process_result_ref_stderr(result);
gsize stdout_length = 0;
gsize stderr_length = 0;
const char *stdout_data = g_bytes_get_data(
stdout_bytes, &stdout_length);
const char *stderr_data = g_bytes_get_data(
stderr_bytes, &stderr_length);
if (stdout_data == NULL)
stdout_data = "";
if (stderr_data == NULL)
stderr_data = "";
if (stdout_length > 0)
version = g_utf8_make_valid(stdout_data, (gssize) stdout_length);
else if (stderr_length > 0)
version = g_utf8_make_valid(stderr_data, (gssize) stderr_length);
if (version != NULL)
g_strstrip(version);
g_bytes_unref(stdout_bytes);
g_bytes_unref(stderr_bytes);
tool_process_result_free(result);
char *version = document_tool_runner_bytes_to_text(
g_bytes_get_size(capture.stdout_bytes) > 0
? capture.stdout_bytes : capture.stderr_bytes);
g_strstrip(version);
document_tool_capture_result_clear(&capture);
return version;
}
static char *document_tool_runner_bytes_to_text(
GBytes *bytes,
gsize limit,
gboolean *truncated
)
{
gsize length = 0;
const char *data = bytes != NULL
? g_bytes_get_data(bytes, &length)
: "";
if (data == NULL)
data = "";
if (length > limit)
{
length = limit;
*truncated = TRUE;
}
return g_utf8_make_valid(data, (gssize) length);
}
gboolean document_tool_runner_run(
gboolean document_tool_runner_run_with_limits(
const char *tool_id,
const char *executable,
const char *const arguments[],
const char *source_path,
const DocumentToolRunnerLimits *limits,
GCancellable *cancellable,
DocumentToolExecution **out_execution,
GError **error
)
{
ToolProcessResult *process_result = NULL;
DocumentToolExecution *execution = NULL;
GError *process_error = NULL;
gboolean stdout_truncated = FALSE;
gboolean stderr_truncated = FALSE;
g_return_val_if_fail(error == NULL || *error == NULL, FALSE);
if (tool_id == NULL || executable == NULL || source_path == NULL ||
out_execution == NULL || *out_execution != NULL)
limits == NULL || limits->stdout_limit == 0 ||
limits->stderr_limit == 0 || out_execution == NULL ||
*out_execution != NULL)
{
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
"Les paramètres de l'outil documentaire sont invalides.");
@ -98,31 +240,34 @@ gboolean document_tool_runner_run(
"Le fichier dépasse la taille maximale d'analyse.");
return FALSE;
}
execution = document_tool_execution_new(tool_id, source_path);
DocumentToolExecution *execution =
document_tool_execution_new(tool_id, source_path);
for (gsize index = 0; arguments != NULL &&
arguments[index] != NULL; index++)
document_tool_execution_add_argument(execution, arguments[index]);
(void) file_hash_compute_sha256(source_path, cancellable,
&execution->source_sha256, NULL, NULL);
if (!tool_process_run(executable, arguments, NULL, cancellable,
&process_result, &process_error))
DocumentToolCaptureResult capture = { 0 };
GError *capture_error = NULL;
if (!document_tool_runner_capture(executable, arguments, limits,
cancellable, &capture, &capture_error))
{
if (g_error_matches(process_error, TOOL_PROCESS_ERROR,
TOOL_PROCESS_ERROR_CANCELLED))
if (g_error_matches(capture_error, G_IO_ERROR, G_IO_ERROR_CANCELLED))
{
execution->state = DOCUMENT_ANALYSIS_STATE_CANCELLED;
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_CANCELLED,
"L'analyse documentaire a été annulée.");
g_propagate_error(error, capture_error);
capture_error = NULL;
}
else
{
execution->state = DOCUMENT_ANALYSIS_STATE_UNAVAILABLE;
g_ptr_array_add(execution->errors,
g_strdup(process_error != NULL ? process_error->message :
"Outil indisponible."));
g_ptr_array_add(execution->errors, g_strdup(
capture_error != NULL ? capture_error->message :
"Outil indisponible."));
}
g_clear_error(&process_error);
g_clear_error(&capture_error);
GDateTime *now = g_date_time_new_now_utc();
execution->finished_at_utc = g_date_time_format_iso8601(now);
g_date_time_unref(now);
@ -130,33 +275,53 @@ gboolean document_tool_runner_run(
return execution->state != DOCUMENT_ANALYSIS_STATE_CANCELLED;
}
GBytes *stdout_bytes = tool_process_result_ref_stdout(process_result);
GBytes *stderr_bytes = tool_process_result_ref_stderr(process_result);
execution->raw_stdout = document_tool_runner_bytes_to_text(stdout_bytes,
DOCUMENT_ANALYSIS_MAX_STDOUT, &stdout_truncated);
execution->raw_stderr = document_tool_runner_bytes_to_text(stderr_bytes,
DOCUMENT_ANALYSIS_MAX_STDERR, &stderr_truncated);
execution->exit_status =
tool_process_result_get_exit_status(process_result);
if (execution->raw_stdout != NULL)
execution->raw_stdout_sha256 = g_compute_checksum_for_string(
G_CHECKSUM_SHA256, execution->raw_stdout, -1);
if (stdout_truncated || stderr_truncated)
{
execution->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
execution->raw_stdout =
document_tool_runner_bytes_to_text(capture.stdout_bytes);
execution->raw_stderr =
document_tool_runner_bytes_to_text(capture.stderr_bytes);
execution->stdout_bytes_observed = capture.stdout_bytes_observed;
execution->stderr_bytes_observed = capture.stderr_bytes_observed;
execution->stdout_truncated = capture.stdout_truncated;
execution->stderr_truncated = capture.stderr_truncated;
execution->exit_status = capture.exit_status;
execution->raw_stdout_sha256 = g_compute_checksum_for_string(
G_CHECKSUM_SHA256, execution->raw_stdout, -1);
if (execution->stdout_truncated)
g_ptr_array_add(execution->warnings,
g_strdup("La sortie de l'outil a été tronquée à la limite."));
}
else
execution->state = tool_process_result_is_success(process_result)
? DOCUMENT_ANALYSIS_STATE_SUCCESS
: DOCUMENT_ANALYSIS_STATE_FAILED;
g_strdup("La sortie standard de l'outil a été tronquée."));
if (execution->stderr_truncated)
g_ptr_array_add(execution->warnings,
g_strdup("La sortie d'erreur de l'outil a été tronquée."));
execution->state =
execution->stdout_truncated || execution->stderr_truncated
? DOCUMENT_ANALYSIS_STATE_PARTIAL
: capture.exited_normally && capture.exit_status == 0
? DOCUMENT_ANALYSIS_STATE_SUCCESS
: DOCUMENT_ANALYSIS_STATE_FAILED;
GDateTime *now = g_date_time_new_now_utc();
execution->finished_at_utc = g_date_time_format_iso8601(now);
g_date_time_unref(now);
g_clear_pointer(&stdout_bytes, g_bytes_unref);
g_clear_pointer(&stderr_bytes, g_bytes_unref);
tool_process_result_free(process_result);
document_tool_capture_result_clear(&capture);
*out_execution = execution;
return TRUE;
}
gboolean document_tool_runner_run(
const char *tool_id,
const char *executable,
const char *const arguments[],
const char *source_path,
GCancellable *cancellable,
DocumentToolExecution **out_execution,
GError **error
)
{
const DocumentToolRunnerLimits limits = {
DOCUMENT_ANALYSIS_MAX_STDOUT,
DOCUMENT_ANALYSIS_MAX_STDERR
};
return document_tool_runner_run_with_limits(tool_id, executable,
arguments, source_path, &limits, cancellable, out_execution, error);
}

40
src/core/eml_pipeline_task.c
View file

@ -18,6 +18,7 @@ typedef struct
char *pdfinfo;
char *pdftotext;
char *pdftoppm;
guint document_analysis_limit;
} EmlPipelineTaskData;
static void eml_pipeline_task_data_free(gpointer user_data)
@ -116,18 +117,19 @@ static gboolean eml_pipeline_task_worker(BackgroundTask *task,
GPtrArray *bank_proposals = g_ptr_array_new_with_free_func((GDestroyNotify) bank_proposal_free);
GPtrArray *document_analyses = g_ptr_array_new_with_free_func(
(GDestroyNotify) document_file_analysis_free);
GPtrArray *pipeline_warnings =
g_ptr_array_new_with_free_func(g_free);
guint skipped_document_analyses = 0;
for (guint i = 0; mime_res->attachments != NULL && i < mime_res->attachments->len; i++)
{
if (document_analyses->len >=
DOCUMENT_ANALYSIS_MAX_PIPELINE_ITEMS)
break;
if (g_cancellable_is_cancelled(cancellable))
{
eml_analysis_free(analysis);
eml_mime_result_free(mime_res);
g_ptr_array_unref(bank_proposals);
g_ptr_array_unref(document_analyses);
g_ptr_array_unref(pipeline_warnings);
g_set_error_literal(
error,
G_IO_ERROR,
@ -160,6 +162,11 @@ static gboolean eml_pipeline_task_worker(BackgroundTask *task,
g_strcmp0(att->detected_mime, "application/pdf") == 0 ||
g_strcmp0(att->content_type, "application/pdf") == 0)
{
if (document_analyses->len >= data->document_analysis_limit)
{
skipped_document_analyses++;
continue;
}
DocumentAnalysisTools tools = {
.exiftool = data->exiftool,
.tesseract = data->tesseract,
@ -182,6 +189,7 @@ static gboolean eml_pipeline_task_worker(BackgroundTask *task,
eml_mime_result_free(mime_res);
g_ptr_array_unref(bank_proposals);
g_ptr_array_unref(document_analyses);
g_ptr_array_unref(pipeline_warnings);
return FALSE;
}
g_clear_error(&analysis_error);
@ -210,7 +218,16 @@ static gboolean eml_pipeline_task_worker(BackgroundTask *task,
res->mime_result = mime_res;
res->bank_proposals = bank_proposals;
res->document_analyses = document_analyses;
res->warnings = g_ptr_array_new_with_free_func(g_free);
res->warnings = pipeline_warnings;
res->skipped_document_analyses = skipped_document_analyses;
res->state = skipped_document_analyses > 0
? DOCUMENT_ANALYSIS_STATE_PARTIAL
: DOCUMENT_ANALYSIS_STATE_SUCCESS;
if (skipped_document_analyses > 0)
g_ptr_array_add(res->warnings, g_strdup_printf(
"La limite d'analyses documentaires est atteinte : "
"%u fichier(s) n'ont pas été analysés.",
skipped_document_analyses));
if (out_result != NULL)
*out_result = res;
@ -238,9 +255,23 @@ BackgroundTask *eml_pipeline_task_new_with_tools(
const char *processed_evidence_dir,
const char *evidence_id,
const DocumentAnalysisTools *tools)
{
return eml_pipeline_task_new_with_tools_and_limit(
eml_path, processed_evidence_dir, evidence_id, tools,
DOCUMENT_ANALYSIS_MAX_PIPELINE_ITEMS);
}
BackgroundTask *eml_pipeline_task_new_with_tools_and_limit(
const char *eml_path,
const char *processed_evidence_dir,
const char *evidence_id,
const DocumentAnalysisTools *tools,
guint document_analysis_limit)
{
if (eml_path == NULL || processed_evidence_dir == NULL || tools == NULL)
return NULL;
if (document_analysis_limit == 0)
return NULL;
EmlPipelineTaskData *data = g_new0(EmlPipelineTaskData, 1);
data->eml_path = g_strdup(eml_path);
@ -251,6 +282,7 @@ BackgroundTask *eml_pipeline_task_new_with_tools(
data->pdfinfo = g_strdup(tools->pdfinfo);
data->pdftotext = g_strdup(tools->pdftotext);
data->pdftoppm = g_strdup(tools->pdftoppm);
data->document_analysis_limit = document_analysis_limit;
BackgroundTask *task = background_task_new(
"Analyse du message EML et de ses pièces jointes");

32
src/core/exiftool_analysis.c
View file

@ -216,12 +216,28 @@ ExiftoolAnalysisResult *exiftool_analysis_run(
GCancellable *cancellable,
GError **error
)
{
const DocumentToolRunnerLimits limits = {
DOCUMENT_ANALYSIS_MAX_STDOUT,
DOCUMENT_ANALYSIS_MAX_STDERR
};
return exiftool_analysis_run_with_limits(
executable, file_path, &limits, cancellable, error);
}
ExiftoolAnalysisResult *exiftool_analysis_run_with_limits(
const char *executable,
const char *file_path,
const DocumentToolRunnerLimits *limits,
GCancellable *cancellable,
GError **error
)
{
const char *arguments[] = { "-j", "-G1", "-n", "--", file_path, NULL };
const char *version_arguments[] = { "-ver", NULL };
DocumentToolExecution *execution = NULL;
if (!document_tool_runner_run("exiftool", executable, arguments,
file_path, cancellable, &execution, error))
if (!document_tool_runner_run_with_limits("exiftool", executable,
arguments, file_path, limits, cancellable, &execution, error))
{
document_tool_execution_free(execution);
return NULL;
@ -237,6 +253,18 @@ ExiftoolAnalysisResult *exiftool_analysis_run(
}
execution->version = document_tool_runner_read_version(
executable, version_arguments, cancellable);
if (execution->stdout_truncated)
{
ExiftoolAnalysisResult *truncated =
g_new0(ExiftoolAnalysisResult, 1);
truncated->execution = execution;
truncated->metadata = g_ptr_array_new_with_free_func(
exiftool_metadata_entry_free);
execution->state = DOCUMENT_ANALYSIS_STATE_FAILED;
g_ptr_array_add(execution->errors, g_strdup(
"Le JSON ExifTool tronqué n'a pas été interprété."));
return truncated;
}
ExiftoolAnalysisResult *result = exiftool_analysis_parse(file_path,
execution->raw_stdout != NULL ? execution->raw_stdout : "",
execution->raw_stderr, execution->exit_status, error);

21
src/core/ocr_analysis.c
View file

@ -29,6 +29,23 @@ OcrAnalysisResult *ocr_analysis_run(
GCancellable *cancellable,
GError **error
)
{
const DocumentToolRunnerLimits limits = {
DOCUMENT_ANALYSIS_MAX_TEXT,
DOCUMENT_ANALYSIS_MAX_STDERR
};
return ocr_analysis_run_with_limits(executable, image_path, languages,
&limits, cancellable, error);
}
OcrAnalysisResult *ocr_analysis_run_with_limits(
const char *executable,
const char *image_path,
const char *languages,
const DocumentToolRunnerLimits *limits,
GCancellable *cancellable,
GError **error
)
{
if (executable == NULL || image_path == NULL || languages == NULL ||
languages[0] == '\0' ||
@ -45,8 +62,8 @@ OcrAnalysisResult *ocr_analysis_run(
};
const char *version_arguments[] = { "--version", NULL };
DocumentToolExecution *execution = NULL;
if (!document_tool_runner_run("tesseract", executable, arguments,
image_path, cancellable, &execution, error))
if (!document_tool_runner_run_with_limits("tesseract", executable,
arguments, image_path, limits, cancellable, &execution, error))
{
document_tool_execution_free(execution);
return NULL;

37
src/core/pdf_analysis.c
View file

@ -139,6 +139,7 @@ static gboolean pdf_analysis_render_and_ocr(
}
gboolean success = TRUE;
char *source_basename = g_path_get_basename(result->source_path);
for (guint page_number = 1; page_number <= pages; page_number++)
{
if (cancellable != NULL &&
@ -150,8 +151,8 @@ static gboolean pdf_analysis_render_and_ocr(
success = FALSE;
break;
}
char *prefix = g_strdup_printf("%s/page-%u",
temporary_directory, page_number);
char *prefix = g_strdup_printf("%s/%s-page-%u",
temporary_directory, source_basename, page_number);
char *page_text = g_strdup_printf("%u", page_number);
const char *render_arguments[] = {
"-f", page_text, "-singlefile", "-png",
@ -162,6 +163,9 @@ static gboolean pdf_analysis_render_and_ocr(
render_arguments, result->source_path, cancellable,
&render_execution, error))
{
result->state = result->pages->len > 0
? DOCUMENT_ANALYSIS_STATE_PARTIAL
: DOCUMENT_ANALYSIS_STATE_CANCELLED;
document_tool_execution_free(render_execution);
g_free(page_text);
g_free(prefix);
@ -198,12 +202,23 @@ static gboolean pdf_analysis_render_and_ocr(
g_ptr_array_add(result->pages, page);
if (page->state != DOCUMENT_ANALYSIS_STATE_SUCCESS)
result->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
if (ocr == NULL && error != NULL && *error != NULL &&
g_error_matches(*error, G_IO_ERROR, G_IO_ERROR_CANCELLED))
{
result->state = result->pages->len > 1
? DOCUMENT_ANALYSIS_STATE_PARTIAL
: DOCUMENT_ANALYSIS_STATE_CANCELLED;
success = FALSE;
}
}
g_remove(image_path);
g_free(image_path);
g_free(page_text);
g_free(prefix);
if (!success)
break;
}
g_free(source_basename);
g_rmdir(temporary_directory);
g_free(temporary_directory);
return success;
@ -233,7 +248,16 @@ PdfAnalysisResult *pdf_analysis_run(
if (!document_tool_runner_run("pdfinfo", tools->pdfinfo,
info_arguments, pdf_path, cancellable,
&result->pdfinfo_execution, error))
{
if (error != NULL && *error != NULL &&
g_error_matches(*error, G_IO_ERROR, G_IO_ERROR_CANCELLED))
{
result->state = DOCUMENT_ANALYSIS_STATE_CANCELLED;
g_clear_error(error);
return result;
}
goto failure;
}
if (result->pdfinfo_execution->state ==
DOCUMENT_ANALYSIS_STATE_UNAVAILABLE)
{
@ -262,7 +286,16 @@ PdfAnalysisResult *pdf_analysis_run(
if (!document_tool_runner_run("pdftotext", tools->pdftotext,
text_arguments, pdf_path, cancellable,
&result->native_execution, error))
{
if (error != NULL && *error != NULL &&
g_error_matches(*error, G_IO_ERROR, G_IO_ERROR_CANCELLED))
{
result->state = DOCUMENT_ANALYSIS_STATE_CANCELLED;
g_clear_error(error);
return result;
}
goto failure;
}
result->native_execution->version =
document_tool_runner_read_version(
tools->pdftotext, version_arguments, cancellable);

75
tests/fake_document_tool.c
View file

@ -5,6 +5,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <poll.h>
#include <unistd.h>
static int has_argument(int argc, char **argv, const char *value)
@ -23,8 +24,56 @@ static const char *find_pdf_path(int argc, char **argv)
return "";
}
static long argument_long(
int argc,
char **argv,
const char *name,
long fallback
)
{
for (int index = 1; index + 1 < argc; index++)
if (strcmp(argv[index], name) == 0)
return strtol(argv[index + 1], NULL, 10);
return fallback;
}
static void write_repeated(FILE *stream, char value, long count)
{
char block[1024];
memset(block, value, sizeof(block));
while (count > 0)
{
size_t amount = (size_t) (count > (long) sizeof(block)
? (long) sizeof(block) : count);
if (fwrite(block, 1, amount, stream) != amount)
return;
fflush(stream);
count -= (long) amount;
}
}
int main(int argc, char **argv)
{
if (has_argument(argc, argv, "--emit"))
{
long stdout_size = argument_long(
argc, argv, "--stdout-size", 0);
long stderr_size = argument_long(
argc, argv, "--stderr-size", 0);
long chunks = argument_long(argc, argv, "--chunks", 1);
int exit_status = (int) argument_long(
argc, argv, "--exit-status", 0);
if (chunks < 1)
chunks = 1;
for (long index = 0; index < chunks; index++)
{
write_repeated(stdout, 'O', stdout_size / chunks);
write_repeated(stderr, 'E', stderr_size / chunks);
if (has_argument(argc, argv, "--slow"))
(void) poll(NULL, 0, 20);
}
return exit_status;
}
if (has_argument(argc, argv, "-ver"))
{
puts("13.00");
@ -37,6 +86,14 @@ int main(int argc, char **argv)
}
if (has_argument(argc, argv, "-j"))
{
if (strstr(argv[argc - 1], "slow") != NULL)
sleep(2);
if (strstr(argv[argc - 1], "large") != NULL)
{
fputs("[{\"File:MIMEType\":\"image/png\",\"Padding\":\"", stdout);
write_repeated(stdout, 'X', 8192);
return 0;
}
puts("[{\"File:MIMEType\":\"image/png\","
"\"File:FileSize\":42,\"EXIF:ImageWidth\":10,"
"\"EXIF:GPSLatitude\":48.5,\"EXIF:GPSLongitude\":2.2,"
@ -46,6 +103,8 @@ int main(int argc, char **argv)
if (has_argument(argc, argv, "-enc"))
{
const char *path = find_pdf_path(argc, argv);
if (strstr(path, "slow-text") != NULL)
sleep(2);
if (strstr(path, "native") != NULL)
{
puts("Texte synthétique suffisamment long pour être considéré "
@ -55,6 +114,12 @@ int main(int argc, char **argv)
}
if (has_argument(argc, argv, "-singlefile"))
{
const char *pdf_path = find_pdf_path(argc, argv);
if (strstr(pdf_path, "slow-render") != NULL)
sleep(2);
if (strstr(pdf_path, "slow-page-2") != NULL &&
strcmp(argv[2], "2") == 0)
sleep(2);
const char *prefix = argv[argc - 1];
char path[4096];
if (snprintf(path, sizeof(path), "%s.png", prefix) < 0)
@ -68,8 +133,14 @@ int main(int argc, char **argv)
}
if (has_argument(argc, argv, "stdout"))
{
if (strstr(argv[1], "sleep") != NULL)
if (strstr(argv[1], "sleep") != NULL ||
strstr(argv[1], "slow-ocr") != NULL)
sleep(2);
if (strstr(argv[1], "large") != NULL)
{
write_repeated(stdout, 'T', 8192);
return 0;
}
if (strstr(argv[1], "page-2") != NULL)
puts("Texte OCR synthétique page deux.");
else
@ -77,6 +148,8 @@ int main(int argc, char **argv)
return 0;
}
const char *path = find_pdf_path(argc, argv);
if (strstr(path, "slow-info") != NULL)
sleep(2);
if (strstr(path, "encrypted") != NULL)
puts("Pages: 2\nEncrypted: yes");
else

160
tests/test_document_tool_runner.c Normal file
View file

@ -0,0 +1,160 @@
/******************************************************************************
* @file test_document_tool_runner.c
* @brief Tests synthétiques de la capture documentaire bornée.
******************************************************************************/
#include "core/document_tool_runner.h"
#include <glib.h>
#include <glib/gstdio.h>
#include <sys/resource.h>
static char *create_source(char **out_directory)
{
GError *error = NULL;
*out_directory = g_dir_make_tmp("labfy-runner-XXXXXX", &error);
g_assert_no_error(error);
char *path = g_build_filename(*out_directory, "source.bin", NULL);
g_assert_true(g_file_set_contents(path, "synthetic", -1, &error));
g_assert_no_error(error);
return path;
}
static void remove_source(char *directory, char *path)
{
g_remove(path);
g_rmdir(directory);
g_free(path);
g_free(directory);
}
static void test_limits_and_concurrent_drain(void)
{
char *directory = NULL;
char *path = create_source(&directory);
const char *arguments[] = {
"--emit", "--stdout-size", "5000", "--stderr-size", "7000",
"--chunks", "20", "--exit-status", "7", NULL
};
DocumentToolRunnerLimits limits = { 1000, 1500 };
DocumentToolExecution *execution = NULL;
GError *error = NULL;
g_assert_true(document_tool_runner_run_with_limits(
"synthetic", "tests/fake_document_tool", arguments, path,
&limits, NULL, &execution, &error));
g_assert_no_error(error);
g_assert_cmpuint(strlen(execution->raw_stdout), ==, 1000);
g_assert_cmpuint(strlen(execution->raw_stderr), ==, 1500);
g_assert_cmpuint(execution->stdout_bytes_observed, ==, 5000);
g_assert_cmpuint(execution->stderr_bytes_observed, ==, 7000);
g_assert_true(execution->stdout_truncated);
g_assert_true(execution->stderr_truncated);
g_assert_cmpint(execution->exit_status, ==, 7);
g_assert_cmpint(execution->state, ==,
DOCUMENT_ANALYSIS_STATE_PARTIAL);
g_assert_cmpint(execution->raw_stdout[0], ==, 'O');
g_assert_cmpint(execution->raw_stderr[0], ==, 'E');
document_tool_execution_free(execution);
remove_source(directory, path);
}
static void test_exact_and_below_limits(void)
{
char *directory = NULL;
char *path = create_source(&directory);
const char *arguments[] = {
"--emit", "--stdout-size", "1000", "--stderr-size", "12", NULL
};
DocumentToolRunnerLimits limits = { 1000, 20 };
DocumentToolExecution *execution = NULL;
g_assert_true(document_tool_runner_run_with_limits(
"synthetic", "tests/fake_document_tool", arguments, path,
&limits, NULL, &execution, NULL));
g_assert_false(execution->stdout_truncated);
g_assert_false(execution->stderr_truncated);
g_assert_cmpint(execution->state, ==,
DOCUMENT_ANALYSIS_STATE_SUCCESS);
document_tool_execution_free(execution);
remove_source(directory, path);
}
static gpointer cancel_later(gpointer user_data)
{
g_usleep(60000);
g_cancellable_cancel(user_data);
return NULL;
}
static void test_cancellation_during_output(void)
{
char *directory = NULL;
char *path = create_source(&directory);
const char *arguments[] = {
"--emit", "--stdout-size", "100000", "--stderr-size", "100000",
"--chunks", "100", "--slow", NULL
};
DocumentToolRunnerLimits limits = { 128, 128 };
DocumentToolExecution *execution = NULL;
GCancellable *cancellable = g_cancellable_new();
GThread *thread = g_thread_new("cancel-runner", cancel_later, cancellable);
GError *error = NULL;
g_assert_false(document_tool_runner_run_with_limits(
"synthetic", "tests/fake_document_tool", arguments, path,
&limits, cancellable, &execution, &error));
g_thread_join(thread);
g_assert_error(error, G_IO_ERROR, G_IO_ERROR_CANCELLED);
g_assert_nonnull(execution);
g_assert_cmpint(execution->state, ==,
DOCUMENT_ANALYSIS_STATE_CANCELLED);
g_clear_error(&error);
document_tool_execution_free(execution);
g_object_unref(cancellable);
const char *success_arguments[] = {
"--emit", "--stdout-size", "1", NULL
};
execution = NULL;
g_assert_true(document_tool_runner_run_with_limits(
"synthetic", "tests/fake_document_tool", success_arguments, path,
&limits, NULL, &execution, NULL));
document_tool_execution_free(execution);
remove_source(directory, path);
}
static void test_repeated_runs_do_not_exhaust_descriptors(void)
{
char *directory = NULL;
char *path = create_source(&directory);
const char *arguments[] = {
"--emit", "--stdout-size", "8", "--stderr-size", "8", NULL
};
DocumentToolRunnerLimits limits = { 16, 16 };
struct rlimit original_limit;
g_assert_cmpint(getrlimit(RLIMIT_NOFILE, &original_limit), ==, 0);
struct rlimit test_limit = original_limit;
test_limit.rlim_cur = MIN(original_limit.rlim_cur, (rlim_t) 64);
g_assert_cmpint(setrlimit(RLIMIT_NOFILE, &test_limit), ==, 0);
for (guint index = 0; index < 96; index++)
{
DocumentToolExecution *execution = NULL;
g_assert_true(document_tool_runner_run_with_limits(
"synthetic", "tests/fake_document_tool", arguments, path,
&limits, NULL, &execution, NULL));
document_tool_execution_free(execution);
}
g_assert_cmpint(setrlimit(RLIMIT_NOFILE, &original_limit), ==, 0);
remove_source(directory, path);
}
int main(int argc, char **argv)
{
g_test_init(&argc, &argv, NULL);
g_test_add_func("/document-tool-runner/limits-concurrent",
test_limits_and_concurrent_drain);
g_test_add_func("/document-tool-runner/exact-below",
test_exact_and_below_limits);
g_test_add_func("/document-tool-runner/cancellation",
test_cancellation_during_output);
g_test_add_func("/document-tool-runner/no-descriptor-exhaustion",
test_repeated_runs_do_not_exhaust_descriptors);
return g_test_run();
}

96
tests/test_eml_pipeline_task.c
View file

@ -136,11 +136,107 @@ static void test_eml_pipeline_document_analysis(void)
g_free(tmp_dir);
}
static void wait_for_task(BackgroundTask *task)
{
while (background_task_get_state(task) ==
BACKGROUND_TASK_STATE_RUNNING ||
background_task_get_state(task) ==
BACKGROUND_TASK_STATE_PENDING)
g_main_context_iteration(NULL, TRUE);
}
static DocumentAnalysisTools synthetic_tools(void)
{
DocumentAnalysisTools tools = {
.exiftool = "tests/fake_document_tool",
.tesseract = "tests/fake_document_tool",
.pdfinfo = "tests/fake_document_tool",
.pdftotext = "tests/fake_document_tool",
.pdftoppm = "tests/fake_document_tool"
};
return tools;
}
static void test_eml_pipeline_pdf_end_to_end_and_limit(void)
{
GError *error = NULL;
char *tmp_dir = g_dir_make_tmp("labfy-eml-pdf-XXXXXX", &error);
g_assert_no_error(error);
char *eml_path = g_build_filename(tmp_dir, "pdf.eml", NULL);
char *processed_dir = g_build_filename(tmp_dir, "processed", NULL);
static const char eml[] =
"From: synthetic@example.test\r\n"
"MIME-Version: 1.0\r\n"
"Content-Type: multipart/mixed; boundary=pdf-boundary\r\n\r\n"
"--pdf-boundary\r\n"
"Content-Type: application/pdf; name=scan.pdf\r\n"
"Content-Disposition: attachment; filename=scan.pdf\r\n"
"Content-Transfer-Encoding: base64\r\n\r\n"
"JVBERi1zeW50aGV0aWM=\r\n"
"--pdf-boundary\r\n"
"Content-Type: application/pdf; name=second.pdf\r\n"
"Content-Disposition: attachment; filename=second.pdf\r\n"
"Content-Transfer-Encoding: base64\r\n\r\n"
"JVBERi1zeW50aGV0aWM=\r\n"
"--pdf-boundary--\r\n";
g_assert_true(g_file_set_contents(eml_path, eml, -1, &error));
g_assert_no_error(error);
char *source_before = NULL;
gsize source_before_length = 0;
g_assert_true(g_file_get_contents(
eml_path, &source_before, &source_before_length, &error));
g_assert_no_error(error);
DocumentAnalysisTools tools = synthetic_tools();
BackgroundTask *task = eml_pipeline_task_new_with_tools_and_limit(
eml_path, processed_dir, "synthetic-evidence", &tools, 1);
g_assert_nonnull(task);
wait_for_task(task);
g_assert_cmpint(background_task_get_state(task), ==,
BACKGROUND_TASK_STATE_COMPLETED);
EmlPipelineResult *result = background_task_get_result(task);
g_assert_nonnull(result);
g_assert_cmpuint(result->mime_result->attachments->len, ==, 2);
g_assert_cmpuint(result->document_analyses->len, ==, 1);
g_assert_cmpuint(result->skipped_document_analyses, ==, 1);
g_assert_cmpint(result->state, ==, DOCUMENT_ANALYSIS_STATE_PARTIAL);
g_assert_cmpuint(result->warnings->len, ==, 1);
DocumentFileAnalysis *document = g_ptr_array_index(
result->document_analyses, 0);
g_assert_nonnull(document->pdf);
g_assert_false(document->pdf->native_text_usable);
g_assert_cmpuint(document->pdf->pages->len, ==, 2);
PdfPageAnalysis *first = g_ptr_array_index(document->pdf->pages, 0);
PdfPageAnalysis *second = g_ptr_array_index(document->pdf->pages, 1);
g_assert_cmpuint(first->page_number, ==, 1);
g_assert_cmpuint(second->page_number, ==, 2);
g_assert_nonnull(first->render_execution);
g_assert_nonnull(first->execution);
char *source_after = NULL;
gsize source_after_length = 0;
g_assert_true(g_file_get_contents(
eml_path, &source_after, &source_after_length, &error));
g_assert_no_error(error);
g_assert_cmpuint(source_after_length, ==, source_before_length);
g_assert_cmpmem(source_after, source_after_length,
source_before, source_before_length);
g_free(source_after);
g_free(source_before);
background_task_unref(task);
g_remove(eml_path);
g_free(processed_dir);
g_free(eml_path);
g_free(tmp_dir);
}
int main(int argc, char **argv)
{
g_test_init(&argc, &argv, NULL);
g_test_add_func("/eml-pipeline-task/basic", test_eml_pipeline_basic);
g_test_add_func("/eml-pipeline-task/document-analysis",
test_eml_pipeline_document_analysis);
g_test_add_func("/eml-pipeline-task/pdf-end-to-end-limit",
test_eml_pipeline_pdf_end_to_end_and_limit);
return g_test_run();
}

50
tests/test_exiftool_analysis.c
View file

@ -68,6 +68,54 @@ static void test_run_and_unavailable(void)
g_free(directory);
}
static gpointer cancel_exiftool(gpointer user_data)
{
g_usleep(50000);
g_cancellable_cancel(user_data);
return NULL;
}
static void test_cancellation_and_truncated_json(void)
{
GError *error = NULL;
char *directory = g_dir_make_tmp("labfy-exif-hardening-XXXXXX", &error);
char *slow_path = g_build_filename(directory, "slow.png", NULL);
char *large_path = g_build_filename(directory, "large.png", NULL);
g_assert_true(g_file_set_contents(slow_path, "PNG", 3, &error));
g_assert_true(g_file_set_contents(large_path, "PNG", 3, &error));
GCancellable *cancellable = g_cancellable_new();
GThread *thread = g_thread_new(
"exif-cancel", cancel_exiftool, cancellable);
ExiftoolAnalysisResult *result = exiftool_analysis_run(
"tests/fake_document_tool", slow_path, cancellable, &error);
g_thread_join(thread);
g_assert_null(result);
g_assert_error(error, G_IO_ERROR, G_IO_ERROR_CANCELLED);
g_clear_error(&error);
g_object_unref(cancellable);
DocumentToolRunnerLimits limits = { 128, 128 };
result = exiftool_analysis_run_with_limits(
"tests/fake_document_tool", large_path, &limits, NULL, &error);
g_assert_no_error(error);
g_assert_nonnull(result);
g_assert_true(result->execution->stdout_truncated);
g_assert_cmpuint(strlen(result->execution->raw_stdout), ==, 128);
g_assert_cmpuint(result->metadata->len, ==, 0);
g_assert_cmpint(result->execution->state, ==,
DOCUMENT_ANALYSIS_STATE_FAILED);
g_assert_cmpuint(result->execution->warnings->len, >, 0);
exiftool_analysis_result_free(result);
g_remove(slow_path);
g_remove(large_path);
g_rmdir(directory);
g_free(slow_path);
g_free(large_path);
g_free(directory);
}
int main(int argc, char **argv)
{
g_test_init(&argc, &argv, NULL);
@ -76,5 +124,7 @@ int main(int argc, char **argv)
g_test_add_func("/exiftool-analysis/invalid-json", test_invalid_json);
g_test_add_func("/exiftool-analysis/run-unavailable",
test_run_and_unavailable);
g_test_add_func("/exiftool-analysis/cancellation-truncated",
test_cancellation_and_truncated_json);
return g_test_run();
}

24
tests/test_ocr_analysis.c
View file

@ -75,6 +75,29 @@ static void test_cancellation(void)
g_free(directory);
}
static void test_truncated_text(void)
{
GError *error = NULL;
char *directory = g_dir_make_tmp("labfy-ocr-limit-XXXXXX", &error);
char *path = g_build_filename(directory, "large.png", NULL);
g_assert_true(g_file_set_contents(path, "PNG", 3, &error));
DocumentToolRunnerLimits limits = { 96, 64 };
OcrAnalysisResult *result = ocr_analysis_run_with_limits(
"tests/fake_document_tool", path, "fra", &limits,
NULL, &error);
g_assert_no_error(error);
g_assert_nonnull(result);
g_assert_cmpuint(strlen(result->text), ==, 96);
g_assert_true(result->execution->stdout_truncated);
g_assert_cmpint(result->execution->state, ==,
DOCUMENT_ANALYSIS_STATE_PARTIAL);
ocr_analysis_result_free(result);
g_remove(path);
g_rmdir(directory);
g_free(path);
g_free(directory);
}
int main(int argc, char **argv)
{
g_test_init(&argc, &argv, NULL);
@ -83,5 +106,6 @@ int main(int argc, char **argv)
g_test_add_func("/ocr-analysis/unavailable-compatible",
test_unavailable_and_compatibility);
g_test_add_func("/ocr-analysis/cancellation", test_cancellation);
g_test_add_func("/ocr-analysis/truncated-text", test_truncated_text);
return g_test_run();
}

60
tests/test_pdf_analysis.c
View file

@ -84,6 +84,64 @@ static void test_heuristic(void)
"Texte synthétique imprimable et suffisamment long pour le test."));
}
static gpointer cancel_pdf(gpointer user_data)
{
g_usleep(70000);
g_cancellable_cancel(user_data);
return NULL;
}
static void assert_cancelled_pdf(
const PdfAnalysisTools *tools,
const char *path,
guint expected_completed_pages
)
{
GCancellable *cancellable = g_cancellable_new();
GThread *thread = g_thread_new("pdf-cancel", cancel_pdf, cancellable);
GError *error = NULL;
PdfAnalysisResult *result = pdf_analysis_run(
tools, path, "fra", cancellable, &error);
g_thread_join(thread);
g_assert_no_error(error);
g_assert_nonnull(result);
g_assert_cmpuint(result->pages->len, ==, expected_completed_pages);
g_assert_true(result->state == DOCUMENT_ANALYSIS_STATE_CANCELLED ||
result->state == DOCUMENT_ANALYSIS_STATE_PARTIAL);
pdf_analysis_result_free(result);
g_object_unref(cancellable);
}
static void test_cancellation_stages_and_completed_pages(void)
{
GError *error = NULL;
char *directory = g_dir_make_tmp("labfy-pdf-cancel-XXXXXX", &error);
g_assert_no_error(error);
PdfAnalysisTools tools = fake_tools();
char *slow_info = create_pdf(directory, "slow-info.pdf");
char *slow_text = create_pdf(directory, "slow-text.pdf");
char *slow_render = create_pdf(directory, "slow-render.pdf");
char *slow_ocr = create_pdf(directory, "slow-ocr.pdf");
char *slow_second = create_pdf(directory, "slow-page-2.pdf");
assert_cancelled_pdf(&tools, slow_info, 0);
assert_cancelled_pdf(&tools, slow_text, 0);
assert_cancelled_pdf(&tools, slow_render, 0);
assert_cancelled_pdf(&tools, slow_ocr, 1);
assert_cancelled_pdf(&tools, slow_second, 1);
g_remove(slow_info);
g_remove(slow_text);
g_remove(slow_render);
g_remove(slow_ocr);
g_remove(slow_second);
g_rmdir(directory);
g_free(slow_info);
g_free(slow_text);
g_free(slow_render);
g_free(slow_ocr);
g_free(slow_second);
g_free(directory);
}
int main(int argc, char **argv)
{
g_test_init(&argc, &argv, NULL);
@ -92,5 +150,7 @@ int main(int argc, char **argv)
g_test_add_func("/pdf-analysis/ocr-fallback-cleanup",
test_ocr_fallback_order_and_cleanup);
g_test_add_func("/pdf-analysis/heuristic", test_heuristic);
g_test_add_func("/pdf-analysis/cancellation-stages",
test_cancellation_stages_and_completed_pages);
return g_test_run();
}