fix: borner les sorties des outils documentaires
This commit is contained in:
parent
d96fc83fc0
commit
015841a20c
17 changed files with 897 additions and 101 deletions
12
Makefile
12
Makefile
|
|
@ -139,6 +139,7 @@ TEST_EML_MIME_EXTRACTOR := tests/test_eml_mime_extractor
|
||||||
TEST_EXIFTOOL_ANALYSIS := tests/test_exiftool_analysis
|
TEST_EXIFTOOL_ANALYSIS := tests/test_exiftool_analysis
|
||||||
TEST_OCR_ANALYSIS := tests/test_ocr_analysis
|
TEST_OCR_ANALYSIS := tests/test_ocr_analysis
|
||||||
TEST_PDF_ANALYSIS := tests/test_pdf_analysis
|
TEST_PDF_ANALYSIS := tests/test_pdf_analysis
|
||||||
|
TEST_DOCUMENT_TOOL_RUNNER := tests/test_document_tool_runner
|
||||||
FAKE_DOCUMENT_TOOL := tests/fake_document_tool
|
FAKE_DOCUMENT_TOOL := tests/fake_document_tool
|
||||||
|
|
||||||
DOCUMENT_ANALYSIS_TEST_SOURCES := \
|
DOCUMENT_ANALYSIS_TEST_SOURCES := \
|
||||||
|
|
@ -147,6 +148,12 @@ DOCUMENT_ANALYSIS_TEST_SOURCES := \
|
||||||
src/core/tool_process.c \
|
src/core/tool_process.c \
|
||||||
src/core/file_hash.c
|
src/core/file_hash.c
|
||||||
|
|
||||||
|
$(TEST_DOCUMENT_TOOL_RUNNER): tests/test_document_tool_runner.c \
|
||||||
|
$(DOCUMENT_ANALYSIS_TEST_SOURCES) $(FAKE_DOCUMENT_TOOL)
|
||||||
|
$(CC) $(TEST_CFLAGS) -Wpedantic \
|
||||||
|
tests/test_document_tool_runner.c \
|
||||||
|
$(DOCUMENT_ANALYSIS_TEST_SOURCES) -o $@ $(TEST_LDFLAGS)
|
||||||
|
|
||||||
all: $(TARGET)
|
all: $(TARGET)
|
||||||
|
|
||||||
$(TEST_BANK_PROPOSAL): \
|
$(TEST_BANK_PROPOSAL): \
|
||||||
|
|
@ -888,7 +895,8 @@ test: \
|
||||||
$(TEST_EML_MIME_EXTRACTOR) \
|
$(TEST_EML_MIME_EXTRACTOR) \
|
||||||
$(TEST_EXIFTOOL_ANALYSIS) \
|
$(TEST_EXIFTOOL_ANALYSIS) \
|
||||||
$(TEST_OCR_ANALYSIS) \
|
$(TEST_OCR_ANALYSIS) \
|
||||||
$(TEST_PDF_ANALYSIS)
|
$(TEST_PDF_ANALYSIS) \
|
||||||
|
$(TEST_DOCUMENT_TOOL_RUNNER)
|
||||||
@echo "Exécution des tests..."
|
@echo "Exécution des tests..."
|
||||||
@./$(TEST_NODE)
|
@./$(TEST_NODE)
|
||||||
@./$(TEST_TREE_MODEL)
|
@./$(TEST_TREE_MODEL)
|
||||||
|
|
@ -961,6 +969,7 @@ test: \
|
||||||
@$(TEST_EXIFTOOL_ANALYSIS)
|
@$(TEST_EXIFTOOL_ANALYSIS)
|
||||||
@$(TEST_OCR_ANALYSIS)
|
@$(TEST_OCR_ANALYSIS)
|
||||||
@$(TEST_PDF_ANALYSIS)
|
@$(TEST_PDF_ANALYSIS)
|
||||||
|
@$(TEST_DOCUMENT_TOOL_RUNNER)
|
||||||
@echo "Tous les tests sont valides."
|
@echo "Tous les tests sont valides."
|
||||||
|
|
||||||
%.o: %.c
|
%.o: %.c
|
||||||
|
|
@ -1037,6 +1046,7 @@ clean:
|
||||||
$(TEST_EXIFTOOL_ANALYSIS) \
|
$(TEST_EXIFTOOL_ANALYSIS) \
|
||||||
$(TEST_OCR_ANALYSIS) \
|
$(TEST_OCR_ANALYSIS) \
|
||||||
$(TEST_PDF_ANALYSIS) \
|
$(TEST_PDF_ANALYSIS) \
|
||||||
|
$(TEST_DOCUMENT_TOOL_RUNNER) \
|
||||||
$(FAKE_DOCUMENT_TOOL)
|
$(FAKE_DOCUMENT_TOOL)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -38,6 +38,10 @@ typedef struct
|
||||||
char *raw_stdout;
|
char *raw_stdout;
|
||||||
char *raw_stdout_sha256;
|
char *raw_stdout_sha256;
|
||||||
char *raw_stderr;
|
char *raw_stderr;
|
||||||
|
gsize stdout_bytes_observed;
|
||||||
|
gsize stderr_bytes_observed;
|
||||||
|
gboolean stdout_truncated;
|
||||||
|
gboolean stderr_truncated;
|
||||||
int exit_status;
|
int exit_status;
|
||||||
DocumentAnalysisState state;
|
DocumentAnalysisState state;
|
||||||
GPtrArray *warnings;
|
GPtrArray *warnings;
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,12 @@
|
||||||
|
|
||||||
G_BEGIN_DECLS
|
G_BEGIN_DECLS
|
||||||
|
|
||||||
|
typedef struct
|
||||||
|
{
|
||||||
|
gsize stdout_limit;
|
||||||
|
gsize stderr_limit;
|
||||||
|
} DocumentToolRunnerLimits;
|
||||||
|
|
||||||
gboolean document_tool_runner_run(
|
gboolean document_tool_runner_run(
|
||||||
const char *tool_id,
|
const char *tool_id,
|
||||||
const char *executable,
|
const char *executable,
|
||||||
|
|
@ -19,6 +25,17 @@ gboolean document_tool_runner_run(
|
||||||
GError **error
|
GError **error
|
||||||
);
|
);
|
||||||
|
|
||||||
|
gboolean document_tool_runner_run_with_limits(
|
||||||
|
const char *tool_id,
|
||||||
|
const char *executable,
|
||||||
|
const char *const arguments[],
|
||||||
|
const char *source_path,
|
||||||
|
const DocumentToolRunnerLimits *limits,
|
||||||
|
GCancellable *cancellable,
|
||||||
|
DocumentToolExecution **out_execution,
|
||||||
|
GError **error
|
||||||
|
);
|
||||||
|
|
||||||
char *document_tool_runner_read_version(
|
char *document_tool_runner_read_version(
|
||||||
const char *executable,
|
const char *executable,
|
||||||
const char *const arguments[],
|
const char *const arguments[],
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,8 @@ typedef struct EmlPipelineResult
|
||||||
GPtrArray *bank_proposals; /**< Tableau de BankProposal* */
|
GPtrArray *bank_proposals; /**< Tableau de BankProposal* */
|
||||||
GPtrArray *document_analyses; /**< Tableau de DocumentFileAnalysis* */
|
GPtrArray *document_analyses; /**< Tableau de DocumentFileAnalysis* */
|
||||||
GPtrArray *warnings; /**< Avertissements globaux */
|
GPtrArray *warnings; /**< Avertissements globaux */
|
||||||
|
DocumentAnalysisState state;
|
||||||
|
guint skipped_document_analyses;
|
||||||
} EmlPipelineResult;
|
} EmlPipelineResult;
|
||||||
|
|
||||||
/** @brief Libère un résultat EmlPipelineResult. */
|
/** @brief Libère un résultat EmlPipelineResult. */
|
||||||
|
|
@ -44,6 +46,14 @@ BackgroundTask *eml_pipeline_task_new_with_tools(
|
||||||
const DocumentAnalysisTools *tools
|
const DocumentAnalysisTools *tools
|
||||||
);
|
);
|
||||||
|
|
||||||
|
BackgroundTask *eml_pipeline_task_new_with_tools_and_limit(
|
||||||
|
const char *eml_path,
|
||||||
|
const char *processed_evidence_dir,
|
||||||
|
const char *evidence_id,
|
||||||
|
const DocumentAnalysisTools *tools,
|
||||||
|
guint document_analysis_limit
|
||||||
|
);
|
||||||
|
|
||||||
G_END_DECLS
|
G_END_DECLS
|
||||||
|
|
||||||
#endif /* LABFY_INVESTIGATION_EML_PIPELINE_TASK_H */
|
#endif /* LABFY_INVESTIGATION_EML_PIPELINE_TASK_H */
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@
|
||||||
#define LABFY_INVESTIGATION_EXIFTOOL_ANALYSIS_H
|
#define LABFY_INVESTIGATION_EXIFTOOL_ANALYSIS_H
|
||||||
|
|
||||||
#include "core/document_analysis.h"
|
#include "core/document_analysis.h"
|
||||||
|
#include "core/document_tool_runner.h"
|
||||||
|
|
||||||
G_BEGIN_DECLS
|
G_BEGIN_DECLS
|
||||||
|
|
||||||
|
|
@ -21,6 +22,13 @@ ExiftoolAnalysisResult *exiftool_analysis_run(
|
||||||
GCancellable *cancellable,
|
GCancellable *cancellable,
|
||||||
GError **error
|
GError **error
|
||||||
);
|
);
|
||||||
|
ExiftoolAnalysisResult *exiftool_analysis_run_with_limits(
|
||||||
|
const char *executable,
|
||||||
|
const char *file_path,
|
||||||
|
const DocumentToolRunnerLimits *limits,
|
||||||
|
GCancellable *cancellable,
|
||||||
|
GError **error
|
||||||
|
);
|
||||||
ExiftoolAnalysisResult *exiftool_analysis_parse(
|
ExiftoolAnalysisResult *exiftool_analysis_parse(
|
||||||
const char *file_path,
|
const char *file_path,
|
||||||
const char *json,
|
const char *json,
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@
|
||||||
#define LABFY_INVESTIGATION_OCR_ANALYSIS_H
|
#define LABFY_INVESTIGATION_OCR_ANALYSIS_H
|
||||||
|
|
||||||
#include "core/document_analysis.h"
|
#include "core/document_analysis.h"
|
||||||
|
#include "core/document_tool_runner.h"
|
||||||
|
|
||||||
G_BEGIN_DECLS
|
G_BEGIN_DECLS
|
||||||
|
|
||||||
|
|
@ -23,6 +24,14 @@ OcrAnalysisResult *ocr_analysis_run(
|
||||||
GCancellable *cancellable,
|
GCancellable *cancellable,
|
||||||
GError **error
|
GError **error
|
||||||
);
|
);
|
||||||
|
OcrAnalysisResult *ocr_analysis_run_with_limits(
|
||||||
|
const char *executable,
|
||||||
|
const char *image_path,
|
||||||
|
const char *languages,
|
||||||
|
const DocumentToolRunnerLimits *limits,
|
||||||
|
GCancellable *cancellable,
|
||||||
|
GError **error
|
||||||
|
);
|
||||||
void ocr_analysis_result_free(OcrAnalysisResult *result);
|
void ocr_analysis_result_free(OcrAnalysisResult *result);
|
||||||
gboolean ocr_analysis_mime_is_compatible(const char *mime_type);
|
gboolean ocr_analysis_mime_is_compatible(const char *mime_type);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,90 +1,232 @@
|
||||||
/******************************************************************************
|
/******************************************************************************
|
||||||
* @file document_tool_runner.c
|
* @file document_tool_runner.c
|
||||||
* @brief Exécution bornée et annulable des outils documentaires.
|
* @brief Exécution réellement bornée et annulable des outils documentaires.
|
||||||
******************************************************************************/
|
******************************************************************************/
|
||||||
#include "core/document_tool_runner.h"
|
#include "core/document_tool_runner.h"
|
||||||
#include "core/file_hash.h"
|
#include "core/file_hash.h"
|
||||||
#include "core/tool_process.h"
|
|
||||||
#include <glib/gstdio.h>
|
#include <glib/gstdio.h>
|
||||||
|
|
||||||
|
#define DOCUMENT_TOOL_RUNNER_READ_BLOCK 4096U
|
||||||
|
#define DOCUMENT_TOOL_RUNNER_VERSION_LIMIT 65536U
|
||||||
|
|
||||||
|
typedef struct
|
||||||
|
{
|
||||||
|
GInputStream *stream;
|
||||||
|
GCancellable *cancellable;
|
||||||
|
GByteArray *prefix;
|
||||||
|
gsize limit;
|
||||||
|
gsize bytes_observed;
|
||||||
|
gboolean truncated;
|
||||||
|
GError *error;
|
||||||
|
} DocumentToolStreamCapture;
|
||||||
|
|
||||||
|
typedef struct
|
||||||
|
{
|
||||||
|
GBytes *stdout_bytes;
|
||||||
|
GBytes *stderr_bytes;
|
||||||
|
gsize stdout_bytes_observed;
|
||||||
|
gsize stderr_bytes_observed;
|
||||||
|
gboolean stdout_truncated;
|
||||||
|
gboolean stderr_truncated;
|
||||||
|
gboolean exited_normally;
|
||||||
|
int exit_status;
|
||||||
|
} DocumentToolCaptureResult;
|
||||||
|
|
||||||
|
static GPtrArray *document_tool_runner_build_argv(
|
||||||
|
const char *executable,
|
||||||
|
const char *const arguments[]
|
||||||
|
)
|
||||||
|
{
|
||||||
|
GPtrArray *argv = g_ptr_array_new();
|
||||||
|
g_ptr_array_add(argv, (gpointer) executable);
|
||||||
|
for (gsize index = 0; arguments != NULL &&
|
||||||
|
arguments[index] != NULL; index++)
|
||||||
|
g_ptr_array_add(argv, (gpointer) arguments[index]);
|
||||||
|
g_ptr_array_add(argv, NULL);
|
||||||
|
return argv;
|
||||||
|
}
|
||||||
|
|
||||||
|
static gpointer document_tool_runner_drain_stream(gpointer user_data)
|
||||||
|
{
|
||||||
|
DocumentToolStreamCapture *capture = user_data;
|
||||||
|
guint8 block[DOCUMENT_TOOL_RUNNER_READ_BLOCK];
|
||||||
|
|
||||||
|
while (TRUE)
|
||||||
|
{
|
||||||
|
gssize bytes_read = g_input_stream_read(
|
||||||
|
capture->stream, block, sizeof(block),
|
||||||
|
capture->cancellable, &capture->error);
|
||||||
|
if (bytes_read <= 0)
|
||||||
|
break;
|
||||||
|
|
||||||
|
gsize observed = (gsize) bytes_read;
|
||||||
|
if (G_MAXSIZE - capture->bytes_observed < observed)
|
||||||
|
capture->bytes_observed = G_MAXSIZE;
|
||||||
|
else
|
||||||
|
capture->bytes_observed += observed;
|
||||||
|
|
||||||
|
gsize remaining = capture->prefix->len < capture->limit
|
||||||
|
? capture->limit - capture->prefix->len : 0;
|
||||||
|
gsize retained = MIN(remaining, observed);
|
||||||
|
if (retained > 0)
|
||||||
|
g_byte_array_append(capture->prefix, block, retained);
|
||||||
|
if (retained < observed)
|
||||||
|
capture->truncated = TRUE;
|
||||||
|
}
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void document_tool_capture_result_clear(
|
||||||
|
DocumentToolCaptureResult *result
|
||||||
|
)
|
||||||
|
{
|
||||||
|
g_clear_pointer(&result->stdout_bytes, g_bytes_unref);
|
||||||
|
g_clear_pointer(&result->stderr_bytes, g_bytes_unref);
|
||||||
|
}
|
||||||
|
|
||||||
|
static gboolean document_tool_runner_capture(
|
||||||
|
const char *executable,
|
||||||
|
const char *const arguments[],
|
||||||
|
const DocumentToolRunnerLimits *limits,
|
||||||
|
GCancellable *cancellable,
|
||||||
|
DocumentToolCaptureResult *out_result,
|
||||||
|
GError **error
|
||||||
|
)
|
||||||
|
{
|
||||||
|
GPtrArray *argv = document_tool_runner_build_argv(executable, arguments);
|
||||||
|
GSubprocessLauncher *launcher = g_subprocess_launcher_new(
|
||||||
|
G_SUBPROCESS_FLAGS_STDOUT_PIPE | G_SUBPROCESS_FLAGS_STDERR_PIPE);
|
||||||
|
GError *local_error = NULL;
|
||||||
|
GSubprocess *process = g_subprocess_launcher_spawnv(
|
||||||
|
launcher, (const char *const *) argv->pdata, &local_error);
|
||||||
|
g_object_unref(launcher);
|
||||||
|
g_ptr_array_unref(argv);
|
||||||
|
if (process == NULL)
|
||||||
|
{
|
||||||
|
g_propagate_error(error, local_error);
|
||||||
|
return FALSE;
|
||||||
|
}
|
||||||
|
|
||||||
|
DocumentToolStreamCapture stdout_capture = {
|
||||||
|
.stream = g_subprocess_get_stdout_pipe(process),
|
||||||
|
.cancellable = cancellable,
|
||||||
|
.prefix = g_byte_array_sized_new(MIN(limits->stdout_limit, 4096U)),
|
||||||
|
.limit = limits->stdout_limit
|
||||||
|
};
|
||||||
|
DocumentToolStreamCapture stderr_capture = {
|
||||||
|
.stream = g_subprocess_get_stderr_pipe(process),
|
||||||
|
.cancellable = cancellable,
|
||||||
|
.prefix = g_byte_array_sized_new(MIN(limits->stderr_limit, 4096U)),
|
||||||
|
.limit = limits->stderr_limit
|
||||||
|
};
|
||||||
|
GThread *stdout_thread = g_thread_new(
|
||||||
|
"document-stdout", document_tool_runner_drain_stream, &stdout_capture);
|
||||||
|
GThread *stderr_thread = g_thread_new(
|
||||||
|
"document-stderr", document_tool_runner_drain_stream, &stderr_capture);
|
||||||
|
|
||||||
|
gboolean waited = g_subprocess_wait(process, cancellable, &local_error);
|
||||||
|
if (!waited)
|
||||||
|
{
|
||||||
|
g_subprocess_force_exit(process);
|
||||||
|
GError *final_wait_error = NULL;
|
||||||
|
(void) g_subprocess_wait(process, NULL, &final_wait_error);
|
||||||
|
g_clear_error(&final_wait_error);
|
||||||
|
}
|
||||||
|
g_thread_join(stdout_thread);
|
||||||
|
g_thread_join(stderr_thread);
|
||||||
|
|
||||||
|
gboolean cancelled =
|
||||||
|
(cancellable != NULL && g_cancellable_is_cancelled(cancellable)) ||
|
||||||
|
g_error_matches(local_error, G_IO_ERROR, G_IO_ERROR_CANCELLED) ||
|
||||||
|
g_error_matches(stdout_capture.error, G_IO_ERROR, G_IO_ERROR_CANCELLED) ||
|
||||||
|
g_error_matches(stderr_capture.error, G_IO_ERROR, G_IO_ERROR_CANCELLED);
|
||||||
|
|
||||||
|
if (!waited || stdout_capture.error != NULL ||
|
||||||
|
stderr_capture.error != NULL)
|
||||||
|
{
|
||||||
|
if (cancelled)
|
||||||
|
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_CANCELLED,
|
||||||
|
"L'exécution de l'outil documentaire a été annulée.");
|
||||||
|
else if (local_error != NULL)
|
||||||
|
g_propagate_error(error, g_steal_pointer(&local_error));
|
||||||
|
else
|
||||||
|
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||||
|
"La lecture des sorties de l'outil documentaire a échoué.");
|
||||||
|
g_clear_error(&local_error);
|
||||||
|
g_clear_error(&stdout_capture.error);
|
||||||
|
g_clear_error(&stderr_capture.error);
|
||||||
|
g_byte_array_unref(stdout_capture.prefix);
|
||||||
|
g_byte_array_unref(stderr_capture.prefix);
|
||||||
|
g_object_unref(process);
|
||||||
|
return FALSE;
|
||||||
|
}
|
||||||
|
|
||||||
|
out_result->stdout_bytes = g_byte_array_free_to_bytes(
|
||||||
|
stdout_capture.prefix);
|
||||||
|
out_result->stderr_bytes = g_byte_array_free_to_bytes(
|
||||||
|
stderr_capture.prefix);
|
||||||
|
out_result->stdout_bytes_observed = stdout_capture.bytes_observed;
|
||||||
|
out_result->stderr_bytes_observed = stderr_capture.bytes_observed;
|
||||||
|
out_result->stdout_truncated = stdout_capture.truncated;
|
||||||
|
out_result->stderr_truncated = stderr_capture.truncated;
|
||||||
|
out_result->exited_normally = g_subprocess_get_if_exited(process);
|
||||||
|
out_result->exit_status = out_result->exited_normally
|
||||||
|
? g_subprocess_get_exit_status(process) : -1;
|
||||||
|
g_clear_error(&local_error);
|
||||||
|
g_object_unref(process);
|
||||||
|
return TRUE;
|
||||||
|
}
|
||||||
|
|
||||||
|
static char *document_tool_runner_bytes_to_text(GBytes *bytes)
|
||||||
|
{
|
||||||
|
gsize length = 0;
|
||||||
|
const char *data = g_bytes_get_data(bytes, &length);
|
||||||
|
return g_utf8_make_valid(data != NULL ? data : "", (gssize) length);
|
||||||
|
}
|
||||||
|
|
||||||
char *document_tool_runner_read_version(
|
char *document_tool_runner_read_version(
|
||||||
const char *executable,
|
const char *executable,
|
||||||
const char *const arguments[],
|
const char *const arguments[],
|
||||||
GCancellable *cancellable
|
GCancellable *cancellable
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
ToolProcessResult *result = NULL;
|
DocumentToolRunnerLimits limits = {
|
||||||
|
DOCUMENT_TOOL_RUNNER_VERSION_LIMIT,
|
||||||
|
DOCUMENT_TOOL_RUNNER_VERSION_LIMIT
|
||||||
|
};
|
||||||
|
DocumentToolCaptureResult capture = { 0 };
|
||||||
GError *error = NULL;
|
GError *error = NULL;
|
||||||
char *version = NULL;
|
if (!document_tool_runner_capture(executable, arguments, &limits,
|
||||||
if (!tool_process_run(executable, arguments, NULL, cancellable,
|
cancellable, &capture, &error))
|
||||||
&result, &error))
|
|
||||||
{
|
{
|
||||||
g_clear_error(&error);
|
g_clear_error(&error);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
GBytes *stdout_bytes = tool_process_result_ref_stdout(result);
|
char *version = document_tool_runner_bytes_to_text(
|
||||||
GBytes *stderr_bytes = tool_process_result_ref_stderr(result);
|
g_bytes_get_size(capture.stdout_bytes) > 0
|
||||||
gsize stdout_length = 0;
|
? capture.stdout_bytes : capture.stderr_bytes);
|
||||||
gsize stderr_length = 0;
|
|
||||||
const char *stdout_data = g_bytes_get_data(
|
|
||||||
stdout_bytes, &stdout_length);
|
|
||||||
const char *stderr_data = g_bytes_get_data(
|
|
||||||
stderr_bytes, &stderr_length);
|
|
||||||
if (stdout_data == NULL)
|
|
||||||
stdout_data = "";
|
|
||||||
if (stderr_data == NULL)
|
|
||||||
stderr_data = "";
|
|
||||||
if (stdout_length > 0)
|
|
||||||
version = g_utf8_make_valid(stdout_data, (gssize) stdout_length);
|
|
||||||
else if (stderr_length > 0)
|
|
||||||
version = g_utf8_make_valid(stderr_data, (gssize) stderr_length);
|
|
||||||
if (version != NULL)
|
|
||||||
g_strstrip(version);
|
g_strstrip(version);
|
||||||
g_bytes_unref(stdout_bytes);
|
document_tool_capture_result_clear(&capture);
|
||||||
g_bytes_unref(stderr_bytes);
|
|
||||||
tool_process_result_free(result);
|
|
||||||
return version;
|
return version;
|
||||||
}
|
}
|
||||||
|
|
||||||
static char *document_tool_runner_bytes_to_text(
|
gboolean document_tool_runner_run_with_limits(
|
||||||
GBytes *bytes,
|
|
||||||
gsize limit,
|
|
||||||
gboolean *truncated
|
|
||||||
)
|
|
||||||
{
|
|
||||||
gsize length = 0;
|
|
||||||
const char *data = bytes != NULL
|
|
||||||
? g_bytes_get_data(bytes, &length)
|
|
||||||
: "";
|
|
||||||
if (data == NULL)
|
|
||||||
data = "";
|
|
||||||
if (length > limit)
|
|
||||||
{
|
|
||||||
length = limit;
|
|
||||||
*truncated = TRUE;
|
|
||||||
}
|
|
||||||
return g_utf8_make_valid(data, (gssize) length);
|
|
||||||
}
|
|
||||||
|
|
||||||
gboolean document_tool_runner_run(
|
|
||||||
const char *tool_id,
|
const char *tool_id,
|
||||||
const char *executable,
|
const char *executable,
|
||||||
const char *const arguments[],
|
const char *const arguments[],
|
||||||
const char *source_path,
|
const char *source_path,
|
||||||
|
const DocumentToolRunnerLimits *limits,
|
||||||
GCancellable *cancellable,
|
GCancellable *cancellable,
|
||||||
DocumentToolExecution **out_execution,
|
DocumentToolExecution **out_execution,
|
||||||
GError **error
|
GError **error
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
ToolProcessResult *process_result = NULL;
|
|
||||||
DocumentToolExecution *execution = NULL;
|
|
||||||
GError *process_error = NULL;
|
|
||||||
gboolean stdout_truncated = FALSE;
|
|
||||||
gboolean stderr_truncated = FALSE;
|
|
||||||
|
|
||||||
g_return_val_if_fail(error == NULL || *error == NULL, FALSE);
|
g_return_val_if_fail(error == NULL || *error == NULL, FALSE);
|
||||||
if (tool_id == NULL || executable == NULL || source_path == NULL ||
|
if (tool_id == NULL || executable == NULL || source_path == NULL ||
|
||||||
out_execution == NULL || *out_execution != NULL)
|
limits == NULL || limits->stdout_limit == 0 ||
|
||||||
|
limits->stderr_limit == 0 || out_execution == NULL ||
|
||||||
|
*out_execution != NULL)
|
||||||
{
|
{
|
||||||
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
|
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
|
||||||
"Les paramètres de l'outil documentaire sont invalides.");
|
"Les paramètres de l'outil documentaire sont invalides.");
|
||||||
|
|
@ -98,31 +240,34 @@ gboolean document_tool_runner_run(
|
||||||
"Le fichier dépasse la taille maximale d'analyse.");
|
"Le fichier dépasse la taille maximale d'analyse.");
|
||||||
return FALSE;
|
return FALSE;
|
||||||
}
|
}
|
||||||
execution = document_tool_execution_new(tool_id, source_path);
|
|
||||||
|
DocumentToolExecution *execution =
|
||||||
|
document_tool_execution_new(tool_id, source_path);
|
||||||
for (gsize index = 0; arguments != NULL &&
|
for (gsize index = 0; arguments != NULL &&
|
||||||
arguments[index] != NULL; index++)
|
arguments[index] != NULL; index++)
|
||||||
document_tool_execution_add_argument(execution, arguments[index]);
|
document_tool_execution_add_argument(execution, arguments[index]);
|
||||||
(void) file_hash_compute_sha256(source_path, cancellable,
|
(void) file_hash_compute_sha256(source_path, cancellable,
|
||||||
&execution->source_sha256, NULL, NULL);
|
&execution->source_sha256, NULL, NULL);
|
||||||
|
|
||||||
if (!tool_process_run(executable, arguments, NULL, cancellable,
|
DocumentToolCaptureResult capture = { 0 };
|
||||||
&process_result, &process_error))
|
GError *capture_error = NULL;
|
||||||
|
if (!document_tool_runner_capture(executable, arguments, limits,
|
||||||
|
cancellable, &capture, &capture_error))
|
||||||
{
|
{
|
||||||
if (g_error_matches(process_error, TOOL_PROCESS_ERROR,
|
if (g_error_matches(capture_error, G_IO_ERROR, G_IO_ERROR_CANCELLED))
|
||||||
TOOL_PROCESS_ERROR_CANCELLED))
|
|
||||||
{
|
{
|
||||||
execution->state = DOCUMENT_ANALYSIS_STATE_CANCELLED;
|
execution->state = DOCUMENT_ANALYSIS_STATE_CANCELLED;
|
||||||
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_CANCELLED,
|
g_propagate_error(error, capture_error);
|
||||||
"L'analyse documentaire a été annulée.");
|
capture_error = NULL;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
execution->state = DOCUMENT_ANALYSIS_STATE_UNAVAILABLE;
|
execution->state = DOCUMENT_ANALYSIS_STATE_UNAVAILABLE;
|
||||||
g_ptr_array_add(execution->errors,
|
g_ptr_array_add(execution->errors, g_strdup(
|
||||||
g_strdup(process_error != NULL ? process_error->message :
|
capture_error != NULL ? capture_error->message :
|
||||||
"Outil indisponible."));
|
"Outil indisponible."));
|
||||||
}
|
}
|
||||||
g_clear_error(&process_error);
|
g_clear_error(&capture_error);
|
||||||
GDateTime *now = g_date_time_new_now_utc();
|
GDateTime *now = g_date_time_new_now_utc();
|
||||||
execution->finished_at_utc = g_date_time_format_iso8601(now);
|
execution->finished_at_utc = g_date_time_format_iso8601(now);
|
||||||
g_date_time_unref(now);
|
g_date_time_unref(now);
|
||||||
|
|
@ -130,33 +275,53 @@ gboolean document_tool_runner_run(
|
||||||
return execution->state != DOCUMENT_ANALYSIS_STATE_CANCELLED;
|
return execution->state != DOCUMENT_ANALYSIS_STATE_CANCELLED;
|
||||||
}
|
}
|
||||||
|
|
||||||
GBytes *stdout_bytes = tool_process_result_ref_stdout(process_result);
|
execution->raw_stdout =
|
||||||
GBytes *stderr_bytes = tool_process_result_ref_stderr(process_result);
|
document_tool_runner_bytes_to_text(capture.stdout_bytes);
|
||||||
execution->raw_stdout = document_tool_runner_bytes_to_text(stdout_bytes,
|
execution->raw_stderr =
|
||||||
DOCUMENT_ANALYSIS_MAX_STDOUT, &stdout_truncated);
|
document_tool_runner_bytes_to_text(capture.stderr_bytes);
|
||||||
execution->raw_stderr = document_tool_runner_bytes_to_text(stderr_bytes,
|
execution->stdout_bytes_observed = capture.stdout_bytes_observed;
|
||||||
DOCUMENT_ANALYSIS_MAX_STDERR, &stderr_truncated);
|
execution->stderr_bytes_observed = capture.stderr_bytes_observed;
|
||||||
execution->exit_status =
|
execution->stdout_truncated = capture.stdout_truncated;
|
||||||
tool_process_result_get_exit_status(process_result);
|
execution->stderr_truncated = capture.stderr_truncated;
|
||||||
if (execution->raw_stdout != NULL)
|
execution->exit_status = capture.exit_status;
|
||||||
execution->raw_stdout_sha256 = g_compute_checksum_for_string(
|
execution->raw_stdout_sha256 = g_compute_checksum_for_string(
|
||||||
G_CHECKSUM_SHA256, execution->raw_stdout, -1);
|
G_CHECKSUM_SHA256, execution->raw_stdout, -1);
|
||||||
if (stdout_truncated || stderr_truncated)
|
|
||||||
{
|
if (execution->stdout_truncated)
|
||||||
execution->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
|
|
||||||
g_ptr_array_add(execution->warnings,
|
g_ptr_array_add(execution->warnings,
|
||||||
g_strdup("La sortie de l'outil a été tronquée à la limite."));
|
g_strdup("La sortie standard de l'outil a été tronquée."));
|
||||||
}
|
if (execution->stderr_truncated)
|
||||||
else
|
g_ptr_array_add(execution->warnings,
|
||||||
execution->state = tool_process_result_is_success(process_result)
|
g_strdup("La sortie d'erreur de l'outil a été tronquée."));
|
||||||
|
execution->state =
|
||||||
|
execution->stdout_truncated || execution->stderr_truncated
|
||||||
|
? DOCUMENT_ANALYSIS_STATE_PARTIAL
|
||||||
|
: capture.exited_normally && capture.exit_status == 0
|
||||||
? DOCUMENT_ANALYSIS_STATE_SUCCESS
|
? DOCUMENT_ANALYSIS_STATE_SUCCESS
|
||||||
: DOCUMENT_ANALYSIS_STATE_FAILED;
|
: DOCUMENT_ANALYSIS_STATE_FAILED;
|
||||||
|
|
||||||
GDateTime *now = g_date_time_new_now_utc();
|
GDateTime *now = g_date_time_new_now_utc();
|
||||||
execution->finished_at_utc = g_date_time_format_iso8601(now);
|
execution->finished_at_utc = g_date_time_format_iso8601(now);
|
||||||
g_date_time_unref(now);
|
g_date_time_unref(now);
|
||||||
g_clear_pointer(&stdout_bytes, g_bytes_unref);
|
document_tool_capture_result_clear(&capture);
|
||||||
g_clear_pointer(&stderr_bytes, g_bytes_unref);
|
|
||||||
tool_process_result_free(process_result);
|
|
||||||
*out_execution = execution;
|
*out_execution = execution;
|
||||||
return TRUE;
|
return TRUE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
gboolean document_tool_runner_run(
|
||||||
|
const char *tool_id,
|
||||||
|
const char *executable,
|
||||||
|
const char *const arguments[],
|
||||||
|
const char *source_path,
|
||||||
|
GCancellable *cancellable,
|
||||||
|
DocumentToolExecution **out_execution,
|
||||||
|
GError **error
|
||||||
|
)
|
||||||
|
{
|
||||||
|
const DocumentToolRunnerLimits limits = {
|
||||||
|
DOCUMENT_ANALYSIS_MAX_STDOUT,
|
||||||
|
DOCUMENT_ANALYSIS_MAX_STDERR
|
||||||
|
};
|
||||||
|
return document_tool_runner_run_with_limits(tool_id, executable,
|
||||||
|
arguments, source_path, &limits, cancellable, out_execution, error);
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@ typedef struct
|
||||||
char *pdfinfo;
|
char *pdfinfo;
|
||||||
char *pdftotext;
|
char *pdftotext;
|
||||||
char *pdftoppm;
|
char *pdftoppm;
|
||||||
|
guint document_analysis_limit;
|
||||||
} EmlPipelineTaskData;
|
} EmlPipelineTaskData;
|
||||||
|
|
||||||
static void eml_pipeline_task_data_free(gpointer user_data)
|
static void eml_pipeline_task_data_free(gpointer user_data)
|
||||||
|
|
@ -116,18 +117,19 @@ static gboolean eml_pipeline_task_worker(BackgroundTask *task,
|
||||||
GPtrArray *bank_proposals = g_ptr_array_new_with_free_func((GDestroyNotify) bank_proposal_free);
|
GPtrArray *bank_proposals = g_ptr_array_new_with_free_func((GDestroyNotify) bank_proposal_free);
|
||||||
GPtrArray *document_analyses = g_ptr_array_new_with_free_func(
|
GPtrArray *document_analyses = g_ptr_array_new_with_free_func(
|
||||||
(GDestroyNotify) document_file_analysis_free);
|
(GDestroyNotify) document_file_analysis_free);
|
||||||
|
GPtrArray *pipeline_warnings =
|
||||||
|
g_ptr_array_new_with_free_func(g_free);
|
||||||
|
guint skipped_document_analyses = 0;
|
||||||
|
|
||||||
for (guint i = 0; mime_res->attachments != NULL && i < mime_res->attachments->len; i++)
|
for (guint i = 0; mime_res->attachments != NULL && i < mime_res->attachments->len; i++)
|
||||||
{
|
{
|
||||||
if (document_analyses->len >=
|
|
||||||
DOCUMENT_ANALYSIS_MAX_PIPELINE_ITEMS)
|
|
||||||
break;
|
|
||||||
if (g_cancellable_is_cancelled(cancellable))
|
if (g_cancellable_is_cancelled(cancellable))
|
||||||
{
|
{
|
||||||
eml_analysis_free(analysis);
|
eml_analysis_free(analysis);
|
||||||
eml_mime_result_free(mime_res);
|
eml_mime_result_free(mime_res);
|
||||||
g_ptr_array_unref(bank_proposals);
|
g_ptr_array_unref(bank_proposals);
|
||||||
g_ptr_array_unref(document_analyses);
|
g_ptr_array_unref(document_analyses);
|
||||||
|
g_ptr_array_unref(pipeline_warnings);
|
||||||
g_set_error_literal(
|
g_set_error_literal(
|
||||||
error,
|
error,
|
||||||
G_IO_ERROR,
|
G_IO_ERROR,
|
||||||
|
|
@ -160,6 +162,11 @@ static gboolean eml_pipeline_task_worker(BackgroundTask *task,
|
||||||
g_strcmp0(att->detected_mime, "application/pdf") == 0 ||
|
g_strcmp0(att->detected_mime, "application/pdf") == 0 ||
|
||||||
g_strcmp0(att->content_type, "application/pdf") == 0)
|
g_strcmp0(att->content_type, "application/pdf") == 0)
|
||||||
{
|
{
|
||||||
|
if (document_analyses->len >= data->document_analysis_limit)
|
||||||
|
{
|
||||||
|
skipped_document_analyses++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
DocumentAnalysisTools tools = {
|
DocumentAnalysisTools tools = {
|
||||||
.exiftool = data->exiftool,
|
.exiftool = data->exiftool,
|
||||||
.tesseract = data->tesseract,
|
.tesseract = data->tesseract,
|
||||||
|
|
@ -182,6 +189,7 @@ static gboolean eml_pipeline_task_worker(BackgroundTask *task,
|
||||||
eml_mime_result_free(mime_res);
|
eml_mime_result_free(mime_res);
|
||||||
g_ptr_array_unref(bank_proposals);
|
g_ptr_array_unref(bank_proposals);
|
||||||
g_ptr_array_unref(document_analyses);
|
g_ptr_array_unref(document_analyses);
|
||||||
|
g_ptr_array_unref(pipeline_warnings);
|
||||||
return FALSE;
|
return FALSE;
|
||||||
}
|
}
|
||||||
g_clear_error(&analysis_error);
|
g_clear_error(&analysis_error);
|
||||||
|
|
@ -210,7 +218,16 @@ static gboolean eml_pipeline_task_worker(BackgroundTask *task,
|
||||||
res->mime_result = mime_res;
|
res->mime_result = mime_res;
|
||||||
res->bank_proposals = bank_proposals;
|
res->bank_proposals = bank_proposals;
|
||||||
res->document_analyses = document_analyses;
|
res->document_analyses = document_analyses;
|
||||||
res->warnings = g_ptr_array_new_with_free_func(g_free);
|
res->warnings = pipeline_warnings;
|
||||||
|
res->skipped_document_analyses = skipped_document_analyses;
|
||||||
|
res->state = skipped_document_analyses > 0
|
||||||
|
? DOCUMENT_ANALYSIS_STATE_PARTIAL
|
||||||
|
: DOCUMENT_ANALYSIS_STATE_SUCCESS;
|
||||||
|
if (skipped_document_analyses > 0)
|
||||||
|
g_ptr_array_add(res->warnings, g_strdup_printf(
|
||||||
|
"La limite d'analyses documentaires est atteinte : "
|
||||||
|
"%u fichier(s) n'ont pas été analysés.",
|
||||||
|
skipped_document_analyses));
|
||||||
|
|
||||||
if (out_result != NULL)
|
if (out_result != NULL)
|
||||||
*out_result = res;
|
*out_result = res;
|
||||||
|
|
@ -238,9 +255,23 @@ BackgroundTask *eml_pipeline_task_new_with_tools(
|
||||||
const char *processed_evidence_dir,
|
const char *processed_evidence_dir,
|
||||||
const char *evidence_id,
|
const char *evidence_id,
|
||||||
const DocumentAnalysisTools *tools)
|
const DocumentAnalysisTools *tools)
|
||||||
|
{
|
||||||
|
return eml_pipeline_task_new_with_tools_and_limit(
|
||||||
|
eml_path, processed_evidence_dir, evidence_id, tools,
|
||||||
|
DOCUMENT_ANALYSIS_MAX_PIPELINE_ITEMS);
|
||||||
|
}
|
||||||
|
|
||||||
|
BackgroundTask *eml_pipeline_task_new_with_tools_and_limit(
|
||||||
|
const char *eml_path,
|
||||||
|
const char *processed_evidence_dir,
|
||||||
|
const char *evidence_id,
|
||||||
|
const DocumentAnalysisTools *tools,
|
||||||
|
guint document_analysis_limit)
|
||||||
{
|
{
|
||||||
if (eml_path == NULL || processed_evidence_dir == NULL || tools == NULL)
|
if (eml_path == NULL || processed_evidence_dir == NULL || tools == NULL)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
if (document_analysis_limit == 0)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
EmlPipelineTaskData *data = g_new0(EmlPipelineTaskData, 1);
|
EmlPipelineTaskData *data = g_new0(EmlPipelineTaskData, 1);
|
||||||
data->eml_path = g_strdup(eml_path);
|
data->eml_path = g_strdup(eml_path);
|
||||||
|
|
@ -251,6 +282,7 @@ BackgroundTask *eml_pipeline_task_new_with_tools(
|
||||||
data->pdfinfo = g_strdup(tools->pdfinfo);
|
data->pdfinfo = g_strdup(tools->pdfinfo);
|
||||||
data->pdftotext = g_strdup(tools->pdftotext);
|
data->pdftotext = g_strdup(tools->pdftotext);
|
||||||
data->pdftoppm = g_strdup(tools->pdftoppm);
|
data->pdftoppm = g_strdup(tools->pdftoppm);
|
||||||
|
data->document_analysis_limit = document_analysis_limit;
|
||||||
|
|
||||||
BackgroundTask *task = background_task_new(
|
BackgroundTask *task = background_task_new(
|
||||||
"Analyse du message EML et de ses pièces jointes");
|
"Analyse du message EML et de ses pièces jointes");
|
||||||
|
|
|
||||||
|
|
@ -216,12 +216,28 @@ ExiftoolAnalysisResult *exiftool_analysis_run(
|
||||||
GCancellable *cancellable,
|
GCancellable *cancellable,
|
||||||
GError **error
|
GError **error
|
||||||
)
|
)
|
||||||
|
{
|
||||||
|
const DocumentToolRunnerLimits limits = {
|
||||||
|
DOCUMENT_ANALYSIS_MAX_STDOUT,
|
||||||
|
DOCUMENT_ANALYSIS_MAX_STDERR
|
||||||
|
};
|
||||||
|
return exiftool_analysis_run_with_limits(
|
||||||
|
executable, file_path, &limits, cancellable, error);
|
||||||
|
}
|
||||||
|
|
||||||
|
ExiftoolAnalysisResult *exiftool_analysis_run_with_limits(
|
||||||
|
const char *executable,
|
||||||
|
const char *file_path,
|
||||||
|
const DocumentToolRunnerLimits *limits,
|
||||||
|
GCancellable *cancellable,
|
||||||
|
GError **error
|
||||||
|
)
|
||||||
{
|
{
|
||||||
const char *arguments[] = { "-j", "-G1", "-n", "--", file_path, NULL };
|
const char *arguments[] = { "-j", "-G1", "-n", "--", file_path, NULL };
|
||||||
const char *version_arguments[] = { "-ver", NULL };
|
const char *version_arguments[] = { "-ver", NULL };
|
||||||
DocumentToolExecution *execution = NULL;
|
DocumentToolExecution *execution = NULL;
|
||||||
if (!document_tool_runner_run("exiftool", executable, arguments,
|
if (!document_tool_runner_run_with_limits("exiftool", executable,
|
||||||
file_path, cancellable, &execution, error))
|
arguments, file_path, limits, cancellable, &execution, error))
|
||||||
{
|
{
|
||||||
document_tool_execution_free(execution);
|
document_tool_execution_free(execution);
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
@ -237,6 +253,18 @@ ExiftoolAnalysisResult *exiftool_analysis_run(
|
||||||
}
|
}
|
||||||
execution->version = document_tool_runner_read_version(
|
execution->version = document_tool_runner_read_version(
|
||||||
executable, version_arguments, cancellable);
|
executable, version_arguments, cancellable);
|
||||||
|
if (execution->stdout_truncated)
|
||||||
|
{
|
||||||
|
ExiftoolAnalysisResult *truncated =
|
||||||
|
g_new0(ExiftoolAnalysisResult, 1);
|
||||||
|
truncated->execution = execution;
|
||||||
|
truncated->metadata = g_ptr_array_new_with_free_func(
|
||||||
|
exiftool_metadata_entry_free);
|
||||||
|
execution->state = DOCUMENT_ANALYSIS_STATE_FAILED;
|
||||||
|
g_ptr_array_add(execution->errors, g_strdup(
|
||||||
|
"Le JSON ExifTool tronqué n'a pas été interprété."));
|
||||||
|
return truncated;
|
||||||
|
}
|
||||||
ExiftoolAnalysisResult *result = exiftool_analysis_parse(file_path,
|
ExiftoolAnalysisResult *result = exiftool_analysis_parse(file_path,
|
||||||
execution->raw_stdout != NULL ? execution->raw_stdout : "",
|
execution->raw_stdout != NULL ? execution->raw_stdout : "",
|
||||||
execution->raw_stderr, execution->exit_status, error);
|
execution->raw_stderr, execution->exit_status, error);
|
||||||
|
|
|
||||||
|
|
@ -29,6 +29,23 @@ OcrAnalysisResult *ocr_analysis_run(
|
||||||
GCancellable *cancellable,
|
GCancellable *cancellable,
|
||||||
GError **error
|
GError **error
|
||||||
)
|
)
|
||||||
|
{
|
||||||
|
const DocumentToolRunnerLimits limits = {
|
||||||
|
DOCUMENT_ANALYSIS_MAX_TEXT,
|
||||||
|
DOCUMENT_ANALYSIS_MAX_STDERR
|
||||||
|
};
|
||||||
|
return ocr_analysis_run_with_limits(executable, image_path, languages,
|
||||||
|
&limits, cancellable, error);
|
||||||
|
}
|
||||||
|
|
||||||
|
OcrAnalysisResult *ocr_analysis_run_with_limits(
|
||||||
|
const char *executable,
|
||||||
|
const char *image_path,
|
||||||
|
const char *languages,
|
||||||
|
const DocumentToolRunnerLimits *limits,
|
||||||
|
GCancellable *cancellable,
|
||||||
|
GError **error
|
||||||
|
)
|
||||||
{
|
{
|
||||||
if (executable == NULL || image_path == NULL || languages == NULL ||
|
if (executable == NULL || image_path == NULL || languages == NULL ||
|
||||||
languages[0] == '\0' ||
|
languages[0] == '\0' ||
|
||||||
|
|
@ -45,8 +62,8 @@ OcrAnalysisResult *ocr_analysis_run(
|
||||||
};
|
};
|
||||||
const char *version_arguments[] = { "--version", NULL };
|
const char *version_arguments[] = { "--version", NULL };
|
||||||
DocumentToolExecution *execution = NULL;
|
DocumentToolExecution *execution = NULL;
|
||||||
if (!document_tool_runner_run("tesseract", executable, arguments,
|
if (!document_tool_runner_run_with_limits("tesseract", executable,
|
||||||
image_path, cancellable, &execution, error))
|
arguments, image_path, limits, cancellable, &execution, error))
|
||||||
{
|
{
|
||||||
document_tool_execution_free(execution);
|
document_tool_execution_free(execution);
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
|
||||||
|
|
@ -139,6 +139,7 @@ static gboolean pdf_analysis_render_and_ocr(
|
||||||
}
|
}
|
||||||
|
|
||||||
gboolean success = TRUE;
|
gboolean success = TRUE;
|
||||||
|
char *source_basename = g_path_get_basename(result->source_path);
|
||||||
for (guint page_number = 1; page_number <= pages; page_number++)
|
for (guint page_number = 1; page_number <= pages; page_number++)
|
||||||
{
|
{
|
||||||
if (cancellable != NULL &&
|
if (cancellable != NULL &&
|
||||||
|
|
@ -150,8 +151,8 @@ static gboolean pdf_analysis_render_and_ocr(
|
||||||
success = FALSE;
|
success = FALSE;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
char *prefix = g_strdup_printf("%s/page-%u",
|
char *prefix = g_strdup_printf("%s/%s-page-%u",
|
||||||
temporary_directory, page_number);
|
temporary_directory, source_basename, page_number);
|
||||||
char *page_text = g_strdup_printf("%u", page_number);
|
char *page_text = g_strdup_printf("%u", page_number);
|
||||||
const char *render_arguments[] = {
|
const char *render_arguments[] = {
|
||||||
"-f", page_text, "-singlefile", "-png",
|
"-f", page_text, "-singlefile", "-png",
|
||||||
|
|
@ -162,6 +163,9 @@ static gboolean pdf_analysis_render_and_ocr(
|
||||||
render_arguments, result->source_path, cancellable,
|
render_arguments, result->source_path, cancellable,
|
||||||
&render_execution, error))
|
&render_execution, error))
|
||||||
{
|
{
|
||||||
|
result->state = result->pages->len > 0
|
||||||
|
? DOCUMENT_ANALYSIS_STATE_PARTIAL
|
||||||
|
: DOCUMENT_ANALYSIS_STATE_CANCELLED;
|
||||||
document_tool_execution_free(render_execution);
|
document_tool_execution_free(render_execution);
|
||||||
g_free(page_text);
|
g_free(page_text);
|
||||||
g_free(prefix);
|
g_free(prefix);
|
||||||
|
|
@ -198,12 +202,23 @@ static gboolean pdf_analysis_render_and_ocr(
|
||||||
g_ptr_array_add(result->pages, page);
|
g_ptr_array_add(result->pages, page);
|
||||||
if (page->state != DOCUMENT_ANALYSIS_STATE_SUCCESS)
|
if (page->state != DOCUMENT_ANALYSIS_STATE_SUCCESS)
|
||||||
result->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
|
result->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
|
||||||
|
if (ocr == NULL && error != NULL && *error != NULL &&
|
||||||
|
g_error_matches(*error, G_IO_ERROR, G_IO_ERROR_CANCELLED))
|
||||||
|
{
|
||||||
|
result->state = result->pages->len > 1
|
||||||
|
? DOCUMENT_ANALYSIS_STATE_PARTIAL
|
||||||
|
: DOCUMENT_ANALYSIS_STATE_CANCELLED;
|
||||||
|
success = FALSE;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
g_remove(image_path);
|
g_remove(image_path);
|
||||||
g_free(image_path);
|
g_free(image_path);
|
||||||
g_free(page_text);
|
g_free(page_text);
|
||||||
g_free(prefix);
|
g_free(prefix);
|
||||||
|
if (!success)
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
g_free(source_basename);
|
||||||
g_rmdir(temporary_directory);
|
g_rmdir(temporary_directory);
|
||||||
g_free(temporary_directory);
|
g_free(temporary_directory);
|
||||||
return success;
|
return success;
|
||||||
|
|
@ -233,7 +248,16 @@ PdfAnalysisResult *pdf_analysis_run(
|
||||||
if (!document_tool_runner_run("pdfinfo", tools->pdfinfo,
|
if (!document_tool_runner_run("pdfinfo", tools->pdfinfo,
|
||||||
info_arguments, pdf_path, cancellable,
|
info_arguments, pdf_path, cancellable,
|
||||||
&result->pdfinfo_execution, error))
|
&result->pdfinfo_execution, error))
|
||||||
|
{
|
||||||
|
if (error != NULL && *error != NULL &&
|
||||||
|
g_error_matches(*error, G_IO_ERROR, G_IO_ERROR_CANCELLED))
|
||||||
|
{
|
||||||
|
result->state = DOCUMENT_ANALYSIS_STATE_CANCELLED;
|
||||||
|
g_clear_error(error);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
goto failure;
|
goto failure;
|
||||||
|
}
|
||||||
if (result->pdfinfo_execution->state ==
|
if (result->pdfinfo_execution->state ==
|
||||||
DOCUMENT_ANALYSIS_STATE_UNAVAILABLE)
|
DOCUMENT_ANALYSIS_STATE_UNAVAILABLE)
|
||||||
{
|
{
|
||||||
|
|
@ -262,7 +286,16 @@ PdfAnalysisResult *pdf_analysis_run(
|
||||||
if (!document_tool_runner_run("pdftotext", tools->pdftotext,
|
if (!document_tool_runner_run("pdftotext", tools->pdftotext,
|
||||||
text_arguments, pdf_path, cancellable,
|
text_arguments, pdf_path, cancellable,
|
||||||
&result->native_execution, error))
|
&result->native_execution, error))
|
||||||
|
{
|
||||||
|
if (error != NULL && *error != NULL &&
|
||||||
|
g_error_matches(*error, G_IO_ERROR, G_IO_ERROR_CANCELLED))
|
||||||
|
{
|
||||||
|
result->state = DOCUMENT_ANALYSIS_STATE_CANCELLED;
|
||||||
|
g_clear_error(error);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
goto failure;
|
goto failure;
|
||||||
|
}
|
||||||
result->native_execution->version =
|
result->native_execution->version =
|
||||||
document_tool_runner_read_version(
|
document_tool_runner_read_version(
|
||||||
tools->pdftotext, version_arguments, cancellable);
|
tools->pdftotext, version_arguments, cancellable);
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
#include <poll.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
static int has_argument(int argc, char **argv, const char *value)
|
static int has_argument(int argc, char **argv, const char *value)
|
||||||
|
|
@ -23,8 +24,56 @@ static const char *find_pdf_path(int argc, char **argv)
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static long argument_long(
|
||||||
|
int argc,
|
||||||
|
char **argv,
|
||||||
|
const char *name,
|
||||||
|
long fallback
|
||||||
|
)
|
||||||
|
{
|
||||||
|
for (int index = 1; index + 1 < argc; index++)
|
||||||
|
if (strcmp(argv[index], name) == 0)
|
||||||
|
return strtol(argv[index + 1], NULL, 10);
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void write_repeated(FILE *stream, char value, long count)
|
||||||
|
{
|
||||||
|
char block[1024];
|
||||||
|
memset(block, value, sizeof(block));
|
||||||
|
while (count > 0)
|
||||||
|
{
|
||||||
|
size_t amount = (size_t) (count > (long) sizeof(block)
|
||||||
|
? (long) sizeof(block) : count);
|
||||||
|
if (fwrite(block, 1, amount, stream) != amount)
|
||||||
|
return;
|
||||||
|
fflush(stream);
|
||||||
|
count -= (long) amount;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
int main(int argc, char **argv)
|
int main(int argc, char **argv)
|
||||||
{
|
{
|
||||||
|
if (has_argument(argc, argv, "--emit"))
|
||||||
|
{
|
||||||
|
long stdout_size = argument_long(
|
||||||
|
argc, argv, "--stdout-size", 0);
|
||||||
|
long stderr_size = argument_long(
|
||||||
|
argc, argv, "--stderr-size", 0);
|
||||||
|
long chunks = argument_long(argc, argv, "--chunks", 1);
|
||||||
|
int exit_status = (int) argument_long(
|
||||||
|
argc, argv, "--exit-status", 0);
|
||||||
|
if (chunks < 1)
|
||||||
|
chunks = 1;
|
||||||
|
for (long index = 0; index < chunks; index++)
|
||||||
|
{
|
||||||
|
write_repeated(stdout, 'O', stdout_size / chunks);
|
||||||
|
write_repeated(stderr, 'E', stderr_size / chunks);
|
||||||
|
if (has_argument(argc, argv, "--slow"))
|
||||||
|
(void) poll(NULL, 0, 20);
|
||||||
|
}
|
||||||
|
return exit_status;
|
||||||
|
}
|
||||||
if (has_argument(argc, argv, "-ver"))
|
if (has_argument(argc, argv, "-ver"))
|
||||||
{
|
{
|
||||||
puts("13.00");
|
puts("13.00");
|
||||||
|
|
@ -37,6 +86,14 @@ int main(int argc, char **argv)
|
||||||
}
|
}
|
||||||
if (has_argument(argc, argv, "-j"))
|
if (has_argument(argc, argv, "-j"))
|
||||||
{
|
{
|
||||||
|
if (strstr(argv[argc - 1], "slow") != NULL)
|
||||||
|
sleep(2);
|
||||||
|
if (strstr(argv[argc - 1], "large") != NULL)
|
||||||
|
{
|
||||||
|
fputs("[{\"File:MIMEType\":\"image/png\",\"Padding\":\"", stdout);
|
||||||
|
write_repeated(stdout, 'X', 8192);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
puts("[{\"File:MIMEType\":\"image/png\","
|
puts("[{\"File:MIMEType\":\"image/png\","
|
||||||
"\"File:FileSize\":42,\"EXIF:ImageWidth\":10,"
|
"\"File:FileSize\":42,\"EXIF:ImageWidth\":10,"
|
||||||
"\"EXIF:GPSLatitude\":48.5,\"EXIF:GPSLongitude\":2.2,"
|
"\"EXIF:GPSLatitude\":48.5,\"EXIF:GPSLongitude\":2.2,"
|
||||||
|
|
@ -46,6 +103,8 @@ int main(int argc, char **argv)
|
||||||
if (has_argument(argc, argv, "-enc"))
|
if (has_argument(argc, argv, "-enc"))
|
||||||
{
|
{
|
||||||
const char *path = find_pdf_path(argc, argv);
|
const char *path = find_pdf_path(argc, argv);
|
||||||
|
if (strstr(path, "slow-text") != NULL)
|
||||||
|
sleep(2);
|
||||||
if (strstr(path, "native") != NULL)
|
if (strstr(path, "native") != NULL)
|
||||||
{
|
{
|
||||||
puts("Texte synthétique suffisamment long pour être considéré "
|
puts("Texte synthétique suffisamment long pour être considéré "
|
||||||
|
|
@ -55,6 +114,12 @@ int main(int argc, char **argv)
|
||||||
}
|
}
|
||||||
if (has_argument(argc, argv, "-singlefile"))
|
if (has_argument(argc, argv, "-singlefile"))
|
||||||
{
|
{
|
||||||
|
const char *pdf_path = find_pdf_path(argc, argv);
|
||||||
|
if (strstr(pdf_path, "slow-render") != NULL)
|
||||||
|
sleep(2);
|
||||||
|
if (strstr(pdf_path, "slow-page-2") != NULL &&
|
||||||
|
strcmp(argv[2], "2") == 0)
|
||||||
|
sleep(2);
|
||||||
const char *prefix = argv[argc - 1];
|
const char *prefix = argv[argc - 1];
|
||||||
char path[4096];
|
char path[4096];
|
||||||
if (snprintf(path, sizeof(path), "%s.png", prefix) < 0)
|
if (snprintf(path, sizeof(path), "%s.png", prefix) < 0)
|
||||||
|
|
@ -68,8 +133,14 @@ int main(int argc, char **argv)
|
||||||
}
|
}
|
||||||
if (has_argument(argc, argv, "stdout"))
|
if (has_argument(argc, argv, "stdout"))
|
||||||
{
|
{
|
||||||
if (strstr(argv[1], "sleep") != NULL)
|
if (strstr(argv[1], "sleep") != NULL ||
|
||||||
|
strstr(argv[1], "slow-ocr") != NULL)
|
||||||
sleep(2);
|
sleep(2);
|
||||||
|
if (strstr(argv[1], "large") != NULL)
|
||||||
|
{
|
||||||
|
write_repeated(stdout, 'T', 8192);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
if (strstr(argv[1], "page-2") != NULL)
|
if (strstr(argv[1], "page-2") != NULL)
|
||||||
puts("Texte OCR synthétique page deux.");
|
puts("Texte OCR synthétique page deux.");
|
||||||
else
|
else
|
||||||
|
|
@ -77,6 +148,8 @@ int main(int argc, char **argv)
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
const char *path = find_pdf_path(argc, argv);
|
const char *path = find_pdf_path(argc, argv);
|
||||||
|
if (strstr(path, "slow-info") != NULL)
|
||||||
|
sleep(2);
|
||||||
if (strstr(path, "encrypted") != NULL)
|
if (strstr(path, "encrypted") != NULL)
|
||||||
puts("Pages: 2\nEncrypted: yes");
|
puts("Pages: 2\nEncrypted: yes");
|
||||||
else
|
else
|
||||||
|
|
|
||||||
160
tests/test_document_tool_runner.c
Normal file
160
tests/test_document_tool_runner.c
Normal file
|
|
@ -0,0 +1,160 @@
|
||||||
|
/******************************************************************************
|
||||||
|
* @file test_document_tool_runner.c
|
||||||
|
* @brief Tests synthétiques de la capture documentaire bornée.
|
||||||
|
******************************************************************************/
|
||||||
|
#include "core/document_tool_runner.h"
|
||||||
|
|
||||||
|
#include <glib.h>
|
||||||
|
#include <glib/gstdio.h>
|
||||||
|
#include <sys/resource.h>
|
||||||
|
|
||||||
|
static char *create_source(char **out_directory)
|
||||||
|
{
|
||||||
|
GError *error = NULL;
|
||||||
|
*out_directory = g_dir_make_tmp("labfy-runner-XXXXXX", &error);
|
||||||
|
g_assert_no_error(error);
|
||||||
|
char *path = g_build_filename(*out_directory, "source.bin", NULL);
|
||||||
|
g_assert_true(g_file_set_contents(path, "synthetic", -1, &error));
|
||||||
|
g_assert_no_error(error);
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void remove_source(char *directory, char *path)
|
||||||
|
{
|
||||||
|
g_remove(path);
|
||||||
|
g_rmdir(directory);
|
||||||
|
g_free(path);
|
||||||
|
g_free(directory);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_limits_and_concurrent_drain(void)
|
||||||
|
{
|
||||||
|
char *directory = NULL;
|
||||||
|
char *path = create_source(&directory);
|
||||||
|
const char *arguments[] = {
|
||||||
|
"--emit", "--stdout-size", "5000", "--stderr-size", "7000",
|
||||||
|
"--chunks", "20", "--exit-status", "7", NULL
|
||||||
|
};
|
||||||
|
DocumentToolRunnerLimits limits = { 1000, 1500 };
|
||||||
|
DocumentToolExecution *execution = NULL;
|
||||||
|
GError *error = NULL;
|
||||||
|
g_assert_true(document_tool_runner_run_with_limits(
|
||||||
|
"synthetic", "tests/fake_document_tool", arguments, path,
|
||||||
|
&limits, NULL, &execution, &error));
|
||||||
|
g_assert_no_error(error);
|
||||||
|
g_assert_cmpuint(strlen(execution->raw_stdout), ==, 1000);
|
||||||
|
g_assert_cmpuint(strlen(execution->raw_stderr), ==, 1500);
|
||||||
|
g_assert_cmpuint(execution->stdout_bytes_observed, ==, 5000);
|
||||||
|
g_assert_cmpuint(execution->stderr_bytes_observed, ==, 7000);
|
||||||
|
g_assert_true(execution->stdout_truncated);
|
||||||
|
g_assert_true(execution->stderr_truncated);
|
||||||
|
g_assert_cmpint(execution->exit_status, ==, 7);
|
||||||
|
g_assert_cmpint(execution->state, ==,
|
||||||
|
DOCUMENT_ANALYSIS_STATE_PARTIAL);
|
||||||
|
g_assert_cmpint(execution->raw_stdout[0], ==, 'O');
|
||||||
|
g_assert_cmpint(execution->raw_stderr[0], ==, 'E');
|
||||||
|
document_tool_execution_free(execution);
|
||||||
|
remove_source(directory, path);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_exact_and_below_limits(void)
|
||||||
|
{
|
||||||
|
char *directory = NULL;
|
||||||
|
char *path = create_source(&directory);
|
||||||
|
const char *arguments[] = {
|
||||||
|
"--emit", "--stdout-size", "1000", "--stderr-size", "12", NULL
|
||||||
|
};
|
||||||
|
DocumentToolRunnerLimits limits = { 1000, 20 };
|
||||||
|
DocumentToolExecution *execution = NULL;
|
||||||
|
g_assert_true(document_tool_runner_run_with_limits(
|
||||||
|
"synthetic", "tests/fake_document_tool", arguments, path,
|
||||||
|
&limits, NULL, &execution, NULL));
|
||||||
|
g_assert_false(execution->stdout_truncated);
|
||||||
|
g_assert_false(execution->stderr_truncated);
|
||||||
|
g_assert_cmpint(execution->state, ==,
|
||||||
|
DOCUMENT_ANALYSIS_STATE_SUCCESS);
|
||||||
|
document_tool_execution_free(execution);
|
||||||
|
remove_source(directory, path);
|
||||||
|
}
|
||||||
|
|
||||||
|
static gpointer cancel_later(gpointer user_data)
|
||||||
|
{
|
||||||
|
g_usleep(60000);
|
||||||
|
g_cancellable_cancel(user_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_cancellation_during_output(void)
|
||||||
|
{
|
||||||
|
char *directory = NULL;
|
||||||
|
char *path = create_source(&directory);
|
||||||
|
const char *arguments[] = {
|
||||||
|
"--emit", "--stdout-size", "100000", "--stderr-size", "100000",
|
||||||
|
"--chunks", "100", "--slow", NULL
|
||||||
|
};
|
||||||
|
DocumentToolRunnerLimits limits = { 128, 128 };
|
||||||
|
DocumentToolExecution *execution = NULL;
|
||||||
|
GCancellable *cancellable = g_cancellable_new();
|
||||||
|
GThread *thread = g_thread_new("cancel-runner", cancel_later, cancellable);
|
||||||
|
GError *error = NULL;
|
||||||
|
g_assert_false(document_tool_runner_run_with_limits(
|
||||||
|
"synthetic", "tests/fake_document_tool", arguments, path,
|
||||||
|
&limits, cancellable, &execution, &error));
|
||||||
|
g_thread_join(thread);
|
||||||
|
g_assert_error(error, G_IO_ERROR, G_IO_ERROR_CANCELLED);
|
||||||
|
g_assert_nonnull(execution);
|
||||||
|
g_assert_cmpint(execution->state, ==,
|
||||||
|
DOCUMENT_ANALYSIS_STATE_CANCELLED);
|
||||||
|
g_clear_error(&error);
|
||||||
|
document_tool_execution_free(execution);
|
||||||
|
g_object_unref(cancellable);
|
||||||
|
|
||||||
|
const char *success_arguments[] = {
|
||||||
|
"--emit", "--stdout-size", "1", NULL
|
||||||
|
};
|
||||||
|
execution = NULL;
|
||||||
|
g_assert_true(document_tool_runner_run_with_limits(
|
||||||
|
"synthetic", "tests/fake_document_tool", success_arguments, path,
|
||||||
|
&limits, NULL, &execution, NULL));
|
||||||
|
document_tool_execution_free(execution);
|
||||||
|
remove_source(directory, path);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_repeated_runs_do_not_exhaust_descriptors(void)
|
||||||
|
{
|
||||||
|
char *directory = NULL;
|
||||||
|
char *path = create_source(&directory);
|
||||||
|
const char *arguments[] = {
|
||||||
|
"--emit", "--stdout-size", "8", "--stderr-size", "8", NULL
|
||||||
|
};
|
||||||
|
DocumentToolRunnerLimits limits = { 16, 16 };
|
||||||
|
struct rlimit original_limit;
|
||||||
|
g_assert_cmpint(getrlimit(RLIMIT_NOFILE, &original_limit), ==, 0);
|
||||||
|
struct rlimit test_limit = original_limit;
|
||||||
|
test_limit.rlim_cur = MIN(original_limit.rlim_cur, (rlim_t) 64);
|
||||||
|
g_assert_cmpint(setrlimit(RLIMIT_NOFILE, &test_limit), ==, 0);
|
||||||
|
for (guint index = 0; index < 96; index++)
|
||||||
|
{
|
||||||
|
DocumentToolExecution *execution = NULL;
|
||||||
|
g_assert_true(document_tool_runner_run_with_limits(
|
||||||
|
"synthetic", "tests/fake_document_tool", arguments, path,
|
||||||
|
&limits, NULL, &execution, NULL));
|
||||||
|
document_tool_execution_free(execution);
|
||||||
|
}
|
||||||
|
g_assert_cmpint(setrlimit(RLIMIT_NOFILE, &original_limit), ==, 0);
|
||||||
|
remove_source(directory, path);
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char **argv)
|
||||||
|
{
|
||||||
|
g_test_init(&argc, &argv, NULL);
|
||||||
|
g_test_add_func("/document-tool-runner/limits-concurrent",
|
||||||
|
test_limits_and_concurrent_drain);
|
||||||
|
g_test_add_func("/document-tool-runner/exact-below",
|
||||||
|
test_exact_and_below_limits);
|
||||||
|
g_test_add_func("/document-tool-runner/cancellation",
|
||||||
|
test_cancellation_during_output);
|
||||||
|
g_test_add_func("/document-tool-runner/no-descriptor-exhaustion",
|
||||||
|
test_repeated_runs_do_not_exhaust_descriptors);
|
||||||
|
return g_test_run();
|
||||||
|
}
|
||||||
|
|
@ -136,11 +136,107 @@ static void test_eml_pipeline_document_analysis(void)
|
||||||
g_free(tmp_dir);
|
g_free(tmp_dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void wait_for_task(BackgroundTask *task)
|
||||||
|
{
|
||||||
|
while (background_task_get_state(task) ==
|
||||||
|
BACKGROUND_TASK_STATE_RUNNING ||
|
||||||
|
background_task_get_state(task) ==
|
||||||
|
BACKGROUND_TASK_STATE_PENDING)
|
||||||
|
g_main_context_iteration(NULL, TRUE);
|
||||||
|
}
|
||||||
|
|
||||||
|
static DocumentAnalysisTools synthetic_tools(void)
|
||||||
|
{
|
||||||
|
DocumentAnalysisTools tools = {
|
||||||
|
.exiftool = "tests/fake_document_tool",
|
||||||
|
.tesseract = "tests/fake_document_tool",
|
||||||
|
.pdfinfo = "tests/fake_document_tool",
|
||||||
|
.pdftotext = "tests/fake_document_tool",
|
||||||
|
.pdftoppm = "tests/fake_document_tool"
|
||||||
|
};
|
||||||
|
return tools;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_eml_pipeline_pdf_end_to_end_and_limit(void)
|
||||||
|
{
|
||||||
|
GError *error = NULL;
|
||||||
|
char *tmp_dir = g_dir_make_tmp("labfy-eml-pdf-XXXXXX", &error);
|
||||||
|
g_assert_no_error(error);
|
||||||
|
char *eml_path = g_build_filename(tmp_dir, "pdf.eml", NULL);
|
||||||
|
char *processed_dir = g_build_filename(tmp_dir, "processed", NULL);
|
||||||
|
static const char eml[] =
|
||||||
|
"From: synthetic@example.test\r\n"
|
||||||
|
"MIME-Version: 1.0\r\n"
|
||||||
|
"Content-Type: multipart/mixed; boundary=pdf-boundary\r\n\r\n"
|
||||||
|
"--pdf-boundary\r\n"
|
||||||
|
"Content-Type: application/pdf; name=scan.pdf\r\n"
|
||||||
|
"Content-Disposition: attachment; filename=scan.pdf\r\n"
|
||||||
|
"Content-Transfer-Encoding: base64\r\n\r\n"
|
||||||
|
"JVBERi1zeW50aGV0aWM=\r\n"
|
||||||
|
"--pdf-boundary\r\n"
|
||||||
|
"Content-Type: application/pdf; name=second.pdf\r\n"
|
||||||
|
"Content-Disposition: attachment; filename=second.pdf\r\n"
|
||||||
|
"Content-Transfer-Encoding: base64\r\n\r\n"
|
||||||
|
"JVBERi1zeW50aGV0aWM=\r\n"
|
||||||
|
"--pdf-boundary--\r\n";
|
||||||
|
g_assert_true(g_file_set_contents(eml_path, eml, -1, &error));
|
||||||
|
g_assert_no_error(error);
|
||||||
|
char *source_before = NULL;
|
||||||
|
gsize source_before_length = 0;
|
||||||
|
g_assert_true(g_file_get_contents(
|
||||||
|
eml_path, &source_before, &source_before_length, &error));
|
||||||
|
g_assert_no_error(error);
|
||||||
|
|
||||||
|
DocumentAnalysisTools tools = synthetic_tools();
|
||||||
|
BackgroundTask *task = eml_pipeline_task_new_with_tools_and_limit(
|
||||||
|
eml_path, processed_dir, "synthetic-evidence", &tools, 1);
|
||||||
|
g_assert_nonnull(task);
|
||||||
|
wait_for_task(task);
|
||||||
|
g_assert_cmpint(background_task_get_state(task), ==,
|
||||||
|
BACKGROUND_TASK_STATE_COMPLETED);
|
||||||
|
EmlPipelineResult *result = background_task_get_result(task);
|
||||||
|
g_assert_nonnull(result);
|
||||||
|
g_assert_cmpuint(result->mime_result->attachments->len, ==, 2);
|
||||||
|
g_assert_cmpuint(result->document_analyses->len, ==, 1);
|
||||||
|
g_assert_cmpuint(result->skipped_document_analyses, ==, 1);
|
||||||
|
g_assert_cmpint(result->state, ==, DOCUMENT_ANALYSIS_STATE_PARTIAL);
|
||||||
|
g_assert_cmpuint(result->warnings->len, ==, 1);
|
||||||
|
DocumentFileAnalysis *document = g_ptr_array_index(
|
||||||
|
result->document_analyses, 0);
|
||||||
|
g_assert_nonnull(document->pdf);
|
||||||
|
g_assert_false(document->pdf->native_text_usable);
|
||||||
|
g_assert_cmpuint(document->pdf->pages->len, ==, 2);
|
||||||
|
PdfPageAnalysis *first = g_ptr_array_index(document->pdf->pages, 0);
|
||||||
|
PdfPageAnalysis *second = g_ptr_array_index(document->pdf->pages, 1);
|
||||||
|
g_assert_cmpuint(first->page_number, ==, 1);
|
||||||
|
g_assert_cmpuint(second->page_number, ==, 2);
|
||||||
|
g_assert_nonnull(first->render_execution);
|
||||||
|
g_assert_nonnull(first->execution);
|
||||||
|
|
||||||
|
char *source_after = NULL;
|
||||||
|
gsize source_after_length = 0;
|
||||||
|
g_assert_true(g_file_get_contents(
|
||||||
|
eml_path, &source_after, &source_after_length, &error));
|
||||||
|
g_assert_no_error(error);
|
||||||
|
g_assert_cmpuint(source_after_length, ==, source_before_length);
|
||||||
|
g_assert_cmpmem(source_after, source_after_length,
|
||||||
|
source_before, source_before_length);
|
||||||
|
g_free(source_after);
|
||||||
|
g_free(source_before);
|
||||||
|
background_task_unref(task);
|
||||||
|
g_remove(eml_path);
|
||||||
|
g_free(processed_dir);
|
||||||
|
g_free(eml_path);
|
||||||
|
g_free(tmp_dir);
|
||||||
|
}
|
||||||
|
|
||||||
int main(int argc, char **argv)
|
int main(int argc, char **argv)
|
||||||
{
|
{
|
||||||
g_test_init(&argc, &argv, NULL);
|
g_test_init(&argc, &argv, NULL);
|
||||||
g_test_add_func("/eml-pipeline-task/basic", test_eml_pipeline_basic);
|
g_test_add_func("/eml-pipeline-task/basic", test_eml_pipeline_basic);
|
||||||
g_test_add_func("/eml-pipeline-task/document-analysis",
|
g_test_add_func("/eml-pipeline-task/document-analysis",
|
||||||
test_eml_pipeline_document_analysis);
|
test_eml_pipeline_document_analysis);
|
||||||
|
g_test_add_func("/eml-pipeline-task/pdf-end-to-end-limit",
|
||||||
|
test_eml_pipeline_pdf_end_to_end_and_limit);
|
||||||
return g_test_run();
|
return g_test_run();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -68,6 +68,54 @@ static void test_run_and_unavailable(void)
|
||||||
g_free(directory);
|
g_free(directory);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static gpointer cancel_exiftool(gpointer user_data)
|
||||||
|
{
|
||||||
|
g_usleep(50000);
|
||||||
|
g_cancellable_cancel(user_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_cancellation_and_truncated_json(void)
|
||||||
|
{
|
||||||
|
GError *error = NULL;
|
||||||
|
char *directory = g_dir_make_tmp("labfy-exif-hardening-XXXXXX", &error);
|
||||||
|
char *slow_path = g_build_filename(directory, "slow.png", NULL);
|
||||||
|
char *large_path = g_build_filename(directory, "large.png", NULL);
|
||||||
|
g_assert_true(g_file_set_contents(slow_path, "PNG", 3, &error));
|
||||||
|
g_assert_true(g_file_set_contents(large_path, "PNG", 3, &error));
|
||||||
|
|
||||||
|
GCancellable *cancellable = g_cancellable_new();
|
||||||
|
GThread *thread = g_thread_new(
|
||||||
|
"exif-cancel", cancel_exiftool, cancellable);
|
||||||
|
ExiftoolAnalysisResult *result = exiftool_analysis_run(
|
||||||
|
"tests/fake_document_tool", slow_path, cancellable, &error);
|
||||||
|
g_thread_join(thread);
|
||||||
|
g_assert_null(result);
|
||||||
|
g_assert_error(error, G_IO_ERROR, G_IO_ERROR_CANCELLED);
|
||||||
|
g_clear_error(&error);
|
||||||
|
g_object_unref(cancellable);
|
||||||
|
|
||||||
|
DocumentToolRunnerLimits limits = { 128, 128 };
|
||||||
|
result = exiftool_analysis_run_with_limits(
|
||||||
|
"tests/fake_document_tool", large_path, &limits, NULL, &error);
|
||||||
|
g_assert_no_error(error);
|
||||||
|
g_assert_nonnull(result);
|
||||||
|
g_assert_true(result->execution->stdout_truncated);
|
||||||
|
g_assert_cmpuint(strlen(result->execution->raw_stdout), ==, 128);
|
||||||
|
g_assert_cmpuint(result->metadata->len, ==, 0);
|
||||||
|
g_assert_cmpint(result->execution->state, ==,
|
||||||
|
DOCUMENT_ANALYSIS_STATE_FAILED);
|
||||||
|
g_assert_cmpuint(result->execution->warnings->len, >, 0);
|
||||||
|
exiftool_analysis_result_free(result);
|
||||||
|
|
||||||
|
g_remove(slow_path);
|
||||||
|
g_remove(large_path);
|
||||||
|
g_rmdir(directory);
|
||||||
|
g_free(slow_path);
|
||||||
|
g_free(large_path);
|
||||||
|
g_free(directory);
|
||||||
|
}
|
||||||
|
|
||||||
int main(int argc, char **argv)
|
int main(int argc, char **argv)
|
||||||
{
|
{
|
||||||
g_test_init(&argc, &argv, NULL);
|
g_test_init(&argc, &argv, NULL);
|
||||||
|
|
@ -76,5 +124,7 @@ int main(int argc, char **argv)
|
||||||
g_test_add_func("/exiftool-analysis/invalid-json", test_invalid_json);
|
g_test_add_func("/exiftool-analysis/invalid-json", test_invalid_json);
|
||||||
g_test_add_func("/exiftool-analysis/run-unavailable",
|
g_test_add_func("/exiftool-analysis/run-unavailable",
|
||||||
test_run_and_unavailable);
|
test_run_and_unavailable);
|
||||||
|
g_test_add_func("/exiftool-analysis/cancellation-truncated",
|
||||||
|
test_cancellation_and_truncated_json);
|
||||||
return g_test_run();
|
return g_test_run();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -75,6 +75,29 @@ static void test_cancellation(void)
|
||||||
g_free(directory);
|
g_free(directory);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_truncated_text(void)
|
||||||
|
{
|
||||||
|
GError *error = NULL;
|
||||||
|
char *directory = g_dir_make_tmp("labfy-ocr-limit-XXXXXX", &error);
|
||||||
|
char *path = g_build_filename(directory, "large.png", NULL);
|
||||||
|
g_assert_true(g_file_set_contents(path, "PNG", 3, &error));
|
||||||
|
DocumentToolRunnerLimits limits = { 96, 64 };
|
||||||
|
OcrAnalysisResult *result = ocr_analysis_run_with_limits(
|
||||||
|
"tests/fake_document_tool", path, "fra", &limits,
|
||||||
|
NULL, &error);
|
||||||
|
g_assert_no_error(error);
|
||||||
|
g_assert_nonnull(result);
|
||||||
|
g_assert_cmpuint(strlen(result->text), ==, 96);
|
||||||
|
g_assert_true(result->execution->stdout_truncated);
|
||||||
|
g_assert_cmpint(result->execution->state, ==,
|
||||||
|
DOCUMENT_ANALYSIS_STATE_PARTIAL);
|
||||||
|
ocr_analysis_result_free(result);
|
||||||
|
g_remove(path);
|
||||||
|
g_rmdir(directory);
|
||||||
|
g_free(path);
|
||||||
|
g_free(directory);
|
||||||
|
}
|
||||||
|
|
||||||
int main(int argc, char **argv)
|
int main(int argc, char **argv)
|
||||||
{
|
{
|
||||||
g_test_init(&argc, &argv, NULL);
|
g_test_init(&argc, &argv, NULL);
|
||||||
|
|
@ -83,5 +106,6 @@ int main(int argc, char **argv)
|
||||||
g_test_add_func("/ocr-analysis/unavailable-compatible",
|
g_test_add_func("/ocr-analysis/unavailable-compatible",
|
||||||
test_unavailable_and_compatibility);
|
test_unavailable_and_compatibility);
|
||||||
g_test_add_func("/ocr-analysis/cancellation", test_cancellation);
|
g_test_add_func("/ocr-analysis/cancellation", test_cancellation);
|
||||||
|
g_test_add_func("/ocr-analysis/truncated-text", test_truncated_text);
|
||||||
return g_test_run();
|
return g_test_run();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -84,6 +84,64 @@ static void test_heuristic(void)
|
||||||
"Texte synthétique imprimable et suffisamment long pour le test."));
|
"Texte synthétique imprimable et suffisamment long pour le test."));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static gpointer cancel_pdf(gpointer user_data)
|
||||||
|
{
|
||||||
|
g_usleep(70000);
|
||||||
|
g_cancellable_cancel(user_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void assert_cancelled_pdf(
|
||||||
|
const PdfAnalysisTools *tools,
|
||||||
|
const char *path,
|
||||||
|
guint expected_completed_pages
|
||||||
|
)
|
||||||
|
{
|
||||||
|
GCancellable *cancellable = g_cancellable_new();
|
||||||
|
GThread *thread = g_thread_new("pdf-cancel", cancel_pdf, cancellable);
|
||||||
|
GError *error = NULL;
|
||||||
|
PdfAnalysisResult *result = pdf_analysis_run(
|
||||||
|
tools, path, "fra", cancellable, &error);
|
||||||
|
g_thread_join(thread);
|
||||||
|
g_assert_no_error(error);
|
||||||
|
g_assert_nonnull(result);
|
||||||
|
g_assert_cmpuint(result->pages->len, ==, expected_completed_pages);
|
||||||
|
g_assert_true(result->state == DOCUMENT_ANALYSIS_STATE_CANCELLED ||
|
||||||
|
result->state == DOCUMENT_ANALYSIS_STATE_PARTIAL);
|
||||||
|
pdf_analysis_result_free(result);
|
||||||
|
g_object_unref(cancellable);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_cancellation_stages_and_completed_pages(void)
|
||||||
|
{
|
||||||
|
GError *error = NULL;
|
||||||
|
char *directory = g_dir_make_tmp("labfy-pdf-cancel-XXXXXX", &error);
|
||||||
|
g_assert_no_error(error);
|
||||||
|
PdfAnalysisTools tools = fake_tools();
|
||||||
|
char *slow_info = create_pdf(directory, "slow-info.pdf");
|
||||||
|
char *slow_text = create_pdf(directory, "slow-text.pdf");
|
||||||
|
char *slow_render = create_pdf(directory, "slow-render.pdf");
|
||||||
|
char *slow_ocr = create_pdf(directory, "slow-ocr.pdf");
|
||||||
|
char *slow_second = create_pdf(directory, "slow-page-2.pdf");
|
||||||
|
assert_cancelled_pdf(&tools, slow_info, 0);
|
||||||
|
assert_cancelled_pdf(&tools, slow_text, 0);
|
||||||
|
assert_cancelled_pdf(&tools, slow_render, 0);
|
||||||
|
assert_cancelled_pdf(&tools, slow_ocr, 1);
|
||||||
|
assert_cancelled_pdf(&tools, slow_second, 1);
|
||||||
|
g_remove(slow_info);
|
||||||
|
g_remove(slow_text);
|
||||||
|
g_remove(slow_render);
|
||||||
|
g_remove(slow_ocr);
|
||||||
|
g_remove(slow_second);
|
||||||
|
g_rmdir(directory);
|
||||||
|
g_free(slow_info);
|
||||||
|
g_free(slow_text);
|
||||||
|
g_free(slow_render);
|
||||||
|
g_free(slow_ocr);
|
||||||
|
g_free(slow_second);
|
||||||
|
g_free(directory);
|
||||||
|
}
|
||||||
|
|
||||||
int main(int argc, char **argv)
|
int main(int argc, char **argv)
|
||||||
{
|
{
|
||||||
g_test_init(&argc, &argv, NULL);
|
g_test_init(&argc, &argv, NULL);
|
||||||
|
|
@ -92,5 +150,7 @@ int main(int argc, char **argv)
|
||||||
g_test_add_func("/pdf-analysis/ocr-fallback-cleanup",
|
g_test_add_func("/pdf-analysis/ocr-fallback-cleanup",
|
||||||
test_ocr_fallback_order_and_cleanup);
|
test_ocr_fallback_order_and_cleanup);
|
||||||
g_test_add_func("/pdf-analysis/heuristic", test_heuristic);
|
g_test_add_func("/pdf-analysis/heuristic", test_heuristic);
|
||||||
|
g_test_add_func("/pdf-analysis/cancellation-stages",
|
||||||
|
test_cancellation_stages_and_completed_pages);
|
||||||
return g_test_run();
|
return g_test_run();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue