fy59/labfy-investigation
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

feat: ajouter l'analyse PDF OCR et ExifTool des EML

Browse source
This commit is contained in:
grayTerminal-sh 2026-07-28 07:15:49 +02:00
parent 58fbf2daae
commit d96fc83fc0
20 changed files with 1784 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

67
Makefile
View file

@ -136,6 +136,16 @@ TEST_CONTROLLED_VOCAB := tests/test_controlled_vocab
TEST_BANK_PROPOSAL := tests/test_bank_proposal
TEST_EML_PIPELINE_TASK := tests/test_eml_pipeline_task
TEST_EML_MIME_EXTRACTOR := tests/test_eml_mime_extractor
TEST_EXIFTOOL_ANALYSIS := tests/test_exiftool_analysis
TEST_OCR_ANALYSIS := tests/test_ocr_analysis
TEST_PDF_ANALYSIS := tests/test_pdf_analysis
FAKE_DOCUMENT_TOOL := tests/fake_document_tool
DOCUMENT_ANALYSIS_TEST_SOURCES := \
src/core/document_analysis.c \
src/core/document_tool_runner.c \
src/core/tool_process.c \
src/core/file_hash.c
all: $(TARGET)
@ -158,9 +168,21 @@ $(TEST_EML_PIPELINE_TASK): \
src/core/eml_pipeline_task.c src/core/eml_mime_extractor.c \
src/core/eml_analyzer.c src/core/bank_proposal.c \
src/core/controlled_vocab.c src/core/iban_analyzer.c \
src/core/rib_ocr.c src/core/file_hash.c src/core/background_task.c
$(CC) $(TEST_CFLAGS) -Wpedantic $^ -o $@ \
$(TEST_LDFLAGS) -lsqlite3
src/core/file_hash.c src/core/background_task.c \
src/core/document_analysis.c src/core/document_tool_runner.c \
src/core/exiftool_analysis.c src/core/ocr_analysis.c \
src/core/pdf_analysis.c src/core/document_file_analysis.c \
src/core/tool_process.c $(FAKE_DOCUMENT_TOOL)
$(CC) $(TEST_CFLAGS) -Wpedantic \
tests/test_eml_pipeline_task.c \
src/core/eml_pipeline_task.c src/core/eml_mime_extractor.c \
src/core/eml_analyzer.c src/core/bank_proposal.c \
src/core/controlled_vocab.c src/core/iban_analyzer.c \
src/core/file_hash.c src/core/background_task.c \
src/core/document_analysis.c src/core/document_tool_runner.c \
src/core/exiftool_analysis.c src/core/ocr_analysis.c \
src/core/pdf_analysis.c src/core/document_file_analysis.c \
src/core/tool_process.c -o $@ $(TEST_LDFLAGS) -lsqlite3
$(TEST_EML_MIME_EXTRACTOR): \
tests/test_eml_mime_extractor.c \
@ -168,6 +190,31 @@ $(TEST_EML_MIME_EXTRACTOR): \
src/core/file_hash.c
$(CC) $(TEST_CFLAGS) -Wpedantic $^ -o $@ $(TEST_LDFLAGS)
$(FAKE_DOCUMENT_TOOL): tests/fake_document_tool.c
$(CC) -std=c17 -Wall -Wextra -Werror -Wpedantic $< -o $@
$(TEST_EXIFTOOL_ANALYSIS): tests/test_exiftool_analysis.c \
src/core/exiftool_analysis.c $(DOCUMENT_ANALYSIS_TEST_SOURCES) \
$(FAKE_DOCUMENT_TOOL)
$(CC) $(TEST_CFLAGS) -Wpedantic \
tests/test_exiftool_analysis.c src/core/exiftool_analysis.c \
$(DOCUMENT_ANALYSIS_TEST_SOURCES) -o $@ $(TEST_LDFLAGS)
$(TEST_OCR_ANALYSIS): tests/test_ocr_analysis.c \
src/core/ocr_analysis.c $(DOCUMENT_ANALYSIS_TEST_SOURCES) \
$(FAKE_DOCUMENT_TOOL)
$(CC) $(TEST_CFLAGS) -Wpedantic \
tests/test_ocr_analysis.c src/core/ocr_analysis.c \
$(DOCUMENT_ANALYSIS_TEST_SOURCES) -o $@ $(TEST_LDFLAGS)
$(TEST_PDF_ANALYSIS): tests/test_pdf_analysis.c \
src/core/pdf_analysis.c src/core/ocr_analysis.c \
$(DOCUMENT_ANALYSIS_TEST_SOURCES) $(FAKE_DOCUMENT_TOOL)
$(CC) $(TEST_CFLAGS) -Wpedantic \
tests/test_pdf_analysis.c src/core/pdf_analysis.c \
src/core/ocr_analysis.c $(DOCUMENT_ANALYSIS_TEST_SOURCES) \
-o $@ $(TEST_LDFLAGS)
$(TEST_RELATION_TYPE_NORMALIZER): \
@ -838,7 +885,10 @@ test: \
$(TEST_CONTROLLED_VOCAB) \
$(TEST_BANK_PROPOSAL) \
$(TEST_EML_PIPELINE_TASK) \
$(TEST_EML_MIME_EXTRACTOR)
$(TEST_EML_MIME_EXTRACTOR) \
$(TEST_EXIFTOOL_ANALYSIS) \
$(TEST_OCR_ANALYSIS) \
$(TEST_PDF_ANALYSIS)
@echo "Exécution des tests..."
@./$(TEST_NODE)
@./$(TEST_TREE_MODEL)
@ -908,6 +958,9 @@ test: \
@$(TEST_BANK_PROPOSAL)
@$(TEST_EML_PIPELINE_TASK)
@$(TEST_EML_MIME_EXTRACTOR)
@$(TEST_EXIFTOOL_ANALYSIS)
@$(TEST_OCR_ANALYSIS)
@$(TEST_PDF_ANALYSIS)
@echo "Tous les tests sont valides."
%.o: %.c
@ -980,7 +1033,11 @@ clean:
$(TEST_CONTROLLED_VOCAB) \
$(TEST_BANK_PROPOSAL) \
$(TEST_EML_PIPELINE_TASK) \
$(TEST_EML_MIME_EXTRACTOR)
$(TEST_EML_MIME_EXTRACTOR) \
$(TEST_EXIFTOOL_ANALYSIS) \
$(TEST_OCR_ANALYSIS) \
$(TEST_PDF_ANALYSIS) \
$(FAKE_DOCUMENT_TOOL)

69
include/core/document_analysis.h Normal file
View file

@ -0,0 +1,69 @@
/******************************************************************************
* @file document_analysis.h
* @brief Modèles communs pour les analyses documentaires en mémoire.
******************************************************************************/
#ifndef LABFY_INVESTIGATION_DOCUMENT_ANALYSIS_H
#define LABFY_INVESTIGATION_DOCUMENT_ANALYSIS_H
#include <gio/gio.h>
G_BEGIN_DECLS
#define DOCUMENT_ANALYSIS_MAX_FILE_SIZE (50U * 1024U * 1024U)
#define DOCUMENT_ANALYSIS_MAX_STDOUT (8U * 1024U * 1024U)
#define DOCUMENT_ANALYSIS_MAX_STDERR (256U * 1024U)
#define DOCUMENT_ANALYSIS_MAX_TEXT (8U * 1024U * 1024U)
#define DOCUMENT_ANALYSIS_MAX_PDF_PAGES 100U
#define DOCUMENT_ANALYSIS_MAX_PIPELINE_ITEMS 128U
typedef enum
{
DOCUMENT_ANALYSIS_STATE_SUCCESS,
DOCUMENT_ANALYSIS_STATE_PARTIAL,
DOCUMENT_ANALYSIS_STATE_UNAVAILABLE,
DOCUMENT_ANALYSIS_STATE_CANCELLED,
DOCUMENT_ANALYSIS_STATE_FAILED,
DOCUMENT_ANALYSIS_STATE_INCOMPATIBLE
} DocumentAnalysisState;
typedef struct
{
char *tool_id;
char *version;
GPtrArray *arguments;
char *started_at_utc;
char *finished_at_utc;
char *source_path;
char *source_sha256;
char *raw_stdout;
char *raw_stdout_sha256;
char *raw_stderr;
int exit_status;
DocumentAnalysisState state;
GPtrArray *warnings;
GPtrArray *errors;
} DocumentToolExecution;
typedef struct
{
char *code;
char *original_group;
char *original_tag;
char *raw_value;
gboolean sensitive;
gboolean requires_confirmation;
} DocumentMetadataEntry;
DocumentToolExecution *document_tool_execution_new(
const char *tool_id,
const char *source_path
);
void document_tool_execution_free(DocumentToolExecution *execution);
void document_tool_execution_add_argument(
DocumentToolExecution *execution,
const char *argument
);
const char *document_analysis_state_code(DocumentAnalysisState state);
G_END_DECLS
#endif

48
include/core/document_file_analysis.h Normal file
View file

@ -0,0 +1,48 @@
/******************************************************************************
* @file document_file_analysis.h
* @brief Orchestration des analyses compatibles d'un fichier dérivé.
******************************************************************************/
#ifndef LABFY_INVESTIGATION_DOCUMENT_FILE_ANALYSIS_H
#define LABFY_INVESTIGATION_DOCUMENT_FILE_ANALYSIS_H
#include "core/exiftool_analysis.h"
#include "core/ocr_analysis.h"
#include "core/pdf_analysis.h"
G_BEGIN_DECLS
typedef struct
{
const char *exiftool;
const char *tesseract;
const char *pdfinfo;
const char *pdftotext;
const char *pdftoppm;
} DocumentAnalysisTools;
typedef struct
{
char *source_path;
char *declared_mime;
char *detected_mime;
ExiftoolAnalysisResult *metadata;
OcrAnalysisResult *ocr;
PdfAnalysisResult *pdf;
DocumentAnalysisState state;
GPtrArray *warnings;
} DocumentFileAnalysis;
DocumentFileAnalysis *document_file_analysis_run(
const DocumentAnalysisTools *tools,
const char *source_path,
const char *declared_mime,
const char *detected_mime,
gboolean request_image_ocr,
const char *ocr_languages,
GCancellable *cancellable,
GError **error
);
void document_file_analysis_free(DocumentFileAnalysis *analysis);
G_END_DECLS
#endif

29
include/core/document_tool_runner.h Normal file
View file

@ -0,0 +1,29 @@
/******************************************************************************
* @file document_tool_runner.h
* @brief Exécution bornée et annulable des outils documentaires.
******************************************************************************/
#ifndef LABFY_INVESTIGATION_DOCUMENT_TOOL_RUNNER_H
#define LABFY_INVESTIGATION_DOCUMENT_TOOL_RUNNER_H
#include "core/document_analysis.h"
G_BEGIN_DECLS
gboolean document_tool_runner_run(
const char *tool_id,
const char *executable,
const char *const arguments[],
const char *source_path,
GCancellable *cancellable,
DocumentToolExecution **out_execution,
GError **error
);
char *document_tool_runner_read_version(
const char *executable,
const char *const arguments[],
GCancellable *cancellable
);
G_END_DECLS
#endif

9
include/core/eml_pipeline_task.h
View file

@ -9,6 +9,7 @@
#include "core/bank_proposal.h"
#include "core/eml_analyzer.h"
#include "core/eml_mime_extractor.h"
#include "core/document_file_analysis.h"
G_BEGIN_DECLS
@ -18,6 +19,7 @@ typedef struct EmlPipelineResult
EmlAnalysis *analysis; /**< Analyse des en-têtes EML */
EmlMimeResult *mime_result; /**< Pièces jointes extraites */
GPtrArray *bank_proposals; /**< Tableau de BankProposal* */
GPtrArray *document_analyses; /**< Tableau de DocumentFileAnalysis* */
GPtrArray *warnings; /**< Avertissements globaux */
} EmlPipelineResult;
@ -35,6 +37,13 @@ BackgroundTask *eml_pipeline_task_new(const char *eml_path,
const char *processed_evidence_dir,
const char *evidence_id);
BackgroundTask *eml_pipeline_task_new_with_tools(
const char *eml_path,
const char *processed_evidence_dir,
const char *evidence_id,
const DocumentAnalysisTools *tools
);
G_END_DECLS
#endif /* LABFY_INVESTIGATION_EML_PIPELINE_TASK_H */

34
include/core/exiftool_analysis.h Normal file
View file

@ -0,0 +1,34 @@
/******************************************************************************
* @file exiftool_analysis.h
* @brief Analyse ExifTool structurée et traçable.
******************************************************************************/
#ifndef LABFY_INVESTIGATION_EXIFTOOL_ANALYSIS_H
#define LABFY_INVESTIGATION_EXIFTOOL_ANALYSIS_H
#include "core/document_analysis.h"
G_BEGIN_DECLS
typedef struct
{
DocumentToolExecution *execution;
GPtrArray *metadata;
} ExiftoolAnalysisResult;
ExiftoolAnalysisResult *exiftool_analysis_run(
const char *executable,
const char *file_path,
GCancellable *cancellable,
GError **error
);
ExiftoolAnalysisResult *exiftool_analysis_parse(
const char *file_path,
const char *json,
const char *stderr_text,
int exit_status,
GError **error
);
void exiftool_analysis_result_free(ExiftoolAnalysisResult *result);
G_END_DECLS
#endif

30
include/core/ocr_analysis.h Normal file
View file

@ -0,0 +1,30 @@
/******************************************************************************
* @file ocr_analysis.h
* @brief OCR Tesseract traçable et annulable.
******************************************************************************/
#ifndef LABFY_INVESTIGATION_OCR_ANALYSIS_H
#define LABFY_INVESTIGATION_OCR_ANALYSIS_H
#include "core/document_analysis.h"
G_BEGIN_DECLS
typedef struct
{
DocumentToolExecution *execution;
char *requested_languages;
char *text;
} OcrAnalysisResult;
OcrAnalysisResult *ocr_analysis_run(
const char *executable,
const char *image_path,
const char *languages,
GCancellable *cancellable,
GError **error
);
void ocr_analysis_result_free(OcrAnalysisResult *result);
gboolean ocr_analysis_mime_is_compatible(const char *mime_type);
G_END_DECLS
#endif

63
include/core/pdf_analysis.h Normal file
View file

@ -0,0 +1,63 @@
/******************************************************************************
* @file pdf_analysis.h
* @brief Extraction PDF native puis OCR de secours.
******************************************************************************/
#ifndef LABFY_INVESTIGATION_PDF_ANALYSIS_H
#define LABFY_INVESTIGATION_PDF_ANALYSIS_H
#include "core/document_analysis.h"
#include "core/ocr_analysis.h"
G_BEGIN_DECLS
typedef enum
{
PDF_PAGE_METHOD_NATIVE,
PDF_PAGE_METHOD_OCR
} PdfPageMethod;
typedef struct
{
guint page_number;
PdfPageMethod method;
char *text;
DocumentAnalysisState state;
DocumentToolExecution *render_execution;
DocumentToolExecution *execution;
GPtrArray *warnings;
} PdfPageAnalysis;
typedef struct
{
const char *pdfinfo;
const char *pdftotext;
const char *pdftoppm;
const char *tesseract;
} PdfAnalysisTools;
typedef struct
{
char *source_path;
gboolean encrypted;
guint page_count;
char *native_text;
gboolean native_text_usable;
DocumentAnalysisState state;
DocumentToolExecution *pdfinfo_execution;
DocumentToolExecution *native_execution;
GPtrArray *pages;
GPtrArray *warnings;
} PdfAnalysisResult;
gboolean pdf_analysis_text_is_usable(const char *text);
PdfAnalysisResult *pdf_analysis_run(
const PdfAnalysisTools *tools,
const char *pdf_path,
const char *ocr_languages,
GCancellable *cancellable,
GError **error
);
void pdf_analysis_result_free(PdfAnalysisResult *result);
G_END_DECLS
#endif

75
src/core/document_analysis.c Normal file
View file

@ -0,0 +1,75 @@
/******************************************************************************
* @file document_analysis.c
* @brief Modèles communs pour les analyses documentaires en mémoire.
******************************************************************************/
#include "core/document_analysis.h"
static char *document_analysis_now_utc(void)
{
GDateTime *now = g_date_time_new_now_utc();
char *value = g_date_time_format_iso8601(now);
g_date_time_unref(now);
return value;
}
DocumentToolExecution *document_tool_execution_new(
const char *tool_id,
const char *source_path
)
{
if (tool_id == NULL || tool_id[0] == '\0' ||
source_path == NULL || source_path[0] == '\0')
return NULL;
DocumentToolExecution *execution = g_new0(DocumentToolExecution, 1);
execution->tool_id = g_strdup(tool_id);
execution->source_path = g_strdup(source_path);
execution->started_at_utc = document_analysis_now_utc();
execution->arguments = g_ptr_array_new_with_free_func(g_free);
execution->warnings = g_ptr_array_new_with_free_func(g_free);
execution->errors = g_ptr_array_new_with_free_func(g_free);
execution->exit_status = -1;
execution->state = DOCUMENT_ANALYSIS_STATE_FAILED;
return execution;
}
void document_tool_execution_free(DocumentToolExecution *execution)
{
if (execution == NULL)
return;
g_free(execution->tool_id);
g_free(execution->version);
g_ptr_array_unref(execution->arguments);
g_free(execution->started_at_utc);
g_free(execution->finished_at_utc);
g_free(execution->source_path);
g_free(execution->source_sha256);
g_free(execution->raw_stdout);
g_free(execution->raw_stdout_sha256);
g_free(execution->raw_stderr);
g_ptr_array_unref(execution->warnings);
g_ptr_array_unref(execution->errors);
g_free(execution);
}
void document_tool_execution_add_argument(
DocumentToolExecution *execution,
const char *argument
)
{
if (execution != NULL && argument != NULL)
g_ptr_array_add(execution->arguments, g_strdup(argument));
}
const char *document_analysis_state_code(DocumentAnalysisState state)
{
switch (state)
{
case DOCUMENT_ANALYSIS_STATE_SUCCESS: return "success";
case DOCUMENT_ANALYSIS_STATE_PARTIAL: return "partial";
case DOCUMENT_ANALYSIS_STATE_UNAVAILABLE: return "unavailable";
case DOCUMENT_ANALYSIS_STATE_CANCELLED: return "cancelled";
case DOCUMENT_ANALYSIS_STATE_INCOMPATIBLE: return "incompatible";
case DOCUMENT_ANALYSIS_STATE_FAILED: return "failed";
}
return "failed";
}

101
src/core/document_file_analysis.c Normal file
View file

@ -0,0 +1,101 @@
/******************************************************************************
* @file document_file_analysis.c
* @brief Orchestration des analyses compatibles d'un fichier dérivé.
******************************************************************************/
#include "core/document_file_analysis.h"
void document_file_analysis_free(DocumentFileAnalysis *analysis)
{
if (analysis == NULL)
return;
g_free(analysis->source_path);
g_free(analysis->declared_mime);
g_free(analysis->detected_mime);
exiftool_analysis_result_free(analysis->metadata);
ocr_analysis_result_free(analysis->ocr);
pdf_analysis_result_free(analysis->pdf);
g_ptr_array_unref(analysis->warnings);
g_free(analysis);
}
static const char *document_file_analysis_effective_mime(
const char *declared_mime,
const char *detected_mime
)
{
return detected_mime != NULL && detected_mime[0] != '\0'
? detected_mime
: declared_mime;
}
DocumentFileAnalysis *document_file_analysis_run(
const DocumentAnalysisTools *tools,
const char *source_path,
const char *declared_mime,
const char *detected_mime,
gboolean request_image_ocr,
const char *ocr_languages,
GCancellable *cancellable,
GError **error
)
{
if (tools == NULL || source_path == NULL)
{
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
"Les paramètres d'analyse du fichier sont invalides.");
return NULL;
}
DocumentFileAnalysis *analysis = g_new0(DocumentFileAnalysis, 1);
analysis->source_path = g_strdup(source_path);
analysis->declared_mime = g_strdup(declared_mime);
analysis->detected_mime = g_strdup(detected_mime);
analysis->warnings = g_ptr_array_new_with_free_func(g_free);
analysis->state = DOCUMENT_ANALYSIS_STATE_SUCCESS;
const char *mime = document_file_analysis_effective_mime(
declared_mime, detected_mime);
analysis->metadata = exiftool_analysis_run(
tools->exiftool, source_path, cancellable, error);
if (analysis->metadata == NULL)
{
if (error != NULL && *error != NULL &&
g_error_matches(*error, G_IO_ERROR, G_IO_ERROR_CANCELLED))
goto failure;
g_clear_error(error);
analysis->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
}
else if (analysis->metadata->execution->state !=
DOCUMENT_ANALYSIS_STATE_SUCCESS)
analysis->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
if (g_strcmp0(mime, "application/pdf") == 0)
{
PdfAnalysisTools pdf_tools = {
.pdfinfo = tools->pdfinfo,
.pdftotext = tools->pdftotext,
.pdftoppm = tools->pdftoppm,
.tesseract = tools->tesseract
};
analysis->pdf = pdf_analysis_run(&pdf_tools, source_path,
ocr_languages, cancellable, error);
if (analysis->pdf == NULL)
goto failure;
if (analysis->pdf->state != DOCUMENT_ANALYSIS_STATE_SUCCESS)
analysis->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
}
else if (request_image_ocr && ocr_analysis_mime_is_compatible(mime))
{
analysis->ocr = ocr_analysis_run(tools->tesseract,
source_path, ocr_languages, cancellable, error);
if (analysis->ocr == NULL)
goto failure;
if (analysis->ocr->execution->state !=
DOCUMENT_ANALYSIS_STATE_SUCCESS)
analysis->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
}
return analysis;
failure:
document_file_analysis_free(analysis);
return NULL;
}

162
src/core/document_tool_runner.c Normal file
View file

@ -0,0 +1,162 @@
/******************************************************************************
* @file document_tool_runner.c
* @brief Exécution bornée et annulable des outils documentaires.
******************************************************************************/
#include "core/document_tool_runner.h"
#include "core/file_hash.h"
#include "core/tool_process.h"
#include <glib/gstdio.h>
char *document_tool_runner_read_version(
const char *executable,
const char *const arguments[],
GCancellable *cancellable
)
{
ToolProcessResult *result = NULL;
GError *error = NULL;
char *version = NULL;
if (!tool_process_run(executable, arguments, NULL, cancellable,
&result, &error))
{
g_clear_error(&error);
return NULL;
}
GBytes *stdout_bytes = tool_process_result_ref_stdout(result);
GBytes *stderr_bytes = tool_process_result_ref_stderr(result);
gsize stdout_length = 0;
gsize stderr_length = 0;
const char *stdout_data = g_bytes_get_data(
stdout_bytes, &stdout_length);
const char *stderr_data = g_bytes_get_data(
stderr_bytes, &stderr_length);
if (stdout_data == NULL)
stdout_data = "";
if (stderr_data == NULL)
stderr_data = "";
if (stdout_length > 0)
version = g_utf8_make_valid(stdout_data, (gssize) stdout_length);
else if (stderr_length > 0)
version = g_utf8_make_valid(stderr_data, (gssize) stderr_length);
if (version != NULL)
g_strstrip(version);
g_bytes_unref(stdout_bytes);
g_bytes_unref(stderr_bytes);
tool_process_result_free(result);
return version;
}
static char *document_tool_runner_bytes_to_text(
GBytes *bytes,
gsize limit,
gboolean *truncated
)
{
gsize length = 0;
const char *data = bytes != NULL
? g_bytes_get_data(bytes, &length)
: "";
if (data == NULL)
data = "";
if (length > limit)
{
length = limit;
*truncated = TRUE;
}
return g_utf8_make_valid(data, (gssize) length);
}
gboolean document_tool_runner_run(
const char *tool_id,
const char *executable,
const char *const arguments[],
const char *source_path,
GCancellable *cancellable,
DocumentToolExecution **out_execution,
GError **error
)
{
ToolProcessResult *process_result = NULL;
DocumentToolExecution *execution = NULL;
GError *process_error = NULL;
gboolean stdout_truncated = FALSE;
gboolean stderr_truncated = FALSE;
g_return_val_if_fail(error == NULL || *error == NULL, FALSE);
if (tool_id == NULL || executable == NULL || source_path == NULL ||
out_execution == NULL || *out_execution != NULL)
{
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
"Les paramètres de l'outil documentaire sont invalides.");
return FALSE;
}
GStatBuf source_stat;
if (g_stat(source_path, &source_stat) == 0 &&
source_stat.st_size > DOCUMENT_ANALYSIS_MAX_FILE_SIZE)
{
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_NO_SPACE,
"Le fichier dépasse la taille maximale d'analyse.");
return FALSE;
}
execution = document_tool_execution_new(tool_id, source_path);
for (gsize index = 0; arguments != NULL &&
arguments[index] != NULL; index++)
document_tool_execution_add_argument(execution, arguments[index]);
(void) file_hash_compute_sha256(source_path, cancellable,
&execution->source_sha256, NULL, NULL);
if (!tool_process_run(executable, arguments, NULL, cancellable,
&process_result, &process_error))
{
if (g_error_matches(process_error, TOOL_PROCESS_ERROR,
TOOL_PROCESS_ERROR_CANCELLED))
{
execution->state = DOCUMENT_ANALYSIS_STATE_CANCELLED;
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_CANCELLED,
"L'analyse documentaire a été annulée.");
}
else
{
execution->state = DOCUMENT_ANALYSIS_STATE_UNAVAILABLE;
g_ptr_array_add(execution->errors,
g_strdup(process_error != NULL ? process_error->message :
"Outil indisponible."));
}
g_clear_error(&process_error);
GDateTime *now = g_date_time_new_now_utc();
execution->finished_at_utc = g_date_time_format_iso8601(now);
g_date_time_unref(now);
*out_execution = execution;
return execution->state != DOCUMENT_ANALYSIS_STATE_CANCELLED;
}
GBytes *stdout_bytes = tool_process_result_ref_stdout(process_result);
GBytes *stderr_bytes = tool_process_result_ref_stderr(process_result);
execution->raw_stdout = document_tool_runner_bytes_to_text(stdout_bytes,
DOCUMENT_ANALYSIS_MAX_STDOUT, &stdout_truncated);
execution->raw_stderr = document_tool_runner_bytes_to_text(stderr_bytes,
DOCUMENT_ANALYSIS_MAX_STDERR, &stderr_truncated);
execution->exit_status =
tool_process_result_get_exit_status(process_result);
if (execution->raw_stdout != NULL)
execution->raw_stdout_sha256 = g_compute_checksum_for_string(
G_CHECKSUM_SHA256, execution->raw_stdout, -1);
if (stdout_truncated || stderr_truncated)
{
execution->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
g_ptr_array_add(execution->warnings,
g_strdup("La sortie de l'outil a été tronquée à la limite."));
}
else
execution->state = tool_process_result_is_success(process_result)
? DOCUMENT_ANALYSIS_STATE_SUCCESS
: DOCUMENT_ANALYSIS_STATE_FAILED;
GDateTime *now = g_date_time_new_now_utc();
execution->finished_at_utc = g_date_time_format_iso8601(now);
g_date_time_unref(now);
g_clear_pointer(&stdout_bytes, g_bytes_unref);
g_clear_pointer(&stderr_bytes, g_bytes_unref);
tool_process_result_free(process_result);
*out_execution = execution;
return TRUE;
}

85
src/core/eml_pipeline_task.c
View file

@ -4,7 +4,6 @@
******************************************************************************/
#include "core/eml_pipeline_task.h"
#include "core/file_hash.h"
#include "core/rib_ocr.h"
#include <gio/gio.h>
#include <glib.h>
#include <string.h>
@ -14,6 +13,11 @@ typedef struct
char *eml_path;
char *processed_evidence_dir;
char *evidence_id;
char *exiftool;
char *tesseract;
char *pdfinfo;
char *pdftotext;
char *pdftoppm;
} EmlPipelineTaskData;
static void eml_pipeline_task_data_free(gpointer user_data)
@ -24,6 +28,11 @@ static void eml_pipeline_task_data_free(gpointer user_data)
g_free(data->eml_path);
g_free(data->processed_evidence_dir);
g_free(data->evidence_id);
g_free(data->exiftool);
g_free(data->tesseract);
g_free(data->pdfinfo);
g_free(data->pdftotext);
g_free(data->pdftoppm);
g_free(data);
}
@ -37,6 +46,8 @@ void eml_pipeline_result_free(EmlPipelineResult *res)
eml_mime_result_free(res->mime_result);
if (res->bank_proposals != NULL)
g_ptr_array_unref(res->bank_proposals);
if (res->document_analyses != NULL)
g_ptr_array_unref(res->document_analyses);
if (res->warnings != NULL)
g_ptr_array_unref(res->warnings);
g_free(res);
@ -103,14 +114,20 @@ static gboolean eml_pipeline_task_worker(BackgroundTask *task,
background_task_report_progress(task, 0.75, "Analyse OCR et détection bancaire...");
GPtrArray *bank_proposals = g_ptr_array_new_with_free_func((GDestroyNotify) bank_proposal_free);
GPtrArray *document_analyses = g_ptr_array_new_with_free_func(
(GDestroyNotify) document_file_analysis_free);
for (guint i = 0; mime_res->attachments != NULL && i < mime_res->attachments->len; i++)
{
if (document_analyses->len >=
DOCUMENT_ANALYSIS_MAX_PIPELINE_ITEMS)
break;
if (g_cancellable_is_cancelled(cancellable))
{
eml_analysis_free(analysis);
eml_mime_result_free(mime_res);
g_ptr_array_unref(bank_proposals);
g_ptr_array_unref(document_analyses);
g_set_error_literal(
error,
G_IO_ERROR,
@ -139,13 +156,41 @@ static gboolean eml_pipeline_task_worker(BackgroundTask *task,
g_free(content);
}
}
else if (g_str_has_suffix(att->extracted_path, ".png") || g_str_has_suffix(att->extracted_path, ".jpg") || g_str_has_suffix(att->extracted_path, ".jpeg"))
else if (ocr_analysis_mime_is_compatible(att->detected_mime) ||
g_strcmp0(att->detected_mime, "application/pdf") == 0 ||
g_strcmp0(att->content_type, "application/pdf") == 0)
{
char *ocr_text = NULL;
char *ocr_version = NULL;
(void) rib_ocr_extract_text(att->extracted_path, &ocr_text,
&ocr_version, NULL);
g_free(ocr_version);
DocumentAnalysisTools tools = {
.exiftool = data->exiftool,
.tesseract = data->tesseract,
.pdfinfo = data->pdfinfo,
.pdftotext = data->pdftotext,
.pdftoppm = data->pdftoppm
};
GError *analysis_error = NULL;
DocumentFileAnalysis *document =
document_file_analysis_run(&tools, att->extracted_path,
att->content_type, att->detected_mime, TRUE,
"fra+eng", cancellable, &analysis_error);
if (document == NULL &&
analysis_error != NULL &&
g_error_matches(analysis_error, G_IO_ERROR,
G_IO_ERROR_CANCELLED))
{
g_propagate_error(error, analysis_error);
eml_analysis_free(analysis);
eml_mime_result_free(mime_res);
g_ptr_array_unref(bank_proposals);
g_ptr_array_unref(document_analyses);
return FALSE;
}
g_clear_error(&analysis_error);
if (document == NULL)
continue;
g_ptr_array_add(document_analyses, document);
const char *ocr_text = document->ocr != NULL
? document->ocr->text
: NULL;
if (ocr_text != NULL)
{
BankProposal *bp = bank_proposal_analyze_text(ocr_text, data->evidence_id);
@ -154,7 +199,6 @@ static gboolean eml_pipeline_task_worker(BackgroundTask *task,
bp->extraction_id = g_strdup(att->part_index);
g_ptr_array_add(bank_proposals, bp);
}
g_free(ocr_text);
}
}
}
@ -165,6 +209,7 @@ static gboolean eml_pipeline_task_worker(BackgroundTask *task,
res->analysis = analysis;
res->mime_result = mime_res;
res->bank_proposals = bank_proposals;
res->document_analyses = document_analyses;
res->warnings = g_ptr_array_new_with_free_func(g_free);
if (out_result != NULL)
@ -177,13 +222,35 @@ BackgroundTask *eml_pipeline_task_new(const char *eml_path,
const char *processed_evidence_dir,
const char *evidence_id)
{
if (eml_path == NULL || processed_evidence_dir == NULL)
DocumentAnalysisTools tools = {
.exiftool = "exiftool",
.tesseract = "tesseract",
.pdfinfo = "pdfinfo",
.pdftotext = "pdftotext",
.pdftoppm = "pdftoppm"
};
return eml_pipeline_task_new_with_tools(eml_path,
processed_evidence_dir, evidence_id, &tools);
}
BackgroundTask *eml_pipeline_task_new_with_tools(
const char *eml_path,
const char *processed_evidence_dir,
const char *evidence_id,
const DocumentAnalysisTools *tools)
{
if (eml_path == NULL || processed_evidence_dir == NULL || tools == NULL)
return NULL;
EmlPipelineTaskData *data = g_new0(EmlPipelineTaskData, 1);
data->eml_path = g_strdup(eml_path);
data->processed_evidence_dir = g_strdup(processed_evidence_dir);
data->evidence_id = g_strdup(evidence_id);
data->exiftool = g_strdup(tools->exiftool);
data->tesseract = g_strdup(tools->tesseract);
data->pdfinfo = g_strdup(tools->pdfinfo);
data->pdftotext = g_strdup(tools->pdftotext);
data->pdftoppm = g_strdup(tools->pdftoppm);
BackgroundTask *task = background_task_new(
"Analyse du message EML et de ses pièces jointes");

251
src/core/exiftool_analysis.c Normal file
View file

@ -0,0 +1,251 @@
/******************************************************************************
* @file exiftool_analysis.c
* @brief Analyse ExifTool structurée et traçable.
******************************************************************************/
#include "core/exiftool_analysis.h"
#include "core/document_tool_runner.h"
#include <string.h>
typedef struct
{
const char *tag;
const char *code;
gboolean sensitive;
} ExiftoolMapping;
static const ExiftoolMapping exiftool_mappings[] = {
{ "File:MIMEType", "file.mime_type", FALSE },
{ "File:FileSize", "file.size_bytes", FALSE },
{ "File:FileTypeExtension", "file.detected_extension", FALSE },
{ "EXIF:ImageWidth", "image.width", FALSE },
{ "EXIF:ImageHeight", "image.height", FALSE },
{ "EXIF:Orientation", "image.orientation", FALSE },
{ "EXIF:Software", "image.software", FALSE },
{ "EXIF:Make", "image.make", FALSE },
{ "EXIF:Model", "image.model", FALSE },
{ "EXIF:DateTimeOriginal", "image.datetime_original", FALSE },
{ "EXIF:GPSLatitude", "image.gps_latitude", TRUE },
{ "EXIF:GPSLongitude", "image.gps_longitude", TRUE },
{ "PDF:Author", "document.author", FALSE },
{ "PDF:Creator", "document.creator", FALSE },
{ "PDF:Producer", "document.producer", FALSE },
{ "PDF:CreateDate", "document.creation_time", FALSE },
{ "PDF:ModifyDate", "document.modification_time", FALSE }
};
static void exiftool_metadata_entry_free(gpointer data)
{
DocumentMetadataEntry *entry = data;
if (entry == NULL)
return;
g_free(entry->code);
g_free(entry->original_group);
g_free(entry->original_tag);
g_free(entry->raw_value);
g_free(entry);
}
void exiftool_analysis_result_free(ExiftoolAnalysisResult *result)
{
if (result == NULL)
return;
document_tool_execution_free(result->execution);
g_ptr_array_unref(result->metadata);
g_free(result);
}
static char *exiftool_json_extract_value(
const char *json,
const char *tag
)
{
char *escaped = g_regex_escape_string(tag, -1);
char *pattern = g_strdup_printf(
"\"%s\"\\s*:\\s*(\"(?:[^\"\\\\]|\\\\.)*\"|-?[0-9]+(?:\\.[0-9]+)?|true|false|null)",
escaped
);
GRegex *regex = g_regex_new(pattern, G_REGEX_DOTALL, 0, NULL);
GMatchInfo *match = NULL;
char *value = NULL;
g_regex_match(regex, json, 0, &match);
if (g_match_info_matches(match))
{
value = g_match_info_fetch(match, 1);
if (value[0] == '"' && strlen(value) >= 2)
{
gsize length = strlen(value);
memmove(value, value + 1, length - 2);
value[length - 2] = '\0';
}
}
g_match_info_free(match);
g_regex_unref(regex);
g_free(pattern);
g_free(escaped);
return value;
}
static gboolean exiftool_json_shape_is_valid(const char *json)
{
char *copy = json != NULL ? g_strdup(json) : NULL;
gboolean valid = FALSE;
if (copy != NULL)
{
g_strstrip(copy);
gsize length = strlen(copy);
valid = length >= 2 && copy[0] == '[' && copy[length - 1] == ']';
}
g_free(copy);
return valid;
}
static gboolean exiftool_metadata_contains_tag(
const GPtrArray *metadata,
const char *group,
const char *tag
)
{
for (guint index = 0; index < metadata->len; index++)
{
const DocumentMetadataEntry *entry =
g_ptr_array_index((GPtrArray *) metadata, index);
if (g_strcmp0(entry->original_group, group) == 0 &&
g_strcmp0(entry->original_tag, tag) == 0)
return TRUE;
}
return FALSE;
}
static void exiftool_analysis_add_unknown_tags(
ExiftoolAnalysisResult *result,
const char *json
)
{
GRegex *regex = g_regex_new(
"\"([A-Za-z0-9_ -]+):([A-Za-z0-9_ -]+)\"\\s*:\\s*"
"(\"(?:[^\"\\\\]|\\\\.)*\"|-?[0-9]+(?:\\.[0-9]+)?|true|false|null)",
G_REGEX_DOTALL, 0, NULL);
GMatchInfo *match = NULL;
g_regex_match(regex, json, 0, &match);
while (g_match_info_matches(match))
{
char *group = g_match_info_fetch(match, 1);
char *tag = g_match_info_fetch(match, 2);
char *value = g_match_info_fetch(match, 3);
if (!exiftool_metadata_contains_tag(result->metadata, group, tag))
{
DocumentMetadataEntry *entry =
g_new0(DocumentMetadataEntry, 1);
entry->code = g_strdup("metadata.unknown");
entry->original_group = group;
entry->original_tag = tag;
entry->raw_value = value;
g_ptr_array_add(result->metadata, entry);
}
else
{
g_free(group);
g_free(tag);
g_free(value);
}
if (!g_match_info_next(match, NULL))
break;
}
g_match_info_free(match);
g_regex_unref(regex);
}
ExiftoolAnalysisResult *exiftool_analysis_parse(
const char *file_path,
const char *json,
const char *stderr_text,
int exit_status,
GError **error
)
{
if (file_path == NULL || json == NULL)
{
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
"Le résultat ExifTool à analyser est invalide.");
return NULL;
}
if (!exiftool_json_shape_is_valid(json))
{
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_DATA,
"La sortie JSON ExifTool est invalide ou tronquée.");
return NULL;
}
ExiftoolAnalysisResult *result = g_new0(ExiftoolAnalysisResult, 1);
result->execution = document_tool_execution_new("exiftool", file_path);
result->metadata = g_ptr_array_new_with_free_func(
exiftool_metadata_entry_free);
result->execution->raw_stdout = g_strdup(json);
result->execution->raw_stdout_sha256 = g_compute_checksum_for_string(
G_CHECKSUM_SHA256, json, -1);
result->execution->raw_stderr = g_strdup(stderr_text);
result->execution->exit_status = exit_status;
result->execution->state = exit_status == 0
? DOCUMENT_ANALYSIS_STATE_SUCCESS
: DOCUMENT_ANALYSIS_STATE_PARTIAL;
for (guint index = 0; index < G_N_ELEMENTS(exiftool_mappings); index++)
{
char *value = exiftool_json_extract_value(
json, exiftool_mappings[index].tag);
if (value == NULL)
continue;
DocumentMetadataEntry *entry = g_new0(DocumentMetadataEntry, 1);
entry->code = g_strdup(exiftool_mappings[index].code);
const char *colon = strchr(exiftool_mappings[index].tag, ':');
entry->original_group = g_strndup(exiftool_mappings[index].tag,
(gsize) (colon - exiftool_mappings[index].tag));
entry->original_tag = g_strdup(colon + 1);
entry->raw_value = value;
entry->sensitive = exiftool_mappings[index].sensitive;
entry->requires_confirmation = entry->sensitive;
g_ptr_array_add(result->metadata, entry);
}
exiftool_analysis_add_unknown_tags(result, json);
return result;
}
ExiftoolAnalysisResult *exiftool_analysis_run(
const char *executable,
const char *file_path,
GCancellable *cancellable,
GError **error
)
{
const char *arguments[] = { "-j", "-G1", "-n", "--", file_path, NULL };
const char *version_arguments[] = { "-ver", NULL };
DocumentToolExecution *execution = NULL;
if (!document_tool_runner_run("exiftool", executable, arguments,
file_path, cancellable, &execution, error))
{
document_tool_execution_free(execution);
return NULL;
}
if (execution->state == DOCUMENT_ANALYSIS_STATE_UNAVAILABLE)
{
ExiftoolAnalysisResult *unavailable =
g_new0(ExiftoolAnalysisResult, 1);
unavailable->execution = execution;
unavailable->metadata = g_ptr_array_new_with_free_func(
exiftool_metadata_entry_free);
return unavailable;
}
execution->version = document_tool_runner_read_version(
executable, version_arguments, cancellable);
ExiftoolAnalysisResult *result = exiftool_analysis_parse(file_path,
execution->raw_stdout != NULL ? execution->raw_stdout : "",
execution->raw_stderr, execution->exit_status, error);
if (result != NULL)
{
document_tool_execution_free(result->execution);
result->execution = execution;
}
else
document_tool_execution_free(execution);
return result;
}

81
src/core/ocr_analysis.c Normal file
View file

@ -0,0 +1,81 @@
/******************************************************************************
* @file ocr_analysis.c
* @brief OCR Tesseract traçable et annulable.
******************************************************************************/
#include "core/ocr_analysis.h"
#include "core/document_tool_runner.h"
gboolean ocr_analysis_mime_is_compatible(const char *mime_type)
{
return g_strcmp0(mime_type, "image/png") == 0 ||
g_strcmp0(mime_type, "image/jpeg") == 0 ||
g_strcmp0(mime_type, "image/tiff") == 0;
}
void ocr_analysis_result_free(OcrAnalysisResult *result)
{
if (result == NULL)
return;
document_tool_execution_free(result->execution);
g_free(result->requested_languages);
g_free(result->text);
g_free(result);
}
OcrAnalysisResult *ocr_analysis_run(
const char *executable,
const char *image_path,
const char *languages,
GCancellable *cancellable,
GError **error
)
{
if (executable == NULL || image_path == NULL || languages == NULL ||
languages[0] == '\0' ||
(!g_str_equal(languages, "fra") &&
!g_str_equal(languages, "eng") &&
!g_str_equal(languages, "fra+eng")))
{
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
"Les paramètres OCR sont invalides.");
return NULL;
}
const char *arguments[] = {
image_path, "stdout", "-l", languages, NULL
};
const char *version_arguments[] = { "--version", NULL };
DocumentToolExecution *execution = NULL;
if (!document_tool_runner_run("tesseract", executable, arguments,
image_path, cancellable, &execution, error))
{
document_tool_execution_free(execution);
return NULL;
}
OcrAnalysisResult *result = g_new0(OcrAnalysisResult, 1);
result->execution = execution;
execution->version = document_tool_runner_read_version(
executable, version_arguments, cancellable);
result->requested_languages = g_strdup(languages);
if (execution->raw_stdout != NULL)
{
gsize length = strlen(execution->raw_stdout);
if (length > DOCUMENT_ANALYSIS_MAX_TEXT)
{
result->text = g_strndup(execution->raw_stdout,
DOCUMENT_ANALYSIS_MAX_TEXT);
execution->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
g_ptr_array_add(execution->warnings,
g_strdup("Le texte OCR dépasse la limite autorisée."));
}
else
result->text = g_strdup(execution->raw_stdout);
}
if (execution->state == DOCUMENT_ANALYSIS_STATE_SUCCESS &&
(result->text == NULL || result->text[0] == '\0'))
{
execution->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
g_ptr_array_add(execution->warnings,
g_strdup("Tesseract n'a produit aucun texte."));
}
return result;
}

294
src/core/pdf_analysis.c Normal file
View file

@ -0,0 +1,294 @@
/******************************************************************************
* @file pdf_analysis.c
* @brief Extraction PDF native puis OCR de secours.
******************************************************************************/
#include "core/pdf_analysis.h"
#include "core/document_tool_runner.h"
#include <glib/gstdio.h>
static void pdf_page_analysis_free(gpointer data)
{
PdfPageAnalysis *page = data;
if (page == NULL)
return;
g_free(page->text);
document_tool_execution_free(page->render_execution);
document_tool_execution_free(page->execution);
g_ptr_array_unref(page->warnings);
g_free(page);
}
void pdf_analysis_result_free(PdfAnalysisResult *result)
{
if (result == NULL)
return;
g_free(result->source_path);
g_free(result->native_text);
document_tool_execution_free(result->pdfinfo_execution);
document_tool_execution_free(result->native_execution);
g_ptr_array_unref(result->pages);
g_ptr_array_unref(result->warnings);
g_free(result);
}
gboolean pdf_analysis_text_is_usable(const char *text)
{
if (text == NULL)
return FALSE;
gsize total = 0;
gsize non_space = 0;
gsize printable = 0;
for (const char *cursor = text; *cursor != '\0';
cursor = g_utf8_next_char(cursor))
{
gunichar character = g_utf8_get_char(cursor);
total++;
if (!g_unichar_isspace(character))
non_space++;
if (g_unichar_isprint(character) || g_unichar_isspace(character))
printable++;
}
return non_space >= 32 && total > 0 &&
((double) printable / (double) total) >= 0.70;
}
static guint pdf_analysis_parse_pages(const char *text)
{
GRegex *regex = g_regex_new("(?im)^Pages:\\s*([0-9]+)", 0, 0, NULL);
GMatchInfo *match = NULL;
guint pages = 0;
g_regex_match(regex, text != NULL ? text : "", 0, &match);
if (g_match_info_matches(match))
{
char *value = g_match_info_fetch(match, 1);
pages = (guint) g_ascii_strtoull(value, NULL, 10);
g_free(value);
}
g_match_info_free(match);
g_regex_unref(regex);
return pages;
}
static gboolean pdf_analysis_parse_encrypted(const char *text)
{
GRegex *regex = g_regex_new(
"(?im)^Encrypted:\\s*(yes|oui|true)", 0, 0, NULL);
gboolean encrypted = g_regex_match(
regex, text != NULL ? text : "", 0, NULL);
g_regex_unref(regex);
return encrypted;
}
static PdfPageAnalysis *pdf_page_new(
guint page_number,
PdfPageMethod method,
const char *text,
DocumentAnalysisState state
)
{
PdfPageAnalysis *page = g_new0(PdfPageAnalysis, 1);
page->page_number = page_number;
page->method = method;
page->text = g_strdup(text);
page->state = state;
page->warnings = g_ptr_array_new_with_free_func(g_free);
return page;
}
static void pdf_analysis_add_native_pages(PdfAnalysisResult *result)
{
char **pages = g_strsplit(result->native_text, "\f", -1);
guint added = 0;
for (guint index = 0; pages[index] != NULL; index++)
{
if (pages[index][0] == '\0' && pages[index + 1] == NULL)
break;
g_ptr_array_add(result->pages, pdf_page_new(
index + 1, PDF_PAGE_METHOD_NATIVE, pages[index],
DOCUMENT_ANALYSIS_STATE_SUCCESS));
added++;
}
if (result->page_count == 0)
result->page_count = added;
g_strfreev(pages);
}
static gboolean pdf_analysis_render_and_ocr(
PdfAnalysisResult *result,
const PdfAnalysisTools *tools,
const char *languages,
GCancellable *cancellable,
GError **error
)
{
GError *temporary_error = NULL;
char *temporary_directory = g_dir_make_tmp(
"labfy-pdf-analysis-XXXXXX", &temporary_error);
if (temporary_directory == NULL)
{
g_propagate_error(error, temporary_error);
return FALSE;
}
guint pages = MIN(result->page_count, DOCUMENT_ANALYSIS_MAX_PDF_PAGES);
if (result->page_count > DOCUMENT_ANALYSIS_MAX_PDF_PAGES)
{
result->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
g_ptr_array_add(result->warnings,
g_strdup("Le nombre de pages PDF dépasse la limite."));
}
gboolean success = TRUE;
for (guint page_number = 1; page_number <= pages; page_number++)
{
if (cancellable != NULL &&
g_cancellable_set_error_if_cancelled(cancellable, error))
{
result->state = result->pages->len > 0
? DOCUMENT_ANALYSIS_STATE_PARTIAL
: DOCUMENT_ANALYSIS_STATE_CANCELLED;
success = FALSE;
break;
}
char *prefix = g_strdup_printf("%s/page-%u",
temporary_directory, page_number);
char *page_text = g_strdup_printf("%u", page_number);
const char *render_arguments[] = {
"-f", page_text, "-singlefile", "-png",
result->source_path, prefix, NULL
};
DocumentToolExecution *render_execution = NULL;
if (!document_tool_runner_run("pdftoppm", tools->pdftoppm,
render_arguments, result->source_path, cancellable,
&render_execution, error))
{
document_tool_execution_free(render_execution);
g_free(page_text);
g_free(prefix);
success = FALSE;
break;
}
char *image_path = g_strconcat(prefix, ".png", NULL);
if (render_execution->state != DOCUMENT_ANALYSIS_STATE_SUCCESS)
{
PdfPageAnalysis *page = pdf_page_new(page_number,
PDF_PAGE_METHOD_OCR, NULL,
DOCUMENT_ANALYSIS_STATE_FAILED);
page->execution = render_execution;
g_ptr_array_add(result->pages, page);
result->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
}
else
{
OcrAnalysisResult *ocr = ocr_analysis_run(
tools->tesseract, image_path, languages,
cancellable, error);
PdfPageAnalysis *page = pdf_page_new(page_number,
PDF_PAGE_METHOD_OCR,
ocr != NULL ? ocr->text : NULL,
ocr != NULL ? ocr->execution->state :
DOCUMENT_ANALYSIS_STATE_FAILED);
page->render_execution = render_execution;
if (ocr != NULL)
{
page->execution = ocr->execution;
ocr->execution = NULL;
ocr_analysis_result_free(ocr);
}
g_ptr_array_add(result->pages, page);
if (page->state != DOCUMENT_ANALYSIS_STATE_SUCCESS)
result->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
}
g_remove(image_path);
g_free(image_path);
g_free(page_text);
g_free(prefix);
}
g_rmdir(temporary_directory);
g_free(temporary_directory);
return success;
}
PdfAnalysisResult *pdf_analysis_run(
const PdfAnalysisTools *tools,
const char *pdf_path,
const char *ocr_languages,
GCancellable *cancellable,
GError **error
)
{
if (tools == NULL || pdf_path == NULL || ocr_languages == NULL)
{
g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
"Les paramètres d'analyse PDF sont invalides.");
return NULL;
}
PdfAnalysisResult *result = g_new0(PdfAnalysisResult, 1);
result->source_path = g_strdup(pdf_path);
result->pages = g_ptr_array_new_with_free_func(pdf_page_analysis_free);
result->warnings = g_ptr_array_new_with_free_func(g_free);
result->state = DOCUMENT_ANALYSIS_STATE_SUCCESS;
const char *info_arguments[] = { pdf_path, NULL };
if (!document_tool_runner_run("pdfinfo", tools->pdfinfo,
info_arguments, pdf_path, cancellable,
&result->pdfinfo_execution, error))
goto failure;
if (result->pdfinfo_execution->state ==
DOCUMENT_ANALYSIS_STATE_UNAVAILABLE)
{
result->state = DOCUMENT_ANALYSIS_STATE_UNAVAILABLE;
return result;
}
const char *version_arguments[] = { "-v", NULL };
result->pdfinfo_execution->version =
document_tool_runner_read_version(
tools->pdfinfo, version_arguments, cancellable);
result->encrypted = pdf_analysis_parse_encrypted(
result->pdfinfo_execution->raw_stdout);
result->page_count = pdf_analysis_parse_pages(
result->pdfinfo_execution->raw_stdout);
if (result->encrypted)
{
result->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
g_ptr_array_add(result->warnings,
g_strdup("Le PDF est chiffré ; aucun contournement n'est tenté."));
return result;
}
const char *text_arguments[] = {
"-enc", "UTF-8", "-layout", pdf_path, "-", NULL
};
if (!document_tool_runner_run("pdftotext", tools->pdftotext,
text_arguments, pdf_path, cancellable,
&result->native_execution, error))
goto failure;
result->native_execution->version =
document_tool_runner_read_version(
tools->pdftotext, version_arguments, cancellable);
if (result->native_execution->state ==
DOCUMENT_ANALYSIS_STATE_SUCCESS)
result->native_text = g_strdup(
result->native_execution->raw_stdout);
result->native_text_usable = pdf_analysis_text_is_usable(
result->native_text);
if (result->native_text_usable)
pdf_analysis_add_native_pages(result);
else if (!pdf_analysis_render_and_ocr(result, tools, ocr_languages,
cancellable, error))
{
if (error != NULL && *error != NULL &&
g_error_matches(*error, G_IO_ERROR, G_IO_ERROR_CANCELLED))
{
g_clear_error(error);
return result;
}
result->state = DOCUMENT_ANALYSIS_STATE_PARTIAL;
g_clear_error(error);
}
return result;
failure:
pdf_analysis_result_free(result);
return NULL;
}

85
tests/fake_document_tool.c Normal file
View file

@ -0,0 +1,85 @@
/******************************************************************************
* @file fake_document_tool.c
* @brief Faux outil documentaire synthétique, sans shell.
******************************************************************************/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
static int has_argument(int argc, char **argv, const char *value)
{
for (int index = 1; index < argc; index++)
if (strcmp(argv[index], value) == 0)
return 1;
return 0;
}
static const char *find_pdf_path(int argc, char **argv)
{
for (int index = 1; index < argc; index++)
if (strstr(argv[index], ".pdf") != NULL)
return argv[index];
return "";
}
int main(int argc, char **argv)
{
if (has_argument(argc, argv, "-ver"))
{
puts("13.00");
return 0;
}
if (has_argument(argc, argv, "--version"))
{
puts("tesseract 5.0.0-synthetic");
return 0;
}
if (has_argument(argc, argv, "-j"))
{
puts("[{\"File:MIMEType\":\"image/png\","
"\"File:FileSize\":42,\"EXIF:ImageWidth\":10,"
"\"EXIF:GPSLatitude\":48.5,\"EXIF:GPSLongitude\":2.2,"
"\"EXIF:SyntheticBoolean\":true}]");
return 0;
}
if (has_argument(argc, argv, "-enc"))
{
const char *path = find_pdf_path(argc, argv);
if (strstr(path, "native") != NULL)
{
puts("Texte synthétique suffisamment long pour être considéré "
"comme exploitable dans cette fixture.\fDeuxième page.");
}
return 0;
}
if (has_argument(argc, argv, "-singlefile"))
{
const char *prefix = argv[argc - 1];
char path[4096];
if (snprintf(path, sizeof(path), "%s.png", prefix) < 0)
return 2;
FILE *file = fopen(path, "wb");
if (file == NULL)
return 3;
fputs("synthetic-image", file);
fclose(file);
return 0;
}
if (has_argument(argc, argv, "stdout"))
{
if (strstr(argv[1], "sleep") != NULL)
sleep(2);
if (strstr(argv[1], "page-2") != NULL)
puts("Texte OCR synthétique page deux.");
else
puts("Texte OCR synthétique page une.");
return 0;
}
const char *path = find_pdf_path(argc, argv);
if (strstr(path, "encrypted") != NULL)
puts("Pages: 2\nEncrypted: yes");
else
puts("Pages: 2\nEncrypted: no");
return 0;
}

52
tests/test_eml_pipeline_task.c
View file

@ -86,9 +86,61 @@ static void test_eml_pipeline_basic(void)
g_free(tmp_dir);
}
static void test_eml_pipeline_document_analysis(void)
{
GError *error = NULL;
char *tmp_dir = g_dir_make_tmp("labfy-eml-document-XXXXXX", &error);
g_assert_no_error(error);
char *eml_path = g_build_filename(tmp_dir, "document.eml", NULL);
char *processed_dir = g_build_filename(
tmp_dir, "02_Preuves_Traitees", NULL);
static const char eml[] =
"From: synthetic@example.test\r\n"
"Content-Type: multipart/mixed; boundary=x\r\n\r\n"
"--x\r\nContent-Type: image/png; name=synthetic.png\r\n"
"Content-Disposition: attachment; filename=synthetic.png\r\n"
"Content-Transfer-Encoding: base64\r\n\r\n"
"UE5H\r\n--x--\r\n";
g_assert_true(g_file_set_contents(eml_path, eml, -1, &error));
g_assert_no_error(error);
DocumentAnalysisTools tools = {
.exiftool = "tests/fake_document_tool",
.tesseract = "tests/fake_document_tool",
.pdfinfo = "tests/fake_document_tool",
.pdftotext = "tests/fake_document_tool",
.pdftoppm = "tests/fake_document_tool"
};
BackgroundTask *task = eml_pipeline_task_new_with_tools(
eml_path, processed_dir, "synthetic-evidence", &tools);
g_assert_nonnull(task);
while (background_task_get_state(task) ==
BACKGROUND_TASK_STATE_RUNNING ||
background_task_get_state(task) ==
BACKGROUND_TASK_STATE_PENDING)
g_main_context_iteration(NULL, TRUE);
g_assert_cmpint(background_task_get_state(task), ==,
BACKGROUND_TASK_STATE_COMPLETED);
EmlPipelineResult *result = background_task_get_result(task);
g_assert_nonnull(result);
g_assert_cmpuint(result->document_analyses->len, ==, 1);
DocumentFileAnalysis *document = g_ptr_array_index(
result->document_analyses, 0);
g_assert_nonnull(document->metadata);
g_assert_nonnull(document->ocr);
g_assert_cmpstr(document->ocr->text, ==,
"Texte OCR synthétique page une.\n");
background_task_unref(task);
g_remove(eml_path);
g_free(processed_dir);
g_free(eml_path);
g_free(tmp_dir);
}
int main(int argc, char **argv)
{
g_test_init(&argc, &argv, NULL);
g_test_add_func("/eml-pipeline-task/basic", test_eml_pipeline_basic);
g_test_add_func("/eml-pipeline-task/document-analysis",
test_eml_pipeline_document_analysis);
return g_test_run();
}

80
tests/test_exiftool_analysis.c Normal file
View file

@ -0,0 +1,80 @@
/******************************************************************************
* @file test_exiftool_analysis.c
* @brief Tests synthétiques de l'analyse ExifTool.
******************************************************************************/
#include "core/exiftool_analysis.h"
#include <glib.h>
#include <glib/gstdio.h>
static void test_parse_and_sensitive_gps(void)
{
const char *json =
"[{\"File:MIMEType\":\"image/png\",\"File:FileSize\":42,"
"\"EXIF:ImageWidth\":10,\"EXIF:GPSLatitude\":48.5,"
"\"EXIF:GPSLongitude\":2.2,\"EXIF:Unknown\":true}]";
GError *error = NULL;
ExiftoolAnalysisResult *result = exiftool_analysis_parse(
"synthetic.png", json, "synthetic warning", 1, &error);
g_assert_no_error(error);
g_assert_nonnull(result);
g_assert_cmpint(result->execution->state, ==,
DOCUMENT_ANALYSIS_STATE_PARTIAL);
g_assert_cmpstr(result->execution->raw_stdout, ==, json);
g_assert_nonnull(result->execution->raw_stdout_sha256);
g_assert_cmpuint(result->metadata->len, ==, 6);
DocumentMetadataEntry *latitude =
g_ptr_array_index(result->metadata, 3);
g_assert_cmpstr(latitude->code, ==, "image.gps_latitude");
g_assert_true(latitude->sensitive);
g_assert_true(latitude->requires_confirmation);
DocumentMetadataEntry *unknown =
g_ptr_array_index(result->metadata, 5);
g_assert_cmpstr(unknown->code, ==, "metadata.unknown");
g_assert_cmpstr(unknown->original_tag, ==, "Unknown");
exiftool_analysis_result_free(result);
}
static void test_invalid_json(void)
{
GError *error = NULL;
g_assert_null(exiftool_analysis_parse(
"synthetic.png", "[{\"broken\":", NULL, 0, &error));
g_assert_error(error, G_IO_ERROR, G_IO_ERROR_INVALID_DATA);
g_clear_error(&error);
}
static void test_run_and_unavailable(void)
{
GError *error = NULL;
char *directory = g_dir_make_tmp("labfy-exif-XXXXXX", &error);
char *path = g_build_filename(directory, "synthetic.png", NULL);
g_assert_true(g_file_set_contents(path, "PNG", 3, &error));
ExiftoolAnalysisResult *result = exiftool_analysis_run(
"tests/fake_document_tool", path, NULL, &error);
g_assert_no_error(error);
g_assert_cmpint(result->execution->state, ==,
DOCUMENT_ANALYSIS_STATE_SUCCESS);
g_assert_cmpstr(result->execution->version, ==, "13.00");
exiftool_analysis_result_free(result);
result = exiftool_analysis_run(
"tests/missing_document_tool", path, NULL, &error);
g_assert_no_error(error);
g_assert_cmpint(result->execution->state, ==,
DOCUMENT_ANALYSIS_STATE_UNAVAILABLE);
exiftool_analysis_result_free(result);
g_remove(path);
g_rmdir(directory);
g_free(path);
g_free(directory);
}
int main(int argc, char **argv)
{
g_test_init(&argc, &argv, NULL);
g_test_add_func("/exiftool-analysis/parse-gps",
test_parse_and_sensitive_gps);
g_test_add_func("/exiftool-analysis/invalid-json", test_invalid_json);
g_test_add_func("/exiftool-analysis/run-unavailable",
test_run_and_unavailable);
return g_test_run();
}

87
tests/test_ocr_analysis.c Normal file
View file

@ -0,0 +1,87 @@
/******************************************************************************
* @file test_ocr_analysis.c
* @brief Tests synthétiques de l'OCR.
******************************************************************************/
#include "core/ocr_analysis.h"
#include <glib.h>
#include <glib/gstdio.h>
static void test_languages_and_raw_text(void)
{
GError *error = NULL;
char *directory = g_dir_make_tmp("labfy-ocr-XXXXXX", &error);
char *path = g_build_filename(directory, "synthetic.png", NULL);
g_assert_true(g_file_set_contents(path, "PNG", 3, &error));
const char *languages[] = { "fra", "eng", "fra+eng" };
for (guint index = 0; index < G_N_ELEMENTS(languages); index++)
{
OcrAnalysisResult *result = ocr_analysis_run(
"tests/fake_document_tool", path, languages[index],
NULL, &error);
g_assert_no_error(error);
g_assert_cmpstr(result->requested_languages, ==, languages[index]);
g_assert_cmpstr(result->text, ==,
"Texte OCR synthétique page une.\n");
g_assert_cmpstr(result->execution->raw_stdout, ==, result->text);
g_assert_nonnull(result->execution->version);
ocr_analysis_result_free(result);
}
g_remove(path);
g_rmdir(directory);
g_free(path);
g_free(directory);
}
static void test_unavailable_and_compatibility(void)
{
GError *error = NULL;
OcrAnalysisResult *result = ocr_analysis_run(
"tests/missing_document_tool", "synthetic.png", "fra",
NULL, &error);
g_assert_no_error(error);
g_assert_cmpint(result->execution->state, ==,
DOCUMENT_ANALYSIS_STATE_UNAVAILABLE);
ocr_analysis_result_free(result);
g_assert_true(ocr_analysis_mime_is_compatible("image/png"));
g_assert_true(ocr_analysis_mime_is_compatible("image/tiff"));
g_assert_false(ocr_analysis_mime_is_compatible("text/plain"));
}
static gpointer cancel_ocr(gpointer user_data)
{
g_usleep(50000);
g_cancellable_cancel(user_data);
return NULL;
}
static void test_cancellation(void)
{
GError *error = NULL;
char *directory = g_dir_make_tmp("labfy-ocr-XXXXXX", &error);
char *path = g_build_filename(directory, "sleep.png", NULL);
g_assert_true(g_file_set_contents(path, "PNG", 3, &error));
GCancellable *cancellable = g_cancellable_new();
GThread *thread = g_thread_new("ocr-cancel", cancel_ocr, cancellable);
OcrAnalysisResult *result = ocr_analysis_run(
"tests/fake_document_tool", path, "fra", cancellable, &error);
g_thread_join(thread);
g_assert_null(result);
g_assert_error(error, G_IO_ERROR, G_IO_ERROR_CANCELLED);
g_clear_error(&error);
g_object_unref(cancellable);
g_remove(path);
g_rmdir(directory);
g_free(path);
g_free(directory);
}
int main(int argc, char **argv)
{
g_test_init(&argc, &argv, NULL);
g_test_add_func("/ocr-analysis/languages-raw",
test_languages_and_raw_text);
g_test_add_func("/ocr-analysis/unavailable-compatible",
test_unavailable_and_compatibility);
g_test_add_func("/ocr-analysis/cancellation", test_cancellation);
return g_test_run();
}

96
tests/test_pdf_analysis.c Normal file
View file

@ -0,0 +1,96 @@
/******************************************************************************
* @file test_pdf_analysis.c
* @brief Tests synthétiques de l'analyse PDF.
******************************************************************************/
#include "core/pdf_analysis.h"
#include <glib.h>
#include <glib/gstdio.h>
static PdfAnalysisTools fake_tools(void)
{
PdfAnalysisTools tools = {
.pdfinfo = "tests/fake_document_tool",
.pdftotext = "tests/fake_document_tool",
.pdftoppm = "tests/fake_document_tool",
.tesseract = "tests/fake_document_tool"
};
return tools;
}
static char *create_pdf(const char *directory, const char *name)
{
char *path = g_build_filename(directory, name, NULL);
g_assert_true(g_file_set_contents(path, "%PDF-synthetic", -1, NULL));
return path;
}
static void test_encrypted_and_native(void)
{
GError *error = NULL;
char *directory = g_dir_make_tmp("labfy-pdf-XXXXXX", &error);
PdfAnalysisTools tools = fake_tools();
char *encrypted = create_pdf(directory, "encrypted.pdf");
PdfAnalysisResult *result = pdf_analysis_run(
&tools, encrypted, "fra", NULL, &error);
g_assert_no_error(error);
g_assert_true(result->encrypted);
g_assert_cmpuint(result->pages->len, ==, 0);
pdf_analysis_result_free(result);
char *native = create_pdf(directory, "native.pdf");
result = pdf_analysis_run(&tools, native, "fra", NULL, &error);
g_assert_no_error(error);
g_assert_true(result->native_text_usable);
g_assert_cmpuint(result->pages->len, ==, 2);
PdfPageAnalysis *page = g_ptr_array_index(result->pages, 0);
g_assert_cmpint(page->method, ==, PDF_PAGE_METHOD_NATIVE);
pdf_analysis_result_free(result);
g_remove(encrypted);
g_remove(native);
g_rmdir(directory);
g_free(encrypted);
g_free(native);
g_free(directory);
}
static void test_ocr_fallback_order_and_cleanup(void)
{
GError *error = NULL;
char *directory = g_dir_make_tmp("labfy-pdf-XXXXXX", &error);
char *scan = create_pdf(directory, "scan.pdf");
PdfAnalysisTools tools = fake_tools();
PdfAnalysisResult *result = pdf_analysis_run(
&tools, scan, "fra+eng", NULL, &error);
g_assert_no_error(error);
g_assert_false(result->native_text_usable);
g_assert_cmpuint(result->pages->len, ==, 2);
PdfPageAnalysis *first = g_ptr_array_index(result->pages, 0);
PdfPageAnalysis *second = g_ptr_array_index(result->pages, 1);
g_assert_cmpuint(first->page_number, ==, 1);
g_assert_cmpuint(second->page_number, ==, 2);
g_assert_cmpint(first->method, ==, PDF_PAGE_METHOD_OCR);
g_assert_nonnull(strstr(second->text, "page deux"));
pdf_analysis_result_free(result);
g_remove(scan);
g_rmdir(directory);
g_free(scan);
g_free(directory);
}
static void test_heuristic(void)
{
g_assert_false(pdf_analysis_text_is_usable(""));
g_assert_false(pdf_analysis_text_is_usable("court"));
g_assert_true(pdf_analysis_text_is_usable(
"Texte synthétique imprimable et suffisamment long pour le test."));
}
int main(int argc, char **argv)
{
g_test_init(&argc, &argv, NULL);
g_test_add_func("/pdf-analysis/encrypted-native",
test_encrypted_and_native);
g_test_add_func("/pdf-analysis/ocr-fallback-cleanup",
test_ocr_fallback_order_and_cleanup);
g_test_add_func("/pdf-analysis/heuristic", test_heuristic);
return g_test_run();
}