feat(bank): ajouter l’extraction bancaire structurée
This commit is contained in:
parent
e97b35a4df
commit
ccf9bdd87a
8 changed files with 1363 additions and 0 deletions
11
Makefile
11
Makefile
|
|
@ -176,6 +176,7 @@ TEST_EML_ANALYZER := tests/test_eml_analyzer
|
|||
TEST_EML_INTEGRATION := tests/test_eml_integration
|
||||
TEST_IBAN_ANALYZER := tests/test_iban_analyzer
|
||||
TEST_FINANCIAL_FOUNDATION := tests/test_financial_foundation
|
||||
TEST_BANK_STRUCTURED_EXTRACTOR := tests/test_bank_structured_extractor
|
||||
TEST_EXIFTOOL_METADATA := tests/test_exiftool_metadata
|
||||
TEST_PDF_PASSWORD_RECOVERY := tests/test_pdf_password_recovery
|
||||
TEST_EXTRACTION_DROP_SERVICE := tests/test_extraction_drop_service
|
||||
|
|
@ -1137,6 +1138,13 @@ $(TEST_FINANCIAL_FOUNDATION): tests/test_financial_foundation.c \
|
|||
src/core/iban_analyzer.c src/core/bank_proposal.c src/core/controlled_vocab.c
|
||||
$(CC) $(TEST_CFLAGS) -Wpedantic $^ -o $@ $(TEST_LDFLAGS)
|
||||
|
||||
$(TEST_BANK_STRUCTURED_EXTRACTOR): tests/test_bank_structured_extractor.c \
|
||||
src/models/bank_analysis.c src/models/financial_foundation.c \
|
||||
src/core/bank_structured_extractor.c src/core/bank_analysis_revision.c \
|
||||
src/core/bank_text_extractor.c src/core/bic_validator.c \
|
||||
src/core/iban_analyzer.c
|
||||
$(CC) $(TEST_CFLAGS) -Wpedantic $^ -o $@ $(TEST_LDFLAGS)
|
||||
|
||||
$(TEST_EXIFTOOL_METADATA): tests/test_exiftool_metadata.c \
|
||||
src/core/exiftool_metadata.c
|
||||
$(CC) $(TEST_CFLAGS) $^ -o $@ $(TEST_LDFLAGS)
|
||||
|
|
@ -1246,6 +1254,7 @@ test: \
|
|||
$(TEST_EML_INTEGRATION) \
|
||||
$(TEST_IBAN_ANALYZER) \
|
||||
$(TEST_FINANCIAL_FOUNDATION) \
|
||||
$(TEST_BANK_STRUCTURED_EXTRACTOR) \
|
||||
$(TEST_EXIFTOOL_METADATA) \
|
||||
$(TEST_PDF_PASSWORD_RECOVERY) \
|
||||
$(TEST_EXTRACTION_DROP_SERVICE) \
|
||||
|
|
@ -1348,6 +1357,7 @@ test: \
|
|||
@$(TEST_EML_INTEGRATION)
|
||||
@$(TEST_IBAN_ANALYZER)
|
||||
@$(TEST_FINANCIAL_FOUNDATION)
|
||||
@$(TEST_BANK_STRUCTURED_EXTRACTOR)
|
||||
@$(TEST_EXIFTOOL_METADATA)
|
||||
@$(TEST_PDF_PASSWORD_RECOVERY)
|
||||
@$(TEST_EXTRACTION_DROP_SERVICE)
|
||||
|
|
@ -1461,6 +1471,7 @@ clean:
|
|||
$(TEST_CONTROLLED_VOCAB) \
|
||||
$(TEST_BANK_PROPOSAL) \
|
||||
$(TEST_FINANCIAL_FOUNDATION) \
|
||||
$(TEST_BANK_STRUCTURED_EXTRACTOR) \
|
||||
$(TEST_EML_PIPELINE_TASK) \
|
||||
$(TEST_EML_MIME_EXTRACTOR) \
|
||||
$(TEST_EXIFTOOL_ANALYSIS) \
|
||||
|
|
|
|||
27
include/core/bank_analysis_revision.h
Normal file
27
include/core/bank_analysis_revision.h
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
#ifndef LABFY_INVESTIGATION_BANK_ANALYSIS_REVISION_H
|
||||
#define LABFY_INVESTIGATION_BANK_ANALYSIS_REVISION_H
|
||||
|
||||
#include "models/bank_analysis.h"
|
||||
|
||||
G_BEGIN_DECLS
|
||||
|
||||
/* Ces opérations ne modifient que les blocs en mémoire. Elles ne suppriment
|
||||
* jamais une occurrence ni ne modifient source_text/raw_value. */
|
||||
gboolean bank_analysis_revision_create_block(BankAnalysisResult *result,
|
||||
const char *identifier, const char *occurrence_identifier, GError **error);
|
||||
gboolean bank_analysis_revision_remove(BankAnalysisResult *result,
|
||||
const char *block_identifier, const char *occurrence_identifier, GError **error);
|
||||
gboolean bank_analysis_revision_move(BankAnalysisResult *result,
|
||||
const char *from_block, const char *to_block, const char *occurrence, GError **error);
|
||||
gboolean bank_analysis_revision_merge(BankAnalysisResult *result,
|
||||
const char *target_block, const char *source_block, GError **error);
|
||||
gboolean bank_analysis_revision_split(BankAnalysisResult *result,
|
||||
const char *source_block, const char *new_block,
|
||||
const char *occurrence_identifier, GError **error);
|
||||
gboolean bank_analysis_revision_mark_ambiguous(BankAnalysisResult *result,
|
||||
const char *block_identifier, gboolean ambiguous, GError **error);
|
||||
gboolean bank_analysis_revision_reject(BankAnalysisResult *result,
|
||||
const char *block_identifier, GError **error);
|
||||
|
||||
G_END_DECLS
|
||||
#endif
|
||||
22
include/core/bank_structured_extractor.h
Normal file
22
include/core/bank_structured_extractor.h
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
#ifndef LABFY_INVESTIGATION_BANK_STRUCTURED_EXTRACTOR_H
|
||||
#define LABFY_INVESTIGATION_BANK_STRUCTURED_EXTRACTOR_H
|
||||
|
||||
#include "models/bank_analysis.h"
|
||||
|
||||
G_BEGIN_DECLS
|
||||
|
||||
/* Analyse pure et déterministe. Le résultat possède toutes ses données.
|
||||
* NULL et le texte vide produisent un résultat vide. Une valeur factuelle
|
||||
* dépassant BANK_ANALYSIS_MAX_VALUE_BYTES fait échouer l'analyse. */
|
||||
/* Un montant exige un marqueur explicite ou une devise adjacente. Sa séquence
|
||||
* numérique est délimitée sans copie et la ponctuation périphérique est
|
||||
* exclue. Les groupes par espaces ou séparateurs ponctués doivent respecter
|
||||
* 1-3 chiffres puis des groupes de 3. Les formes à séparateur unique suivi de
|
||||
* trois chiffres (1,234 et 1.234) restent ambiguës. Les formes incohérentes
|
||||
* restent entières, invalides et sans normalisation. Le contexte puis toutes
|
||||
* les limites sont contrôlés avant toute allocation proportionnelle. */
|
||||
BankAnalysisResult *bank_structured_extractor_analyze(const char *text,
|
||||
GError **error);
|
||||
|
||||
G_END_DECLS
|
||||
#endif
|
||||
114
include/models/bank_analysis.h
Normal file
114
include/models/bank_analysis.h
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
#ifndef LABFY_INVESTIGATION_BANK_ANALYSIS_H
|
||||
#define LABFY_INVESTIGATION_BANK_ANALYSIS_H
|
||||
|
||||
#include "models/financial_foundation.h"
|
||||
|
||||
G_BEGIN_DECLS
|
||||
|
||||
#define BANK_ANALYSIS_MAX_OCCURRENCES 4096U
|
||||
#define BANK_ANALYSIS_MAX_BLOCKS 1024U
|
||||
#define BANK_ANALYSIS_MAX_TRANSACTIONS 256U
|
||||
#define BANK_ANALYSIS_MAX_VALUE_BYTES 512U
|
||||
#define BANK_ANALYSIS_MAX_CONTEXT_BYTES 1024U
|
||||
#define BANK_ANALYSIS_MAX_ITEMS_PER_BLOCK 256U
|
||||
#define BANK_ANALYSIS_MAX_WARNINGS 256U
|
||||
#define BANK_ANALYSIS_MAX_ID_BYTES 128U
|
||||
#define BANK_ANALYSIS_MAX_AMOUNT_BYTES 128U
|
||||
#define BANK_ANALYSIS_MAX_AMOUNT_DIGITS 64U
|
||||
#define BANK_ANALYSIS_MAX_AMOUNT_GROUPS 16U
|
||||
#define BANK_ANALYSIS_MAX_AMOUNT_SEPARATORS 16U
|
||||
#define BANK_ANALYSIS_MAX_AMOUNT_PRECISION 8U
|
||||
|
||||
typedef enum {
|
||||
BANK_OCCURRENCE_IBAN, BANK_OCCURRENCE_BIC,
|
||||
BANK_OCCURRENCE_DECLARED_HOLDER, BANK_OCCURRENCE_DECLARED_INSTITUTION,
|
||||
BANK_OCCURRENCE_DECLARED_ADDRESS, BANK_OCCURRENCE_AMOUNT,
|
||||
BANK_OCCURRENCE_CURRENCY, BANK_OCCURRENCE_OBSERVED_DATE,
|
||||
BANK_OCCURRENCE_OBSERVED_TIME, BANK_OCCURRENCE_OBSERVED_STATUS,
|
||||
BANK_OCCURRENCE_BANK_REFERENCE, BANK_OCCURRENCE_UETR,
|
||||
BANK_OCCURRENCE_DECLARED_PAYER, BANK_OCCURRENCE_DECLARED_BENEFICIARY,
|
||||
BANK_OCCURRENCE_DEBTOR_ACCOUNT, BANK_OCCURRENCE_CREDITOR_ACCOUNT,
|
||||
BANK_OCCURRENCE_COUNT
|
||||
} BankOccurrenceType;
|
||||
|
||||
/* Toutes les chaînes sont possédées. warnings est possédé par l'occurrence,
|
||||
* possède ses éléments char* et les détruit avec g_free. Les offsets sont
|
||||
* des octets dans source_text ; end_offset est exclusif et désigne exactement
|
||||
* raw_value. bank_occurrence_free() détruit l'ensemble. */
|
||||
typedef struct {
|
||||
char *identifier;
|
||||
BankOccurrenceType type;
|
||||
char *raw_value;
|
||||
char *normalized_value;
|
||||
gsize start_offset;
|
||||
gsize end_offset;
|
||||
BankValidationState validation;
|
||||
char *extraction_rule;
|
||||
GPtrArray *warnings;
|
||||
char *source_context;
|
||||
} BankOccurrence;
|
||||
|
||||
typedef struct {
|
||||
char *identifier;
|
||||
/* Tableau possédé de char* possédés, détruits avec g_free. Tout élément
|
||||
* lu directement est emprunté et invalidé par retrait ou destruction. */
|
||||
GPtrArray *occurrence_identifiers;
|
||||
gsize start_offset;
|
||||
gsize end_offset;
|
||||
/* Tableau possédé de char* possédés, détruits avec g_free. Tout élément
|
||||
* lu directement est emprunté et invalidé par retrait ou destruction. */
|
||||
GPtrArray *reasons;
|
||||
gboolean ambiguous;
|
||||
BankOrigin origin;
|
||||
BankHumanDecision decision;
|
||||
} BankProposedBlock;
|
||||
|
||||
typedef struct {
|
||||
char *identifier;
|
||||
/* Chaque tableau ci-dessous est possédé, possède ses éléments char* et
|
||||
* les détruit avec g_free. Un élément lu directement est emprunté et
|
||||
* invalidé par retrait ou destruction du tableau. */
|
||||
GPtrArray *source_occurrence_identifiers;
|
||||
GPtrArray *missing_fields;
|
||||
GPtrArray *ambiguities;
|
||||
GPtrArray *reasons;
|
||||
BankHumanDecision decision;
|
||||
gboolean create_transaction;
|
||||
gboolean create_account;
|
||||
gboolean create_person;
|
||||
gboolean create_relation;
|
||||
} BankTransactionProposal;
|
||||
|
||||
typedef struct {
|
||||
char *source_text;
|
||||
/* Tableaux possédés. occurrences, blocks et transaction_proposals
|
||||
* possèdent leurs objets et appellent respectivement bank_occurrence_free,
|
||||
* bank_proposed_block_free et bank_transaction_proposal_free. warnings
|
||||
* possède ses char* et les détruit avec g_free. Un élément lu directement
|
||||
* est emprunté ; suppression/fusion ou destruction peut l'invalider. */
|
||||
GPtrArray *occurrences;
|
||||
GPtrArray *blocks;
|
||||
GPtrArray *transaction_proposals;
|
||||
GPtrArray *warnings;
|
||||
gboolean occurrence_limit_reached;
|
||||
gboolean block_limit_reached;
|
||||
gboolean transaction_limit_reached;
|
||||
gboolean amount_limit_reached;
|
||||
} BankAnalysisResult;
|
||||
|
||||
const char *bank_occurrence_type_to_code(BankOccurrenceType type);
|
||||
void bank_occurrence_free(BankOccurrence *occurrence);
|
||||
void bank_proposed_block_free(BankProposedBlock *block);
|
||||
void bank_transaction_proposal_free(BankTransactionProposal *proposal);
|
||||
void bank_analysis_result_free(BankAnalysisResult *result);
|
||||
/* Ces fonctions retournent des pointeurs empruntés appartenant à result ;
|
||||
* l'appelant ne doit jamais les libérer. Une occurrence reste valide pendant
|
||||
* la vie de result. Un bloc reste valide jusqu'à sa suppression éventuelle
|
||||
* par une fusion de révision, ou jusqu'à la libération de result. */
|
||||
BankOccurrence *bank_analysis_find_occurrence(const BankAnalysisResult *result,
|
||||
const char *identifier);
|
||||
BankProposedBlock *bank_analysis_find_block(const BankAnalysisResult *result,
|
||||
const char *identifier);
|
||||
|
||||
G_END_DECLS
|
||||
#endif
|
||||
163
src/core/bank_analysis_revision.c
Normal file
163
src/core/bank_analysis_revision.c
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
#include "core/bank_analysis_revision.h"
|
||||
|
||||
static GQuark revision_error(void)
|
||||
{ return g_quark_from_static_string("bank-analysis-revision-error"); }
|
||||
|
||||
static gboolean fail(GError **error, const char *message)
|
||||
{
|
||||
g_set_error_literal(error, revision_error(), 1, message); return FALSE;
|
||||
}
|
||||
|
||||
static gboolean valid_identifier(const char *identifier)
|
||||
{
|
||||
if (identifier == NULL || identifier[0] == '\0') return FALSE;
|
||||
gsize length = 0;
|
||||
while (length <= BANK_ANALYSIS_MAX_ID_BYTES && identifier[length] != '\0')
|
||||
length++;
|
||||
return length <= BANK_ANALYSIS_MAX_ID_BYTES;
|
||||
}
|
||||
|
||||
static gint identifier_index(const GPtrArray *items, const char *identifier)
|
||||
{
|
||||
if (items == NULL || identifier == NULL) return -1;
|
||||
for (guint i = 0; i < items->len; i++)
|
||||
if (g_strcmp0(g_ptr_array_index((GPtrArray *)items, i), identifier) == 0)
|
||||
return (gint)i;
|
||||
return -1;
|
||||
}
|
||||
|
||||
static void recompute_bounds(BankAnalysisResult *result, BankProposedBlock *block)
|
||||
{
|
||||
if (block->occurrence_identifiers->len == 0) {
|
||||
block->start_offset = block->end_offset = 0; return;
|
||||
}
|
||||
block->start_offset = G_MAXSIZE; block->end_offset = 0;
|
||||
for (guint i = 0; i < block->occurrence_identifiers->len; i++) {
|
||||
BankOccurrence *o = bank_analysis_find_occurrence(result,
|
||||
g_ptr_array_index(block->occurrence_identifiers, i));
|
||||
block->start_offset = MIN(block->start_offset, o->start_offset);
|
||||
block->end_offset = MAX(block->end_offset, o->end_offset);
|
||||
}
|
||||
}
|
||||
|
||||
gboolean bank_analysis_revision_create_block(BankAnalysisResult *result,
|
||||
const char *identifier, const char *occurrence_identifier, GError **error)
|
||||
{
|
||||
if (result == NULL || !valid_identifier(identifier) ||
|
||||
(occurrence_identifier != NULL && !valid_identifier(occurrence_identifier)) ||
|
||||
bank_analysis_find_block(result, identifier) != NULL ||
|
||||
result->blocks->len >= BANK_ANALYSIS_MAX_BLOCKS)
|
||||
return fail(error, "Le bloc manuel est invalide.");
|
||||
BankOccurrence *o = occurrence_identifier == NULL ? NULL :
|
||||
bank_analysis_find_occurrence(result, occurrence_identifier);
|
||||
if (occurrence_identifier != NULL && o == NULL)
|
||||
return fail(error, "L’occurrence est introuvable.");
|
||||
BankProposedBlock *block = g_new0(BankProposedBlock, 1);
|
||||
block->identifier = g_strdup(identifier);
|
||||
block->occurrence_identifiers = g_ptr_array_new_with_free_func(g_free);
|
||||
block->reasons = g_ptr_array_new_with_free_func(g_free);
|
||||
g_ptr_array_add(block->reasons, g_strdup("human-modification"));
|
||||
block->origin = BANK_ORIGIN_HUMAN; block->decision = BANK_DECISION_PROPOSED;
|
||||
if (o != NULL) {
|
||||
g_ptr_array_add(block->occurrence_identifiers, g_strdup(o->identifier));
|
||||
block->start_offset = o->start_offset; block->end_offset = o->end_offset;
|
||||
}
|
||||
g_ptr_array_add(result->blocks, block); return TRUE;
|
||||
}
|
||||
|
||||
gboolean bank_analysis_revision_remove(BankAnalysisResult *result,
|
||||
const char *block_id, const char *occurrence_id, GError **error)
|
||||
{
|
||||
if (!valid_identifier(block_id) || !valid_identifier(occurrence_id))
|
||||
return fail(error, "Un identifiant est invalide.");
|
||||
BankProposedBlock *block = bank_analysis_find_block(result, block_id);
|
||||
gint index = block == NULL ? -1 : identifier_index(block->occurrence_identifiers, occurrence_id);
|
||||
if (index < 0) return fail(error, "Le bloc ou l’occurrence est introuvable.");
|
||||
g_ptr_array_remove_index(block->occurrence_identifiers, (guint)index);
|
||||
block->origin = BANK_ORIGIN_HUMAN; recompute_bounds(result, block); return TRUE;
|
||||
}
|
||||
|
||||
gboolean bank_analysis_revision_move(BankAnalysisResult *result,
|
||||
const char *from_id, const char *to_id, const char *occurrence_id, GError **error)
|
||||
{
|
||||
if (!valid_identifier(from_id) || !valid_identifier(to_id) ||
|
||||
!valid_identifier(occurrence_id))
|
||||
return fail(error, "Un identifiant est invalide.");
|
||||
BankProposedBlock *from = bank_analysis_find_block(result, from_id);
|
||||
BankProposedBlock *to = bank_analysis_find_block(result, to_id);
|
||||
gint source_index = from == NULL ? -1 : identifier_index(from->occurrence_identifiers, occurrence_id);
|
||||
if (source_index < 0 || to == NULL ||
|
||||
identifier_index(to->occurrence_identifiers, occurrence_id) >= 0 ||
|
||||
to->occurrence_identifiers->len >= BANK_ANALYSIS_MAX_ITEMS_PER_BLOCK)
|
||||
return fail(error, "Le déplacement demandé est invalide.");
|
||||
char *copy = g_strdup(occurrence_id);
|
||||
g_ptr_array_add(to->occurrence_identifiers, copy);
|
||||
g_ptr_array_remove_index(from->occurrence_identifiers, (guint)source_index);
|
||||
from->origin = to->origin = BANK_ORIGIN_HUMAN;
|
||||
recompute_bounds(result, from); recompute_bounds(result, to); return TRUE;
|
||||
}
|
||||
|
||||
gboolean bank_analysis_revision_merge(BankAnalysisResult *result,
|
||||
const char *target_id, const char *source_id, GError **error)
|
||||
{
|
||||
if (!valid_identifier(target_id) || !valid_identifier(source_id))
|
||||
return fail(error, "Un identifiant est invalide.");
|
||||
BankProposedBlock *target = bank_analysis_find_block(result, target_id);
|
||||
BankProposedBlock *source = bank_analysis_find_block(result, source_id);
|
||||
if (target == NULL || source == NULL || target == source)
|
||||
return fail(error, "La fusion demandée est invalide.");
|
||||
guint additions = 0;
|
||||
for (guint i = 0; i < source->occurrence_identifiers->len; i++)
|
||||
if (identifier_index(target->occurrence_identifiers,
|
||||
g_ptr_array_index(source->occurrence_identifiers, i)) < 0) additions++;
|
||||
if (target->occurrence_identifiers->len + additions > BANK_ANALYSIS_MAX_ITEMS_PER_BLOCK)
|
||||
return fail(error, "Le bloc fusionné dépasserait la limite.");
|
||||
GPtrArray *copies = g_ptr_array_new_with_free_func(g_free);
|
||||
for (guint i = 0; i < source->occurrence_identifiers->len; i++) {
|
||||
const char *id = g_ptr_array_index(source->occurrence_identifiers, i);
|
||||
if (identifier_index(target->occurrence_identifiers, id) < 0)
|
||||
g_ptr_array_add(copies, g_strdup(id));
|
||||
}
|
||||
for (guint i = 0; i < copies->len; i++)
|
||||
g_ptr_array_add(target->occurrence_identifiers, g_strdup(g_ptr_array_index(copies, i)));
|
||||
g_ptr_array_unref(copies); target->origin = BANK_ORIGIN_HUMAN;
|
||||
g_ptr_array_add(target->reasons, g_strdup("human-modification"));
|
||||
recompute_bounds(result, target);
|
||||
for (guint i = 0; i < result->blocks->len; i++)
|
||||
if (g_ptr_array_index(result->blocks, i) == source) {
|
||||
g_ptr_array_remove_index(result->blocks, i); break;
|
||||
}
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
gboolean bank_analysis_revision_split(BankAnalysisResult *result,
|
||||
const char *source_id, const char *new_id, const char *occurrence_id, GError **error)
|
||||
{
|
||||
if (!valid_identifier(source_id) || !valid_identifier(new_id) ||
|
||||
!valid_identifier(occurrence_id))
|
||||
return fail(error, "Un identifiant est invalide.");
|
||||
BankProposedBlock *source = bank_analysis_find_block(result, source_id);
|
||||
if (source == NULL || identifier_index(source->occurrence_identifiers, occurrence_id) < 0)
|
||||
return fail(error, "La séparation demandée est invalide.");
|
||||
if (!bank_analysis_revision_create_block(result, new_id, occurrence_id, error)) return FALSE;
|
||||
return bank_analysis_revision_remove(result, source_id, occurrence_id, error);
|
||||
}
|
||||
|
||||
gboolean bank_analysis_revision_mark_ambiguous(BankAnalysisResult *result,
|
||||
const char *block_id, gboolean ambiguous, GError **error)
|
||||
{
|
||||
if (!valid_identifier(block_id)) return fail(error, "L’identifiant est invalide.");
|
||||
BankProposedBlock *block = bank_analysis_find_block(result, block_id);
|
||||
if (block == NULL) return fail(error, "Le bloc est introuvable.");
|
||||
block->ambiguous = ambiguous; block->origin = BANK_ORIGIN_HUMAN; return TRUE;
|
||||
}
|
||||
|
||||
gboolean bank_analysis_revision_reject(BankAnalysisResult *result,
|
||||
const char *block_id, GError **error)
|
||||
{
|
||||
if (!valid_identifier(block_id)) return fail(error, "L’identifiant est invalide.");
|
||||
BankProposedBlock *block = bank_analysis_find_block(result, block_id);
|
||||
if (block == NULL) return fail(error, "Le bloc est introuvable.");
|
||||
block->decision = BANK_DECISION_REJECTED; block->origin = BANK_ORIGIN_HUMAN;
|
||||
return TRUE;
|
||||
}
|
||||
539
src/core/bank_structured_extractor.c
Normal file
539
src/core/bank_structured_extractor.c
Normal file
|
|
@ -0,0 +1,539 @@
|
|||
#include "core/bank_structured_extractor.h"
|
||||
#include "core/bank_text_extractor.h"
|
||||
#include "core/bic_validator.h"
|
||||
#include <string.h>
|
||||
|
||||
static GQuark analysis_error(void)
|
||||
{ return g_quark_from_static_string("bank-structured-extractor-error"); }
|
||||
|
||||
static char *bounded_context(const char *text, gsize start, gsize end)
|
||||
{
|
||||
gsize length = strlen(text), left = start > 128U ? start - 128U : 0U;
|
||||
gsize right = MIN(length, end + 128U);
|
||||
if (right - left > BANK_ANALYSIS_MAX_CONTEXT_BYTES)
|
||||
right = left + BANK_ANALYSIS_MAX_CONTEXT_BYTES;
|
||||
return g_strndup(text + left, right - left);
|
||||
}
|
||||
|
||||
static BankOccurrence *occurrence_new(const char *text, BankOccurrenceType type,
|
||||
gsize start, gsize end, const char *normalized, BankValidationState validation,
|
||||
const char *rule, GError **error)
|
||||
{
|
||||
gsize length = strlen(text);
|
||||
if (start > end || end > length || end - start > BANK_ANALYSIS_MAX_VALUE_BYTES) {
|
||||
g_set_error_literal(error, analysis_error(), 2,
|
||||
"Une valeur bancaire dépasse les limites autorisées.");
|
||||
return NULL;
|
||||
}
|
||||
BankOccurrence *o = g_new0(BankOccurrence, 1);
|
||||
o->type = type; o->raw_value = g_strndup(text + start, end - start);
|
||||
o->normalized_value = g_strdup(normalized); o->start_offset = start;
|
||||
o->end_offset = end; o->validation = validation;
|
||||
o->extraction_rule = g_strdup(rule);
|
||||
o->warnings = g_ptr_array_new_with_free_func(g_free);
|
||||
o->source_context = bounded_context(text, start, end);
|
||||
return o;
|
||||
}
|
||||
|
||||
static gboolean add_occurrence(BankAnalysisResult *r, BankOccurrence *o)
|
||||
{
|
||||
if (r->occurrences->len >= BANK_ANALYSIS_MAX_OCCURRENCES) {
|
||||
r->occurrence_limit_reached = TRUE; bank_occurrence_free(o); return FALSE;
|
||||
}
|
||||
g_ptr_array_add(r->occurrences, o); return TRUE;
|
||||
}
|
||||
|
||||
static gint occurrence_compare(gconstpointer a, gconstpointer b)
|
||||
{
|
||||
const BankOccurrence *left = *(BankOccurrence * const *)a;
|
||||
const BankOccurrence *right = *(BankOccurrence * const *)b;
|
||||
if (left->start_offset != right->start_offset)
|
||||
return left->start_offset < right->start_offset ? -1 : 1;
|
||||
if (left->end_offset != right->end_offset)
|
||||
return left->end_offset < right->end_offset ? -1 : 1;
|
||||
return (gint)left->type - (gint)right->type;
|
||||
}
|
||||
|
||||
static gboolean overlaps_type(const BankAnalysisResult *r, BankOccurrenceType type,
|
||||
gsize start, gsize end)
|
||||
{
|
||||
for (guint i = 0; i < r->occurrences->len; i++) {
|
||||
BankOccurrence *o = g_ptr_array_index(r->occurrences, i);
|
||||
if (o->type == type && o->start_offset == start && o->end_offset == end)
|
||||
return TRUE;
|
||||
}
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
static gboolean scan_regex(BankAnalysisResult *r, const char *pattern,
|
||||
BankOccurrenceType type, guint capture, BankValidationState validation,
|
||||
const char *rule, gboolean normalize_upper, GError **error)
|
||||
{
|
||||
GRegex *regex = g_regex_new(pattern, G_REGEX_OPTIMIZE, 0, error);
|
||||
if (regex == NULL) return FALSE;
|
||||
gsize text_length = strlen(r->source_text);
|
||||
GMatchInfo *matches = NULL; g_regex_match(regex, r->source_text, 0, &matches);
|
||||
while (g_match_info_matches(matches)) {
|
||||
gint start = 0, end = 0; g_match_info_fetch_pos(matches, capture, &start, &end);
|
||||
if (start >= 0 && end >= start && !overlaps_type(r, type, start, end)) {
|
||||
if ((gsize)end < text_length &&
|
||||
r->source_text[end] != '\r' && r->source_text[end] != '\n') {
|
||||
g_set_error_literal(error, analysis_error(), 2,
|
||||
"Une valeur bancaire dépasse les limites autorisées.");
|
||||
g_match_info_free(matches); g_regex_unref(regex); return FALSE;
|
||||
}
|
||||
char *raw = g_strndup(r->source_text + start, (gsize)(end - start));
|
||||
char *normalized = normalize_upper ? g_utf8_strup(raw, -1) : NULL;
|
||||
BankOccurrence *o = occurrence_new(r->source_text, type, start, end,
|
||||
normalized, validation, rule, error);
|
||||
g_free(normalized); g_free(raw);
|
||||
if (o == NULL) { g_match_info_free(matches); g_regex_unref(regex); return FALSE; }
|
||||
add_occurrence(r, o);
|
||||
}
|
||||
if (!g_match_info_next(matches, error)) break;
|
||||
}
|
||||
g_match_info_free(matches); g_regex_unref(regex);
|
||||
return error == NULL || *error == NULL;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
BankValidationState state;
|
||||
gsize decimal_offset;
|
||||
guint digits;
|
||||
} AmountGrammar;
|
||||
|
||||
static gboolean amount_byte(const char *text, gsize offset, gsize length)
|
||||
{
|
||||
if (offset >= length) return FALSE;
|
||||
return g_ascii_isdigit(text[offset]) || text[offset] == ',' ||
|
||||
text[offset] == '.' || text[offset] == ' ' || text[offset] == '\t' ||
|
||||
((guchar)text[offset] == 0xc2U && offset + 1U < length &&
|
||||
(guchar)text[offset + 1U] == 0xa0U);
|
||||
}
|
||||
|
||||
static gboolean amount_has_numeric_before(const char *text, gsize offset)
|
||||
{
|
||||
if (offset == 0U) return FALSE;
|
||||
gsize cursor = offset;
|
||||
while (cursor > 0U && (text[cursor - 1U] == ' ' || text[cursor - 1U] == '\t'))
|
||||
cursor--;
|
||||
if (cursor == 0U) return FALSE;
|
||||
return g_ascii_isdigit(text[cursor - 1U]) || text[cursor - 1U] == ',' ||
|
||||
text[cursor - 1U] == '.';
|
||||
}
|
||||
|
||||
static gboolean currency_at(const char *text, gsize offset, gsize length,
|
||||
gsize *currency_length)
|
||||
{
|
||||
if (offset < length && text[offset] == '$') { *currency_length = 1U; return TRUE; }
|
||||
if (offset + 3U <= length && memcmp(text + offset, "€", 3U) == 0) {
|
||||
*currency_length = 3U; return TRUE;
|
||||
}
|
||||
if (offset + 3U <= length &&
|
||||
(g_ascii_strncasecmp(text + offset, "EUR", 3U) == 0 ||
|
||||
g_ascii_strncasecmp(text + offset, "USD", 3U) == 0)) {
|
||||
*currency_length = 3U; return TRUE;
|
||||
}
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
static gboolean amount_marker_before(const char *text, gsize number_start)
|
||||
{
|
||||
gsize line_start = number_start;
|
||||
while (line_start > 0U && text[line_start - 1U] != '\n' &&
|
||||
text[line_start - 1U] != '\r') line_start--;
|
||||
gsize length = number_start - line_start;
|
||||
if (length > 64U) line_start = number_start - 64U, length = 64U;
|
||||
char *prefix = g_strndup(text + line_start, length);
|
||||
gboolean found = g_regex_match_simple(
|
||||
"(?i)(montant|amount|somme|total)[ \\t]*:[ \\t]*(?:\\(|\\[)?[ \\t]*$",
|
||||
prefix, 0, 0);
|
||||
g_free(prefix); return found;
|
||||
}
|
||||
|
||||
static gboolean currency_before(const char *text, gsize number_start,
|
||||
gsize *currency_start, gsize *currency_length)
|
||||
{
|
||||
gsize cursor = number_start;
|
||||
while (cursor > 0U && (text[cursor - 1U] == ' ' || text[cursor - 1U] == '\t'))
|
||||
cursor--;
|
||||
gsize candidates[] = {3U, 1U};
|
||||
for (guint i = 0; i < G_N_ELEMENTS(candidates); i++) {
|
||||
if (cursor < candidates[i]) continue;
|
||||
gsize start = cursor - candidates[i], length = 0;
|
||||
if (currency_at(text, start, number_start, &length) && length == candidates[i]) {
|
||||
if (start > 0U && g_ascii_isalnum(text[start - 1U])) continue;
|
||||
*currency_start = start; *currency_length = length; return TRUE;
|
||||
}
|
||||
}
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
static gboolean currency_after(const char *text, gsize number_end, gsize text_length,
|
||||
gsize *currency_start, gsize *currency_length)
|
||||
{
|
||||
gsize cursor = number_end;
|
||||
while (cursor < text_length && (text[cursor] == ' ' || text[cursor] == '\t'))
|
||||
cursor++;
|
||||
if (!currency_at(text, cursor, text_length, currency_length)) return FALSE;
|
||||
if (cursor + *currency_length < text_length &&
|
||||
g_ascii_isalnum(text[cursor + *currency_length])) return FALSE;
|
||||
*currency_start = cursor; return TRUE;
|
||||
}
|
||||
|
||||
static gboolean amount_limits_ok(const char *raw, gsize length,
|
||||
guint *digits, guint *separators, guint *groups)
|
||||
{
|
||||
*digits = *separators = *groups = 0;
|
||||
if (length > BANK_ANALYSIS_MAX_AMOUNT_BYTES) return FALSE;
|
||||
for (gsize i = 0; i < length; i++) {
|
||||
if (g_ascii_isdigit(raw[i])) (*digits)++;
|
||||
else if (raw[i] == ',' || raw[i] == '.') (*separators)++;
|
||||
else if (raw[i] == ' ' || raw[i] == '\t' || (guchar)raw[i] == 0xc2U) {
|
||||
(*groups)++;
|
||||
if ((guchar)raw[i] == 0xc2U && i + 1U < length &&
|
||||
(guchar)raw[i + 1U] == 0xa0U) i++;
|
||||
}
|
||||
if (*digits > BANK_ANALYSIS_MAX_AMOUNT_DIGITS ||
|
||||
*separators > BANK_ANALYSIS_MAX_AMOUNT_SEPARATORS ||
|
||||
*groups > BANK_ANALYSIS_MAX_AMOUNT_GROUPS) return FALSE;
|
||||
}
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
static gboolean grouped_integer_valid(const char *raw, gsize end, char separator)
|
||||
{
|
||||
guint group_digits = 0, group_index = 0;
|
||||
for (gsize i = 0; i <= end; i++) {
|
||||
if (i < end && g_ascii_isdigit(raw[i])) { group_digits++; continue; }
|
||||
if (i < end && raw[i] != separator) return FALSE;
|
||||
if ((group_index == 0U && (group_digits == 0U || group_digits > 3U)) ||
|
||||
(group_index > 0U && group_digits != 3U)) return FALSE;
|
||||
group_index++; group_digits = 0;
|
||||
}
|
||||
return group_index > 1U;
|
||||
}
|
||||
|
||||
static AmountGrammar analyze_amount_grammar(const char *raw, gsize length)
|
||||
{
|
||||
AmountGrammar grammar = {BANK_VALIDATION_INVALID, G_MAXSIZE, 0U};
|
||||
guint commas = 0, dots = 0, spaces = 0;
|
||||
gsize last_comma = G_MAXSIZE, last_dot = G_MAXSIZE;
|
||||
for (gsize i = 0; i < length; i++) {
|
||||
if (g_ascii_isdigit(raw[i])) grammar.digits++;
|
||||
else if (raw[i] == ',') commas++, last_comma = i;
|
||||
else if (raw[i] == '.') dots++, last_dot = i;
|
||||
else if (raw[i] == ' ') spaces++;
|
||||
else return grammar; /* tabulation, espace insécable ou octet inattendu */
|
||||
}
|
||||
if (grammar.digits == 0U) return grammar;
|
||||
guint punctuations = commas + dots;
|
||||
if (spaces > 0U) {
|
||||
if (punctuations > 1U) return grammar;
|
||||
grammar.decimal_offset = punctuations == 1U ?
|
||||
(commas == 1U ? last_comma : last_dot) : G_MAXSIZE;
|
||||
gsize integer_end = grammar.decimal_offset == G_MAXSIZE ? length : grammar.decimal_offset;
|
||||
if (!grouped_integer_valid(raw, integer_end, ' ')) return grammar;
|
||||
if (grammar.decimal_offset != G_MAXSIZE) {
|
||||
gsize precision = length - grammar.decimal_offset - 1U;
|
||||
if (precision == 0U || precision > BANK_ANALYSIS_MAX_AMOUNT_PRECISION)
|
||||
return grammar;
|
||||
}
|
||||
grammar.state = BANK_VALIDATION_VALID; return grammar;
|
||||
}
|
||||
if (punctuations == 0U) {
|
||||
grammar.state = BANK_VALIDATION_VALID; return grammar;
|
||||
}
|
||||
if (punctuations == 1U) {
|
||||
grammar.decimal_offset = commas == 1U ? last_comma : last_dot;
|
||||
gsize precision = length - grammar.decimal_offset - 1U;
|
||||
if (precision == 0U || precision > BANK_ANALYSIS_MAX_AMOUNT_PRECISION)
|
||||
return grammar;
|
||||
grammar.state = precision == 3U ? BANK_VALIDATION_AMBIGUOUS :
|
||||
BANK_VALIDATION_VALID;
|
||||
if (grammar.state != BANK_VALIDATION_VALID) grammar.decimal_offset = G_MAXSIZE;
|
||||
return grammar;
|
||||
}
|
||||
if (commas > 0U && dots > 0U) {
|
||||
grammar.decimal_offset = MAX(last_comma, last_dot);
|
||||
char decimal = raw[grammar.decimal_offset];
|
||||
char thousands = decimal == ',' ? '.' : ',';
|
||||
gsize precision = length - grammar.decimal_offset - 1U;
|
||||
if (precision == 0U || precision > BANK_ANALYSIS_MAX_AMOUNT_PRECISION ||
|
||||
!grouped_integer_valid(raw, grammar.decimal_offset, thousands))
|
||||
return grammar;
|
||||
for (gsize i = 0; i < grammar.decimal_offset; i++)
|
||||
if (!g_ascii_isdigit(raw[i]) && raw[i] != thousands) return grammar;
|
||||
for (gsize i = grammar.decimal_offset + 1U; i < length; i++)
|
||||
if (!g_ascii_isdigit(raw[i])) return grammar;
|
||||
grammar.state = BANK_VALIDATION_VALID; return grammar;
|
||||
}
|
||||
char separator = commas > 0U ? ',' : '.';
|
||||
if (grouped_integer_valid(raw, length, separator)) {
|
||||
grammar.state = BANK_VALIDATION_VALID; grammar.decimal_offset = G_MAXSIZE;
|
||||
}
|
||||
return grammar;
|
||||
}
|
||||
|
||||
static char *normalize_valid_amount(const char *raw, gsize length,
|
||||
const AmountGrammar *grammar)
|
||||
{
|
||||
GString *normalized = g_string_sized_new(grammar->digits + 1U);
|
||||
for (gsize i = 0; i < length; i++) {
|
||||
if (g_ascii_isdigit(raw[i])) g_string_append_c(normalized, raw[i]);
|
||||
else if (i == grammar->decimal_offset) g_string_append_c(normalized, '.');
|
||||
}
|
||||
return g_string_free(normalized, FALSE);
|
||||
}
|
||||
|
||||
static void signal_amount_limit(BankAnalysisResult *result)
|
||||
{
|
||||
result->amount_limit_reached = TRUE;
|
||||
for (guint i = 0; i < result->warnings->len; i++)
|
||||
if (g_strcmp0(g_ptr_array_index(result->warnings, i),
|
||||
"amount-limit-reached") == 0) return;
|
||||
if (result->warnings->len < BANK_ANALYSIS_MAX_WARNINGS)
|
||||
g_ptr_array_add(result->warnings, g_strdup("amount-limit-reached"));
|
||||
}
|
||||
|
||||
static gboolean scan_amounts(BankAnalysisResult *r, GError **error)
|
||||
{
|
||||
const char *text = r->source_text; gsize text_length = strlen(text);
|
||||
for (gsize start = 0; start < text_length; start++) {
|
||||
if (!g_ascii_isdigit(text[start]) || amount_has_numeric_before(text, start))
|
||||
continue;
|
||||
gsize scan_end = start + 1U;
|
||||
while (amount_byte(text, scan_end, text_length)) {
|
||||
if ((guchar)text[scan_end] == 0xc2U) scan_end += 2U;
|
||||
else scan_end++;
|
||||
}
|
||||
gsize number_end = scan_end;
|
||||
while (number_end > start && (text[number_end - 1U] == ' ' ||
|
||||
text[number_end - 1U] == '\t')) number_end--;
|
||||
while (number_end > start && (text[number_end - 1U] == ',' ||
|
||||
text[number_end - 1U] == '.')) number_end--;
|
||||
gsize before_currency_start = 0, before_currency_length = 0;
|
||||
gsize after_currency_start = 0, after_currency_length = 0;
|
||||
gboolean has_before_currency = currency_before(text, start,
|
||||
&before_currency_start, &before_currency_length);
|
||||
gboolean has_after_currency = currency_after(text, number_end, text_length,
|
||||
&after_currency_start, &after_currency_length);
|
||||
gboolean has_context = amount_marker_before(text, start) ||
|
||||
has_before_currency || has_after_currency;
|
||||
if (!has_context) { start = scan_end - 1U; continue; }
|
||||
gsize candidate_length = number_end - start;
|
||||
if (candidate_length > BANK_ANALYSIS_MAX_AMOUNT_BYTES) {
|
||||
signal_amount_limit(r); start = scan_end - 1U; continue;
|
||||
}
|
||||
guint digits = 0, separators = 0, groups = 0;
|
||||
if (!amount_limits_ok(text + start, candidate_length,
|
||||
&digits, &separators, &groups)) {
|
||||
signal_amount_limit(r); start = scan_end - 1U; continue;
|
||||
}
|
||||
AmountGrammar grammar = analyze_amount_grammar(text + start, candidate_length);
|
||||
char *normalized = grammar.state == BANK_VALIDATION_VALID ?
|
||||
normalize_valid_amount(text + start, candidate_length, &grammar) : NULL;
|
||||
BankOccurrence *amount = occurrence_new(text, BANK_OCCURRENCE_AMOUNT,
|
||||
start, number_end, normalized, grammar.state, "amount-explicit", error);
|
||||
g_free(normalized); if (amount == NULL) return FALSE;
|
||||
add_occurrence(r, amount);
|
||||
gsize currency_start = has_after_currency ? after_currency_start : before_currency_start;
|
||||
gsize currency_length = has_after_currency ? after_currency_length : before_currency_length;
|
||||
if (has_after_currency || has_before_currency) {
|
||||
const char *currency_raw = text + currency_start;
|
||||
const char *code = currency_length == 3U &&
|
||||
memcmp(currency_raw, "€", 3U) == 0 ? "EUR" :
|
||||
(currency_length == 1U ? "USD" : NULL);
|
||||
char *upper = code != NULL ? g_strdup(code) :
|
||||
g_ascii_strup(currency_raw, currency_length);
|
||||
if (!overlaps_type(r, BANK_OCCURRENCE_CURRENCY, currency_start,
|
||||
currency_start + currency_length)) {
|
||||
BankOccurrence *currency = occurrence_new(text, BANK_OCCURRENCE_CURRENCY,
|
||||
currency_start, currency_start + currency_length, upper,
|
||||
BANK_VALIDATION_VALID, "explicit-currency", error);
|
||||
g_free(upper); if (currency == NULL) return FALSE;
|
||||
add_occurrence(r, currency);
|
||||
} else g_free(upper);
|
||||
}
|
||||
start = scan_end - 1U;
|
||||
}
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
static gboolean same_paragraph(const char *text, gsize left, gsize right)
|
||||
{
|
||||
if (left > right) { gsize tmp = left; left = right; right = tmp; }
|
||||
for (gsize i = left; i + 1U < right; i++)
|
||||
if (text[i] == '\n' && text[i + 1U] == '\n') return FALSE;
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
static void build_blocks(BankAnalysisResult *r)
|
||||
{
|
||||
BankProposedBlock *current = NULL;
|
||||
for (guint i = 0; i < r->occurrences->len; i++) {
|
||||
BankOccurrence *o = g_ptr_array_index(r->occurrences, i);
|
||||
if (current == NULL || !same_paragraph(r->source_text,
|
||||
current->end_offset, o->start_offset)) {
|
||||
if (r->blocks->len >= BANK_ANALYSIS_MAX_BLOCKS) {
|
||||
r->block_limit_reached = TRUE; break;
|
||||
}
|
||||
current = g_new0(BankProposedBlock, 1);
|
||||
current->identifier = g_strdup_printf("block-%04u", r->blocks->len + 1U);
|
||||
current->occurrence_identifiers = g_ptr_array_new_with_free_func(g_free);
|
||||
current->reasons = g_ptr_array_new_with_free_func(g_free);
|
||||
g_ptr_array_add(current->reasons, g_strdup("same-paragraph"));
|
||||
current->start_offset = o->start_offset; current->origin = BANK_ORIGIN_AUTOMATIC;
|
||||
current->decision = BANK_DECISION_PROPOSED; g_ptr_array_add(r->blocks, current);
|
||||
}
|
||||
if (current->occurrence_identifiers->len < BANK_ANALYSIS_MAX_ITEMS_PER_BLOCK)
|
||||
g_ptr_array_add(current->occurrence_identifiers, g_strdup(o->identifier));
|
||||
else current->ambiguous = TRUE;
|
||||
current->end_offset = MAX(current->end_offset, o->end_offset);
|
||||
}
|
||||
for (guint i = 0; i < r->blocks->len; i++) {
|
||||
BankProposedBlock *block = g_ptr_array_index(r->blocks, i);
|
||||
guint holders = 0, ibans = 0, bics = 0;
|
||||
for (guint j = 0; j < block->occurrence_identifiers->len; j++) {
|
||||
BankOccurrence *o = bank_analysis_find_occurrence(r,
|
||||
g_ptr_array_index(block->occurrence_identifiers, j));
|
||||
holders += o->type == BANK_OCCURRENCE_DECLARED_HOLDER;
|
||||
ibans += o->type == BANK_OCCURRENCE_IBAN;
|
||||
bics += o->type == BANK_OCCURRENCE_BIC;
|
||||
}
|
||||
if (holders > 1U || (bics > 0U && ibans > 1U)) {
|
||||
block->ambiguous = TRUE;
|
||||
g_ptr_array_add(block->reasons, g_strdup("multiple-possible-associations"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static guint block_count_type(const BankAnalysisResult *r,
|
||||
const BankProposedBlock *block, BankOccurrenceType type)
|
||||
{
|
||||
guint count = 0;
|
||||
for (guint i = 0; i < block->occurrence_identifiers->len; i++) {
|
||||
BankOccurrence *o = bank_analysis_find_occurrence(r,
|
||||
g_ptr_array_index(block->occurrence_identifiers, i));
|
||||
count += o->type == type;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
static void build_transaction_proposal(BankAnalysisResult *r)
|
||||
{
|
||||
for (guint block_index = 0; block_index < r->blocks->len; block_index++) {
|
||||
BankProposedBlock *block = g_ptr_array_index(r->blocks, block_index);
|
||||
gsize paragraph_start = block->start_offset;
|
||||
while (paragraph_start > 0 && !(paragraph_start > 1U &&
|
||||
r->source_text[paragraph_start - 1U] == '\n' &&
|
||||
r->source_text[paragraph_start - 2U] == '\n')) paragraph_start--;
|
||||
char *prefix = g_strndup(r->source_text + paragraph_start,
|
||||
block->start_offset - paragraph_start);
|
||||
gboolean marker = g_regex_match_simple(
|
||||
"(?i)\\b(transaction|virement|transfer)\\b", prefix, 0, 0);
|
||||
g_free(prefix);
|
||||
guint debtors = block_count_type(r, block, BANK_OCCURRENCE_DEBTOR_ACCOUNT);
|
||||
guint creditors = block_count_type(r, block, BANK_OCCURRENCE_CREDITOR_ACCOUNT);
|
||||
guint amounts = block_count_type(r, block, BANK_OCCURRENCE_AMOUNT);
|
||||
if (!marker || debtors == 0 || creditors == 0 || amounts == 0) continue;
|
||||
if (r->transaction_proposals->len >= BANK_ANALYSIS_MAX_TRANSACTIONS) {
|
||||
r->transaction_limit_reached = TRUE; break;
|
||||
}
|
||||
BankTransactionProposal *p = g_new0(BankTransactionProposal, 1);
|
||||
p->identifier = g_strdup_printf("transaction-proposal-%04u",
|
||||
r->transaction_proposals->len + 1U);
|
||||
p->source_occurrence_identifiers = g_ptr_array_new_with_free_func(g_free);
|
||||
p->missing_fields = g_ptr_array_new_with_free_func(g_free);
|
||||
p->ambiguities = g_ptr_array_new_with_free_func(g_free);
|
||||
p->reasons = g_ptr_array_new_with_free_func(g_free);
|
||||
g_ptr_array_add(p->reasons, g_strdup("explicit-transaction-marker"));
|
||||
g_ptr_array_add(p->reasons, g_strdup("explicit-debit-credit-markers"));
|
||||
p->decision = BANK_DECISION_PROPOSED;
|
||||
for (guint i = 0; i < block->occurrence_identifiers->len; i++)
|
||||
g_ptr_array_add(p->source_occurrence_identifiers,
|
||||
g_strdup(g_ptr_array_index(block->occurrence_identifiers, i)));
|
||||
if (debtors > 1 || creditors > 1 || amounts > 1)
|
||||
g_ptr_array_add(p->ambiguities, g_strdup("multiple-explicit-candidates"));
|
||||
if (block_count_type(r, block, BANK_OCCURRENCE_OBSERVED_DATE) == 0)
|
||||
g_ptr_array_add(p->missing_fields, g_strdup("observed_date"));
|
||||
g_ptr_array_add(r->transaction_proposals, p);
|
||||
}
|
||||
}
|
||||
|
||||
BankAnalysisResult *bank_structured_extractor_analyze(const char *text, GError **error)
|
||||
{
|
||||
BankAnalysisResult *r = g_new0(BankAnalysisResult, 1);
|
||||
r->source_text = g_strdup(text == NULL ? "" : text);
|
||||
r->occurrences = g_ptr_array_new_with_free_func((GDestroyNotify)bank_occurrence_free);
|
||||
r->blocks = g_ptr_array_new_with_free_func((GDestroyNotify)bank_proposed_block_free);
|
||||
r->transaction_proposals = g_ptr_array_new_with_free_func((GDestroyNotify)bank_transaction_proposal_free);
|
||||
r->warnings = g_ptr_array_new_with_free_func(g_free);
|
||||
if (text == NULL || text[0] == '\0') return r;
|
||||
if (strlen(text) > FINANCIAL_TEXT_MAX_BYTES) {
|
||||
g_set_error_literal(error, analysis_error(), 1, "Le texte dépasse la limite autorisée.");
|
||||
bank_analysis_result_free(r); return NULL;
|
||||
}
|
||||
GPtrArray *ibans = bank_text_extractor_extract(text, error);
|
||||
if (error != NULL && *error != NULL) { g_ptr_array_unref(ibans); bank_analysis_result_free(r); return NULL; }
|
||||
if (ibans->len >= FINANCIAL_CANDIDATE_MAX_RESULTS)
|
||||
r->occurrence_limit_reached = TRUE;
|
||||
for (guint i = 0; i < ibans->len; i++) {
|
||||
BankCandidate *c = g_ptr_array_index(ibans, i);
|
||||
BankOccurrence *o = occurrence_new(text, BANK_OCCURRENCE_IBAN,
|
||||
c->start_offset, c->end_offset, c->normalized_iban, c->validation,
|
||||
"canonical-iban", error);
|
||||
if (o == NULL) { g_ptr_array_unref(ibans); bank_analysis_result_free(r); return NULL; }
|
||||
for (guint w = 0; w < c->warnings->len; w++)
|
||||
g_ptr_array_add(o->warnings, g_strdup(g_ptr_array_index(c->warnings, w)));
|
||||
add_occurrence(r, o);
|
||||
}
|
||||
g_ptr_array_unref(ibans);
|
||||
if (!scan_regex(r, "(?im)^(?:BIC|SWIFT)[ \\t]*:[ \\t]*([^\\r\\n]{1,32})", BANK_OCCURRENCE_BIC, 1,
|
||||
BANK_VALIDATION_UNVERIFIABLE, "explicit-bic", TRUE, error) ||
|
||||
!scan_regex(r, "(?im)^(?:Titulaire|Nom du titulaire|Account holder)[ \\t]*:[ \\t]*([^\\r\\n]{1,512})", BANK_OCCURRENCE_DECLARED_HOLDER, 1,
|
||||
BANK_VALIDATION_UNVERIFIABLE, "explicit-holder", FALSE, error) ||
|
||||
!scan_regex(r, "(?im)^(?:Banque|Établissement)[ \\t]*:[ \\t]*([^\\r\\n]{1,512})", BANK_OCCURRENCE_DECLARED_INSTITUTION, 1,
|
||||
BANK_VALIDATION_UNVERIFIABLE, "explicit-institution", FALSE, error) ||
|
||||
!scan_regex(r, "(?im)^Adresse de la banque[ \\t]*:[ \\t]*([^\\r\\n]{1,512})", BANK_OCCURRENCE_DECLARED_ADDRESS, 1,
|
||||
BANK_VALIDATION_UNVERIFIABLE, "explicit-bank-address", FALSE, error) ||
|
||||
!scan_regex(r, "(?im)^Payeur[ \\t]*:[ \\t]*([^\\r\\n]{1,512})", BANK_OCCURRENCE_DECLARED_PAYER, 1,
|
||||
BANK_VALIDATION_UNVERIFIABLE, "explicit-payer", FALSE, error) ||
|
||||
!scan_regex(r, "(?im)^Bénéficiaire[ \\t]*:[ \\t]*([^\\r\\n]{1,512})", BANK_OCCURRENCE_DECLARED_BENEFICIARY, 1,
|
||||
BANK_VALIDATION_UNVERIFIABLE, "explicit-beneficiary", FALSE, error) ||
|
||||
!scan_regex(r, "(?im)^(?:Compte débité|Débiteur)[ \\t]*:[ \\t]*([^\\r\\n]{1,512})", BANK_OCCURRENCE_DEBTOR_ACCOUNT, 1,
|
||||
BANK_VALIDATION_UNVERIFIABLE, "explicit-debtor", FALSE, error) ||
|
||||
!scan_regex(r, "(?im)^(?:Compte crédité|Créditeur)[ \\t]*:[ \\t]*([^\\r\\n]{1,512})", BANK_OCCURRENCE_CREDITOR_ACCOUNT, 1,
|
||||
BANK_VALIDATION_UNVERIFIABLE, "explicit-creditor", FALSE, error) ||
|
||||
!scan_regex(r, "(?im)^Date[ \\t]*:[ \\t]*([^\\r\\n]{1,64})", BANK_OCCURRENCE_OBSERVED_DATE, 1,
|
||||
BANK_VALIDATION_UNVERIFIABLE, "explicit-date", FALSE, error) ||
|
||||
!scan_regex(r, "(?im)^Heure[ \\t]*:[ \\t]*([^\\r\\n]{1,32})", BANK_OCCURRENCE_OBSERVED_TIME, 1,
|
||||
BANK_VALIDATION_UNVERIFIABLE, "explicit-time", FALSE, error) ||
|
||||
!scan_regex(r, "(?im)^Statut[ \\t]*:[ \\t]*([^\\r\\n]{1,128})", BANK_OCCURRENCE_OBSERVED_STATUS, 1,
|
||||
BANK_VALIDATION_UNVERIFIABLE, "explicit-observed-status", FALSE, error) ||
|
||||
!scan_regex(r, "(?im)^(?:Référence|Référence de transaction)[ \\t]*:[ \\t]*([^\\r\\n]{1,256})", BANK_OCCURRENCE_BANK_REFERENCE, 1,
|
||||
BANK_VALIDATION_UNVERIFIABLE, "explicit-reference", FALSE, error) ||
|
||||
!scan_regex(r, "(?im)^UETR[ \\t]*:[ \\t]*([^\\r\\n]{1,64})", BANK_OCCURRENCE_UETR, 1,
|
||||
BANK_VALIDATION_UNVERIFIABLE, "explicit-uetr", TRUE, error) || !scan_amounts(r, error)) {
|
||||
bank_analysis_result_free(r); return NULL;
|
||||
}
|
||||
for (guint i = 0; i < r->occurrences->len; i++) {
|
||||
BankOccurrence *o = g_ptr_array_index(r->occurrences, i);
|
||||
if (o->type == BANK_OCCURRENCE_BIC) {
|
||||
BicValidatorResult check = bic_validator_check(o->normalized_value);
|
||||
o->validation = check == BIC_VALIDATOR_VALID ? BANK_VALIDATION_VALID : BANK_VALIDATION_INVALID;
|
||||
if (check != BIC_VALIDATOR_VALID) g_clear_pointer(&o->normalized_value, g_free);
|
||||
} else if (o->type == BANK_OCCURRENCE_UETR) {
|
||||
GRegex *uuid = g_regex_new("^[0-9A-F]{8}-[0-9A-F]{4}-[1-5][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$", 0, 0, NULL);
|
||||
o->validation = g_regex_match(uuid, o->normalized_value, 0, NULL) ? BANK_VALIDATION_VALID : BANK_VALIDATION_INVALID;
|
||||
if (o->validation != BANK_VALIDATION_VALID) g_clear_pointer(&o->normalized_value, g_free);
|
||||
g_regex_unref(uuid);
|
||||
}
|
||||
}
|
||||
g_ptr_array_sort(r->occurrences, occurrence_compare);
|
||||
for (guint i = 0; i < r->occurrences->len; i++) {
|
||||
BankOccurrence *o = g_ptr_array_index(r->occurrences, i);
|
||||
o->identifier = g_strdup_printf("occ-%04u-%s", i + 1U, bank_occurrence_type_to_code(o->type));
|
||||
}
|
||||
build_blocks(r); build_transaction_proposal(r); return r;
|
||||
}
|
||||
70
src/models/bank_analysis.c
Normal file
70
src/models/bank_analysis.c
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
#include "models/bank_analysis.h"
|
||||
|
||||
const char *bank_occurrence_type_to_code(BankOccurrenceType type)
|
||||
{
|
||||
static const char *codes[] = {"iban","bic","declared_holder",
|
||||
"declared_institution","declared_address","amount","currency",
|
||||
"observed_date","observed_time","observed_status","bank_reference",
|
||||
"uetr","declared_payer","declared_beneficiary","debtor_account",
|
||||
"creditor_account"};
|
||||
if ((gint)type < 0 || type >= BANK_OCCURRENCE_COUNT)
|
||||
return "unknown";
|
||||
return codes[type];
|
||||
}
|
||||
|
||||
void bank_occurrence_free(BankOccurrence *o)
|
||||
{
|
||||
if (o == NULL) return;
|
||||
g_free(o->identifier); g_free(o->raw_value); g_free(o->normalized_value);
|
||||
g_free(o->extraction_rule); g_clear_pointer(&o->warnings, g_ptr_array_unref);
|
||||
g_free(o->source_context); g_free(o);
|
||||
}
|
||||
|
||||
void bank_proposed_block_free(BankProposedBlock *b)
|
||||
{
|
||||
if (b == NULL) return;
|
||||
g_free(b->identifier);
|
||||
g_clear_pointer(&b->occurrence_identifiers, g_ptr_array_unref);
|
||||
g_clear_pointer(&b->reasons, g_ptr_array_unref); g_free(b);
|
||||
}
|
||||
|
||||
void bank_transaction_proposal_free(BankTransactionProposal *p)
|
||||
{
|
||||
if (p == NULL) return;
|
||||
g_free(p->identifier);
|
||||
g_clear_pointer(&p->source_occurrence_identifiers, g_ptr_array_unref);
|
||||
g_clear_pointer(&p->missing_fields, g_ptr_array_unref);
|
||||
g_clear_pointer(&p->ambiguities, g_ptr_array_unref);
|
||||
g_clear_pointer(&p->reasons, g_ptr_array_unref); g_free(p);
|
||||
}
|
||||
|
||||
void bank_analysis_result_free(BankAnalysisResult *r)
|
||||
{
|
||||
if (r == NULL) return;
|
||||
g_free(r->source_text); g_clear_pointer(&r->occurrences, g_ptr_array_unref);
|
||||
g_clear_pointer(&r->blocks, g_ptr_array_unref);
|
||||
g_clear_pointer(&r->transaction_proposals, g_ptr_array_unref);
|
||||
g_clear_pointer(&r->warnings, g_ptr_array_unref); g_free(r);
|
||||
}
|
||||
|
||||
BankOccurrence *bank_analysis_find_occurrence(const BankAnalysisResult *r,
|
||||
const char *id)
|
||||
{
|
||||
if (r == NULL || id == NULL) return NULL;
|
||||
for (guint i = 0; i < r->occurrences->len; i++) {
|
||||
BankOccurrence *o = g_ptr_array_index(r->occurrences, i);
|
||||
if (g_strcmp0(o->identifier, id) == 0) return o;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
BankProposedBlock *bank_analysis_find_block(const BankAnalysisResult *r,
|
||||
const char *id)
|
||||
{
|
||||
if (r == NULL || id == NULL) return NULL;
|
||||
for (guint i = 0; i < r->blocks->len; i++) {
|
||||
BankProposedBlock *b = g_ptr_array_index(r->blocks, i);
|
||||
if (g_strcmp0(b->identifier, id) == 0) return b;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
417
tests/test_bank_structured_extractor.c
Normal file
417
tests/test_bank_structured_extractor.c
Normal file
|
|
@ -0,0 +1,417 @@
|
|||
#include "core/bank_analysis_revision.h"
|
||||
#include "core/bank_structured_extractor.h"
|
||||
#include <glib.h>
|
||||
#include <string.h>
|
||||
|
||||
static const char *fr = "FR4830002005500000000000052";
|
||||
static const char *de = "DE89370400440532013000";
|
||||
static const char *gb = "GB82WEST12345698765432";
|
||||
|
||||
static guint count_type(const BankAnalysisResult *result, BankOccurrenceType type)
|
||||
{
|
||||
guint count = 0;
|
||||
for (guint i = 0; i < result->occurrences->len; i++)
|
||||
if (((BankOccurrence *)g_ptr_array_index(result->occurrences, i))->type == type) count++;
|
||||
return count;
|
||||
}
|
||||
|
||||
static BankOccurrence *first_type(const BankAnalysisResult *result, BankOccurrenceType type)
|
||||
{
|
||||
for (guint i = 0; i < result->occurrences->len; i++) {
|
||||
BankOccurrence *o = g_ptr_array_index(result->occurrences, i);
|
||||
if (o->type == type) return o;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void assert_offsets(const BankAnalysisResult *result)
|
||||
{
|
||||
for (guint i = 0; i < result->occurrences->len; i++) {
|
||||
BankOccurrence *o = g_ptr_array_index(result->occurrences, i);
|
||||
g_assert_cmpuint(o->start_offset, <=, o->end_offset);
|
||||
g_assert_cmpmem(result->source_text + o->start_offset,
|
||||
o->end_offset - o->start_offset, o->raw_value, strlen(o->raw_value));
|
||||
if (i > 0) {
|
||||
BankOccurrence *previous = g_ptr_array_index(result->occurrences, i - 1U);
|
||||
g_assert_true(previous->start_offset < o->start_offset ||
|
||||
(previous->start_offset == o->start_offset && previous->end_offset <= o->end_offset));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void test_three_accounts_and_holders(void)
|
||||
{
|
||||
char *text = g_strdup_printf(
|
||||
"Titulaire : SPECIMEN ALPHA\nBanque : SPECIMEN BANK\nBIC : BNPAFRPP\n"
|
||||
"IBAN : FR48 3000 2005 5000 0000 0000 052\nIBAN : %s\n\n"
|
||||
"Titulaire : SPECIMEN BETA\nBIC : DEUTDEFFXXX\nIBAN : %s", de, gb);
|
||||
BankAnalysisResult *r = bank_structured_extractor_analyze(text, NULL);
|
||||
g_assert_nonnull(r); g_assert_cmpuint(count_type(r, BANK_OCCURRENCE_IBAN), ==, 3);
|
||||
g_assert_cmpuint(count_type(r, BANK_OCCURRENCE_BIC), ==, 2);
|
||||
g_assert_cmpuint(count_type(r, BANK_OCCURRENCE_DECLARED_HOLDER), ==, 2);
|
||||
g_assert_cmpuint(count_type(r, BANK_OCCURRENCE_DECLARED_INSTITUTION), ==, 1);
|
||||
g_assert_cmpuint(r->blocks->len, ==, 2); g_assert_cmpuint(r->transaction_proposals->len, ==, 0);
|
||||
assert_offsets(r); bank_analysis_result_free(r); g_free(text);
|
||||
}
|
||||
|
||||
static void test_orphans_ambiguities_duplicates(void)
|
||||
{
|
||||
char *text = g_strdup_printf("BIC : BNPAFRPP\n\nIBAN : FR00 3000 2005 5000 0000 0000 021\n\n"
|
||||
"IBAN : FRO8 3000 2005 5000 0000 0000 052\n\nIBAN : AD1200000000000000000000\n\n%s puis %s", fr, fr);
|
||||
BankAnalysisResult *r = bank_structured_extractor_analyze(text, NULL);
|
||||
g_assert_nonnull(r); g_assert_cmpuint(count_type(r, BANK_OCCURRENCE_BIC), ==, 1);
|
||||
g_assert_cmpuint(count_type(r, BANK_OCCURRENCE_IBAN), >=, 5);
|
||||
guint ambiguous = 0, unverifiable = 0;
|
||||
for (guint i = 0; i < r->occurrences->len; i++) {
|
||||
BankOccurrence *o = g_ptr_array_index(r->occurrences, i);
|
||||
ambiguous += o->validation == BANK_VALIDATION_AMBIGUOUS;
|
||||
unverifiable += o->validation == BANK_VALIDATION_UNVERIFIABLE;
|
||||
}
|
||||
g_assert_cmpuint(ambiguous, >=, 1); g_assert_cmpuint(unverifiable, >=, 1);
|
||||
assert_offsets(r); bank_analysis_result_free(r); g_free(text);
|
||||
}
|
||||
|
||||
static void test_utf8_punctuation_crlf(void)
|
||||
{
|
||||
char *text = g_strdup_printf("Préfixe SPECIMEN : [%s],\r\nBIC : BNPAFRPP", fr);
|
||||
BankAnalysisResult *r = bank_structured_extractor_analyze(text, NULL);
|
||||
BankOccurrence *iban = first_type(r, BANK_OCCURRENCE_IBAN);
|
||||
g_assert_nonnull(iban); g_assert_cmpuint(iban->start_offset, ==,
|
||||
(gsize)(strstr(text, fr) - text)); assert_offsets(r);
|
||||
bank_analysis_result_free(r); g_free(text);
|
||||
}
|
||||
|
||||
static void test_amounts_and_currency(void)
|
||||
{
|
||||
const char *text = "Montant : 1234,56 EUR\nMontant : 1 234,56 €\n"
|
||||
"Amount : EUR 1234.56\nMontant : 99 USD\nMontant : 1,234\nMontant : 12.123456789";
|
||||
BankAnalysisResult *r = bank_structured_extractor_analyze(text, NULL);
|
||||
g_assert_cmpuint(count_type(r, BANK_OCCURRENCE_AMOUNT), ==, 6);
|
||||
g_assert_cmpuint(count_type(r, BANK_OCCURRENCE_CURRENCY), ==, 4);
|
||||
guint ambiguous = 0, invalid = 0;
|
||||
for (guint i = 0; i < r->occurrences->len; i++) {
|
||||
BankOccurrence *o = g_ptr_array_index(r->occurrences, i);
|
||||
if (o->type == BANK_OCCURRENCE_AMOUNT) {
|
||||
ambiguous += o->validation == BANK_VALIDATION_AMBIGUOUS;
|
||||
invalid += o->validation == BANK_VALIDATION_INVALID;
|
||||
}
|
||||
}
|
||||
g_assert_cmpuint(ambiguous, ==, 1); g_assert_cmpuint(invalid, ==, 1);
|
||||
bank_analysis_result_free(r);
|
||||
}
|
||||
|
||||
static void test_explicit_transaction(void)
|
||||
{
|
||||
char *text = g_strdup_printf("Virement SPECIMEN\nCompte débité : %s\nCompte crédité : %s\n"
|
||||
"Montant : 1234,56 EUR\nPayeur : SPECIMEN ALPHA\nBénéficiaire : SPECIMEN BETA\n"
|
||||
"Date : 2026-02-03\nHeure : 14:05\nStatut : exécuté\n"
|
||||
"Référence de transaction : SPECIMEN-REF-42\n"
|
||||
"UETR : 11111111-2222-4333-8444-555555555555", fr, de);
|
||||
BankAnalysisResult *r = bank_structured_extractor_analyze(text, NULL);
|
||||
g_assert_cmpuint(r->transaction_proposals->len, ==, 1);
|
||||
BankTransactionProposal *p = g_ptr_array_index(r->transaction_proposals, 0);
|
||||
g_assert_cmpint(p->decision, ==, BANK_DECISION_PROPOSED);
|
||||
g_assert_false(p->create_transaction); g_assert_false(p->create_account);
|
||||
g_assert_false(p->create_person); g_assert_false(p->create_relation);
|
||||
BankOccurrence *status = first_type(r, BANK_OCCURRENCE_OBSERVED_STATUS);
|
||||
g_assert_cmpstr(status->raw_value, ==, "exécuté");
|
||||
g_assert_cmpint(first_type(r, BANK_OCCURRENCE_UETR)->validation, ==, BANK_VALIDATION_VALID);
|
||||
bank_analysis_result_free(r); g_free(text);
|
||||
}
|
||||
|
||||
static void test_insufficient_transaction(void)
|
||||
{
|
||||
char *text = g_strdup_printf("%s\n%s\nMontant : 10 EUR\nStatut : pending\n"
|
||||
"Référence : SPECIMEN-ONLY\nUETR : invalid", fr, de);
|
||||
BankAnalysisResult *r = bank_structured_extractor_analyze(text, NULL);
|
||||
g_assert_cmpuint(r->transaction_proposals->len, ==, 0);
|
||||
g_assert_cmpint(first_type(r, BANK_OCCURRENCE_UETR)->validation, ==, BANK_VALIDATION_INVALID);
|
||||
bank_analysis_result_free(r); g_free(text);
|
||||
}
|
||||
|
||||
static void test_memory_revision(void)
|
||||
{
|
||||
char *text = g_strdup_printf("Titulaire : SPECIMEN\nIBAN : %s\nIBAN : %s", fr, de);
|
||||
BankAnalysisResult *r = bank_structured_extractor_analyze(text, NULL);
|
||||
g_assert_cmpuint(r->blocks->len, ==, 1); BankProposedBlock *first = g_ptr_array_index(r->blocks, 0);
|
||||
char *first_id = g_strdup(first->identifier);
|
||||
char *occurrence = g_strdup(g_ptr_array_index(first->occurrence_identifiers, 0));
|
||||
g_assert_true(bank_analysis_revision_split(r, first_id, "manual-a", occurrence, NULL));
|
||||
g_assert_cmpuint(r->blocks->len, ==, 2);
|
||||
g_assert_true(bank_analysis_revision_move(r, "manual-a", first_id, occurrence, NULL));
|
||||
g_assert_true(bank_analysis_revision_create_block(r, "manual-empty", NULL, NULL));
|
||||
g_assert_true(bank_analysis_revision_mark_ambiguous(r, first_id, TRUE, NULL));
|
||||
g_assert_true(bank_analysis_revision_reject(r, first_id, NULL));
|
||||
g_assert_cmpint(bank_analysis_find_block(r, first_id)->decision, ==, BANK_DECISION_REJECTED);
|
||||
g_assert_nonnull(bank_analysis_find_occurrence(r, occurrence));
|
||||
guint before = bank_analysis_find_block(r, first_id)->occurrence_identifiers->len;
|
||||
g_assert_false(bank_analysis_revision_move(r, first_id, "absent", occurrence, NULL));
|
||||
g_assert_cmpuint(bank_analysis_find_block(r, first_id)->occurrence_identifiers->len, ==, before);
|
||||
g_assert_true(bank_analysis_revision_merge(r, first_id, "manual-empty", NULL));
|
||||
g_free(occurrence); g_free(first_id); bank_analysis_result_free(r); g_free(text);
|
||||
}
|
||||
|
||||
static void test_determinism_and_limits(void)
|
||||
{
|
||||
char *text = g_strdup_printf("IBAN : %s\nBIC : BNPAFRPP", fr);
|
||||
BankAnalysisResult *a = bank_structured_extractor_analyze(text, NULL);
|
||||
BankAnalysisResult *b = bank_structured_extractor_analyze(text, NULL);
|
||||
g_assert_cmpuint(a->occurrences->len, ==, b->occurrences->len);
|
||||
for (guint i = 0; i < a->occurrences->len; i++)
|
||||
g_assert_cmpstr(((BankOccurrence *)g_ptr_array_index(a->occurrences, i))->identifier,
|
||||
==, ((BankOccurrence *)g_ptr_array_index(b->occurrences, i))->identifier);
|
||||
bank_analysis_result_free(a); bank_analysis_result_free(b); g_free(text);
|
||||
a = bank_structured_extractor_analyze(NULL, NULL); g_assert_cmpuint(a->occurrences->len, ==, 0);
|
||||
bank_analysis_result_free(a);
|
||||
char *large = g_malloc0(FINANCIAL_TEXT_MAX_BYTES + 2U);
|
||||
memset(large, 'A', FINANCIAL_TEXT_MAX_BYTES + 1U); GError *error = NULL;
|
||||
g_assert_null(bank_structured_extractor_analyze(large, &error)); g_assert_nonnull(error);
|
||||
g_clear_error(&error); g_free(large);
|
||||
GString *many = g_string_new(NULL);
|
||||
for (guint i = 0; i < BANK_ANALYSIS_MAX_OCCURRENCES + 1U; i++)
|
||||
g_string_append_printf(many, "%s,", fr);
|
||||
a = bank_structured_extractor_analyze(many->str, NULL);
|
||||
g_assert_nonnull(a); g_assert_true(a->occurrence_limit_reached);
|
||||
g_assert_cmpuint(a->occurrences->len, ==, BANK_ANALYSIS_MAX_OCCURRENCES);
|
||||
bank_analysis_result_free(a); g_string_free(many, TRUE);
|
||||
GString *sections = g_string_new(NULL);
|
||||
for (guint i = 0; i < BANK_ANALYSIS_MAX_BLOCKS + 1U; i++)
|
||||
g_string_append_printf(sections, "%s\n\n", fr);
|
||||
a = bank_structured_extractor_analyze(sections->str, NULL);
|
||||
g_assert_nonnull(a); g_assert_true(a->block_limit_reached);
|
||||
g_assert_cmpuint(a->blocks->len, ==, BANK_ANALYSIS_MAX_BLOCKS);
|
||||
bank_analysis_result_free(a); g_string_free(sections, TRUE);
|
||||
GString *long_holder = g_string_new("Titulaire : ");
|
||||
for (guint i = 0; i < BANK_ANALYSIS_MAX_VALUE_BYTES + 1U; i++)
|
||||
g_string_append_c(long_holder, 'A');
|
||||
error = NULL; g_assert_null(bank_structured_extractor_analyze(long_holder->str, &error));
|
||||
g_assert_nonnull(error); g_clear_error(&error); g_string_free(long_holder, TRUE);
|
||||
}
|
||||
|
||||
static void assert_amount(const BankAnalysisResult *result, const char *raw,
|
||||
BankValidationState state, const char *normalized)
|
||||
{
|
||||
for (guint i = 0; i < result->occurrences->len; i++) {
|
||||
BankOccurrence *o = g_ptr_array_index(result->occurrences, i);
|
||||
if (o->type == BANK_OCCURRENCE_AMOUNT && g_strcmp0(o->raw_value, raw) == 0) {
|
||||
g_assert_cmpint(o->validation, ==, state);
|
||||
g_assert_cmpstr(o->normalized_value, ==, normalized);
|
||||
g_assert_cmpmem(result->source_text + o->start_offset,
|
||||
o->end_offset - o->start_offset, raw, strlen(raw));
|
||||
return;
|
||||
}
|
||||
}
|
||||
g_assert_not_reached();
|
||||
}
|
||||
|
||||
static void test_amount_complete_ambiguities(void)
|
||||
{
|
||||
const char *text = "Montant : 1,234\nMontant : 1.234\nMontant : 1,234,567\n"
|
||||
"Montant : 1.234.567\nMontant : 1,234.56 USD\nMontant : 1.234,56 EUR\n"
|
||||
"Montant : 12,34,56\nMontant : 1..234\nMontant : 1,,234\n"
|
||||
"Montant : 1234,567,89 EUR\nMontant : (1234,56 EUR),";
|
||||
BankAnalysisResult *r = bank_structured_extractor_analyze(text, NULL);
|
||||
g_assert_nonnull(r);
|
||||
assert_amount(r, "1,234", BANK_VALIDATION_AMBIGUOUS, NULL);
|
||||
assert_amount(r, "1.234", BANK_VALIDATION_AMBIGUOUS, NULL);
|
||||
assert_amount(r, "1,234,567", BANK_VALIDATION_VALID, "1234567");
|
||||
assert_amount(r, "1.234.567", BANK_VALIDATION_VALID, "1234567");
|
||||
assert_amount(r, "1,234.56", BANK_VALIDATION_VALID, "1234.56");
|
||||
assert_amount(r, "1.234,56", BANK_VALIDATION_VALID, "1234.56");
|
||||
assert_amount(r, "12,34,56", BANK_VALIDATION_INVALID, NULL);
|
||||
assert_amount(r, "1..234", BANK_VALIDATION_INVALID, NULL);
|
||||
assert_amount(r, "1,,234", BANK_VALIDATION_INVALID, NULL);
|
||||
assert_amount(r, "1234,567,89", BANK_VALIDATION_INVALID, NULL);
|
||||
assert_amount(r, "1234,56", BANK_VALIDATION_VALID, "1234.56");
|
||||
bank_analysis_result_free(r);
|
||||
}
|
||||
|
||||
static void test_amount_bounds_before_copy(void)
|
||||
{
|
||||
GString *text = g_string_new("Montant : ");
|
||||
for (guint i = 0; i < BANK_ANALYSIS_MAX_AMOUNT_BYTES + 1U; i++)
|
||||
g_string_append_c(text, '1');
|
||||
BankAnalysisResult *r = bank_structured_extractor_analyze(text->str, NULL);
|
||||
g_assert_nonnull(r); g_assert_true(r->amount_limit_reached);
|
||||
g_assert_cmpuint(count_type(r, BANK_OCCURRENCE_AMOUNT), ==, 0);
|
||||
g_assert_cmpuint(r->warnings->len, >, 0);
|
||||
bank_analysis_result_free(r); g_string_free(text, TRUE);
|
||||
|
||||
text = g_string_new("Montant : ");
|
||||
for (guint i = 0; i < BANK_ANALYSIS_MAX_AMOUNT_DIGITS; i++)
|
||||
g_string_append_c(text, '1');
|
||||
r = bank_structured_extractor_analyze(text->str, NULL);
|
||||
g_assert_nonnull(r); g_assert_false(r->amount_limit_reached);
|
||||
g_assert_cmpuint(count_type(r, BANK_OCCURRENCE_AMOUNT), ==, 1);
|
||||
bank_analysis_result_free(r); g_string_free(text, TRUE);
|
||||
|
||||
text = g_string_new("Montant : 1");
|
||||
for (guint i = 0; i < BANK_ANALYSIS_MAX_AMOUNT_GROUPS + 1U; i++)
|
||||
g_string_append(text, " 234");
|
||||
r = bank_structured_extractor_analyze(text->str, NULL);
|
||||
g_assert_nonnull(r); g_assert_true(r->amount_limit_reached);
|
||||
g_assert_cmpuint(count_type(r, BANK_OCCURRENCE_AMOUNT), ==, 0);
|
||||
bank_analysis_result_free(r); g_string_free(text, TRUE);
|
||||
|
||||
r = bank_structured_extractor_analyze("Montant : 12.12345678 EUR\n"
|
||||
"Montant : 12.123456789 EUR", NULL);
|
||||
assert_amount(r, "12.12345678", BANK_VALIDATION_VALID, "12.12345678");
|
||||
assert_amount(r, "12.123456789", BANK_VALIDATION_INVALID, NULL);
|
||||
bank_analysis_result_free(r);
|
||||
}
|
||||
|
||||
static void test_amount_strict_groups(void)
|
||||
{
|
||||
const char *text = "Montant: 12,34.56 USD\nMontant: 1.2,34 EUR\n"
|
||||
"Montant: 1,23.456 EUR\nMontant: 1.234,5,67 EUR\n"
|
||||
"Montant: 1,234.5.67 USD\nMontant: 10 20 EUR\n"
|
||||
"Montant: 1 23 456 EUR\nMontant: 1 234 56 EUR\n"
|
||||
"Montant: 1 234 EUR\nMontant: 1234,56 EUR\n"
|
||||
"Montant: 1234.56 USD\nMontant: 1 234,56 EUR\n"
|
||||
"Montant: 1 234.56 USD\nMontant: 1.234.567,89 EUR\n"
|
||||
"Montant: 1,234,567.89 USD";
|
||||
BankAnalysisResult *r = bank_structured_extractor_analyze(text, NULL);
|
||||
const char *invalid[] = {"12,34.56", "1.2,34", "1,23.456",
|
||||
"1.234,5,67", "1,234.5.67", "10 20", "1 23 456",
|
||||
"1 234 56", "1 234"};
|
||||
for (guint i = 0; i < G_N_ELEMENTS(invalid); i++)
|
||||
assert_amount(r, invalid[i], BANK_VALIDATION_INVALID, NULL);
|
||||
assert_amount(r, "1234,56", BANK_VALIDATION_VALID, "1234.56");
|
||||
assert_amount(r, "1234.56", BANK_VALIDATION_VALID, "1234.56");
|
||||
assert_amount(r, "1 234,56", BANK_VALIDATION_VALID, "1234.56");
|
||||
assert_amount(r, "1 234.56", BANK_VALIDATION_VALID, "1234.56");
|
||||
assert_amount(r, "1.234.567,89", BANK_VALIDATION_VALID, "1234567.89");
|
||||
assert_amount(r, "1,234,567.89", BANK_VALIDATION_VALID, "1234567.89");
|
||||
assert_offsets(r); bank_analysis_result_free(r);
|
||||
}
|
||||
|
||||
static void test_amount_terminal_punctuation(void)
|
||||
{
|
||||
const char *text = "Montant: 1234 EUR,\nMontant: 1234 EUR,,\n"
|
||||
"Montant: 1234 EUR...\nMontant: (1234 EUR)\nMontant: [1234 EUR]";
|
||||
BankAnalysisResult *r = bank_structured_extractor_analyze(text, NULL);
|
||||
g_assert_cmpuint(count_type(r, BANK_OCCURRENCE_AMOUNT), ==, 5);
|
||||
for (guint i = 0; i < r->occurrences->len; i++) {
|
||||
BankOccurrence *o = g_ptr_array_index(r->occurrences, i);
|
||||
if (o->type == BANK_OCCURRENCE_AMOUNT) {
|
||||
g_assert_cmpstr(o->raw_value, ==, "1234");
|
||||
g_assert_cmpint(o->validation, ==, BANK_VALIDATION_VALID);
|
||||
}
|
||||
}
|
||||
assert_offsets(r); bank_analysis_result_free(r);
|
||||
}
|
||||
|
||||
static void test_amount_context_before_limits(void)
|
||||
{
|
||||
GString *text = g_string_new("SPECIMEN ");
|
||||
for (guint i = 0; i < BANK_ANALYSIS_MAX_AMOUNT_BYTES + 64U; i++)
|
||||
g_string_append_c(text, '7');
|
||||
g_string_append(text, "\nRéférence: ");
|
||||
for (guint i = 0; i < BANK_ANALYSIS_MAX_AMOUNT_BYTES + 64U; i++)
|
||||
g_string_append_c(text, '8');
|
||||
g_string_append(text, "\nIdentifiant: 123456789012345678901234567890"
|
||||
"\nTéléphone SPECIMEN: 01 02 03 04 05");
|
||||
g_string_append(text, "\nSPECIMEN ");
|
||||
for (guint i = 0; i < BANK_ANALYSIS_MAX_AMOUNT_BYTES + 64U; i++)
|
||||
g_string_append_c(text, '5');
|
||||
g_string_append(text, " EURREFERENCE");
|
||||
BankAnalysisResult *r = bank_structured_extractor_analyze(text->str, NULL);
|
||||
g_assert_cmpuint(count_type(r, BANK_OCCURRENCE_AMOUNT), ==, 0);
|
||||
g_assert_false(r->amount_limit_reached); g_assert_cmpuint(r->warnings->len, ==, 0);
|
||||
bank_analysis_result_free(r); g_string_free(text, TRUE);
|
||||
|
||||
text = g_string_new("Montant: ");
|
||||
for (guint i = 0; i < BANK_ANALYSIS_MAX_AMOUNT_BYTES + 1U; i++)
|
||||
g_string_append_c(text, '9');
|
||||
g_string_append(text, "\nSPECIMEN ");
|
||||
for (guint i = 0; i < BANK_ANALYSIS_MAX_AMOUNT_BYTES + 1U; i++)
|
||||
g_string_append_c(text, '6');
|
||||
g_string_append(text, " EUR");
|
||||
r = bank_structured_extractor_analyze(text->str, NULL);
|
||||
g_assert_true(r->amount_limit_reached); g_assert_cmpuint(r->warnings->len, ==, 1);
|
||||
g_assert_cmpuint(count_type(r, BANK_OCCURRENCE_AMOUNT), ==, 0);
|
||||
bank_analysis_result_free(r); g_string_free(text, TRUE);
|
||||
}
|
||||
|
||||
static char *revision_snapshot(const BankAnalysisResult *result)
|
||||
{
|
||||
GString *snapshot = g_string_new(NULL);
|
||||
g_string_append_printf(snapshot, "%u|", result->blocks->len);
|
||||
for (guint i = 0; i < result->blocks->len; i++) {
|
||||
BankProposedBlock *b = g_ptr_array_index(result->blocks, i);
|
||||
g_string_append_printf(snapshot, "%s:%zu:%zu:%d:%d:%d[", b->identifier,
|
||||
b->start_offset, b->end_offset, b->ambiguous, b->origin, b->decision);
|
||||
for (guint j = 0; j < b->occurrence_identifiers->len; j++)
|
||||
g_string_append_printf(snapshot, "%s,",
|
||||
(char *)g_ptr_array_index(b->occurrence_identifiers, j));
|
||||
g_string_append_c(snapshot, ']');
|
||||
}
|
||||
return g_string_free(snapshot, FALSE);
|
||||
}
|
||||
|
||||
#define ASSERT_REVISION_FAILURE_UNCHANGED(result, expression) G_STMT_START { \
|
||||
char *before_snapshot = revision_snapshot(result); \
|
||||
g_assert_false((expression)); \
|
||||
char *after_snapshot = revision_snapshot(result); \
|
||||
g_assert_cmpstr(after_snapshot, ==, before_snapshot); \
|
||||
g_free(after_snapshot); g_free(before_snapshot); \
|
||||
} G_STMT_END
|
||||
|
||||
static void test_revision_identifier_limits_and_atomicity(void)
|
||||
{
|
||||
char *text = g_strdup_printf("IBAN : %s\nIBAN : %s", fr, de);
|
||||
BankAnalysisResult *r = bank_structured_extractor_analyze(text, NULL);
|
||||
BankProposedBlock *base = g_ptr_array_index(r->blocks, 0);
|
||||
const char *occurrence = g_ptr_array_index(base->occurrence_identifiers, 0);
|
||||
char valid[BANK_ANALYSIS_MAX_ID_BYTES + 1U];
|
||||
memset(valid, 'a', BANK_ANALYSIS_MAX_ID_BYTES); valid[BANK_ANALYSIS_MAX_ID_BYTES] = '\0';
|
||||
g_assert_true(bank_analysis_revision_create_block(r, valid, NULL, NULL));
|
||||
char *too_long = g_malloc0(BANK_ANALYSIS_MAX_ID_BYTES + 2U);
|
||||
memset(too_long, 'b', BANK_ANALYSIS_MAX_ID_BYTES + 1U);
|
||||
ASSERT_REVISION_FAILURE_UNCHANGED(r,
|
||||
bank_analysis_revision_create_block(r, too_long, NULL, NULL));
|
||||
ASSERT_REVISION_FAILURE_UNCHANGED(r,
|
||||
bank_analysis_revision_create_block(r, valid, NULL, NULL));
|
||||
ASSERT_REVISION_FAILURE_UNCHANGED(r,
|
||||
bank_analysis_revision_split(r, base->identifier, "new", "absent", NULL));
|
||||
ASSERT_REVISION_FAILURE_UNCHANGED(r,
|
||||
bank_analysis_revision_merge(r, base->identifier, base->identifier, NULL));
|
||||
ASSERT_REVISION_FAILURE_UNCHANGED(r,
|
||||
bank_analysis_revision_move(r, base->identifier, "absent", occurrence, NULL));
|
||||
ASSERT_REVISION_FAILURE_UNCHANGED(r,
|
||||
bank_analysis_revision_remove(r, base->identifier, "absent", NULL));
|
||||
ASSERT_REVISION_FAILURE_UNCHANGED(r,
|
||||
bank_analysis_revision_mark_ambiguous(r, "absent", TRUE, NULL));
|
||||
ASSERT_REVISION_FAILURE_UNCHANGED(r,
|
||||
bank_analysis_revision_reject(r, "absent", NULL));
|
||||
ASSERT_REVISION_FAILURE_UNCHANGED(r,
|
||||
bank_analysis_revision_split(r, base->identifier, too_long, occurrence, NULL));
|
||||
char *immense = g_malloc0(FINANCIAL_TEXT_MAX_BYTES + 1U);
|
||||
memset(immense, 'c', FINANCIAL_TEXT_MAX_BYTES);
|
||||
ASSERT_REVISION_FAILURE_UNCHANGED(r,
|
||||
bank_analysis_revision_create_block(r, immense, NULL, NULL));
|
||||
g_assert_nonnull(bank_analysis_find_occurrence(r, occurrence));
|
||||
g_free(immense); g_free(too_long); bank_analysis_result_free(r); g_free(text);
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
g_test_init(&argc, &argv, NULL);
|
||||
g_test_add_func("/bank-structured/three-accounts", test_three_accounts_and_holders);
|
||||
g_test_add_func("/bank-structured/orphans", test_orphans_ambiguities_duplicates);
|
||||
g_test_add_func("/bank-structured/utf8", test_utf8_punctuation_crlf);
|
||||
g_test_add_func("/bank-structured/amounts", test_amounts_and_currency);
|
||||
g_test_add_func("/bank-structured/transaction", test_explicit_transaction);
|
||||
g_test_add_func("/bank-structured/insufficient", test_insufficient_transaction);
|
||||
g_test_add_func("/bank-structured/revision", test_memory_revision);
|
||||
g_test_add_func("/bank-structured/determinism-limits", test_determinism_and_limits);
|
||||
g_test_add_func("/bank-structured/amount-complete", test_amount_complete_ambiguities);
|
||||
g_test_add_func("/bank-structured/amount-bounds", test_amount_bounds_before_copy);
|
||||
g_test_add_func("/bank-structured/amount-strict-groups", test_amount_strict_groups);
|
||||
g_test_add_func("/bank-structured/amount-punctuation", test_amount_terminal_punctuation);
|
||||
g_test_add_func("/bank-structured/amount-context-limits",
|
||||
test_amount_context_before_limits);
|
||||
g_test_add_func("/bank-structured/revision-atomicity",
|
||||
test_revision_identifier_limits_and_atomicity);
|
||||
return g_test_run();
|
||||
}
|
||||
Loading…
Reference in a new issue