tracks: optimize S21 publication validation
This commit is contained in:
fy59
parent
f13512ad04
commit
da2ac7bdd9
7 changed files with 274 additions and 73 deletions
|
|
@ -320,12 +320,16 @@ power of two at least `1.5 × N + 1`, and `F` the number of distinct Feature
|
|||
Sets resolved from the exact GVR scope, and `M_max` the largest parent Match
|
||||
Result `match_count` in that scope, it reserves:
|
||||
|
||||
`4 MiB + 101 × N + 8 × E_raw + 16 × S + 640 × F + 12 × M_max` bytes.
|
||||
`4 MiB + 133 × N + 8 × E_raw + 16 × S + 640 × F + 12 × M_max` bytes.
|
||||
|
||||
The `101 × N` term is documented code accounting: retained compact nodes (16),
|
||||
The `133 × N` term is documented code accounting: retained compact nodes (16),
|
||||
DSU/group/conflict scratch (17), flat canonical output (24), and worst-case
|
||||
simultaneous per-Track publication serialization (44): stored observation
|
||||
capacity (16), vector objects/capacity (up to 12), and publication rows (16).
|
||||
capacity (16), vector objects/capacity (up to 12), and publication rows (16),
|
||||
plus two 16-byte transient sorted publication-validation keys. Those keys prove
|
||||
global observation uniqueness and one-image-per-Track independently of the
|
||||
Builder before the atomic database transaction. They are released before
|
||||
`COMMIT` and are not persistent state.
|
||||
The other terms are actual reserved
|
||||
edge capacity, actual power-of-two identity slot capacity, and a conservative
|
||||
Feature Set projection/cache/hash allowance. The final term is the largest
|
||||
|
|
@ -340,7 +344,7 @@ coefficients and multiplier did not name live allocations. It is not the
|
|||
current estimator and must not be reinterpreted as such. At S21
|
||||
`E_raw = 6,628,174`, it produced `19,546,898,688` bytes (18.204 GiB), exceeding
|
||||
the `12,750,811,136`-byte host capacity after canonical reserve. The compact
|
||||
formula with `F=0, M_max=0` produces `1,932,981,756` bytes; real admission adds
|
||||
formula with `F=0, M_max=0` produces `2,357,184,892` bytes; real admission adds
|
||||
exactly `640 × F + 12 × M_max` and therefore remains derived from the exact scope rather than from
|
||||
an invented fixed 10.48 GiB target.
|
||||
|
||||
|
|
@ -724,3 +728,39 @@ and 862 MiB, minimum observed MemAvailable was about 8 GiB, no swap delta was
|
|||
observed, and Governor admission passed. No scratch or scratch lease was used.
|
||||
No Feature, Candidate Pair, Matcher or GVR work was replayed or created, and
|
||||
Sparse SfM/Dense remained at zero.
|
||||
|
||||
## Operational performance slice — PASS
|
||||
|
||||
This post-freeze slice changes no Track science, identity, fingerprint,
|
||||
serialization, Task identity, recovery rule or publication boundary. Phase-A
|
||||
profiling on the exact S21 scope measured 6,628,174 raw inlier edges and found
|
||||
the graph work small: resolve/register 6.099 s, DSU/canonicalization 1.477 s,
|
||||
serialization preparation 0.056 s and final snapshot revalidation 1.618 s.
|
||||
The dominant cost was instead the pre-publication defensive validator, which
|
||||
prepared SQLite statements per observation and compared every pair of Tracks.
|
||||
At 912,447 Tracks, that latter all-pairs relation was accidentally
|
||||
superlinear.
|
||||
|
||||
Publication validation now sorts bounded 16-byte observation and image keys,
|
||||
reuses one Feature Set lookup statement per distinct Feature Set, and rejects
|
||||
exactly the prior invalid inputs: invalid feature index, duplicate observation
|
||||
membership or repeated image in one Track. Its complexity is `O(V log V)` for
|
||||
`V` published observations, with an explicit additional 32-byte-per-node peak
|
||||
in the Governor model above. The atomic `BEGIN IMMEDIATE`/`COMMIT` publication
|
||||
and rollback behavior remain unchanged.
|
||||
|
||||
On dedicated reflinks of the immutable S21 GV v3 source, fresh Task 2835
|
||||
published Track Set 1 in 21 s and the interrupted pending Task 2835 recovered
|
||||
to the same result in 24 s, versus the frozen 3,316 s baseline (approximately
|
||||
157.9× and 138.2× respectively). Both retained 912,447 Tracks, 2,495,768
|
||||
observations, min/max/mean length 2/42/2.7352470883240341, zero duplicate
|
||||
memberships and zero duplicate-image components. Their complete canonical row
|
||||
stream is identical to the frozen proof, so the persistent digest remains
|
||||
`c30eba192627bf73eaf21ff30d81038d8cc6bbf36a69226f88cdc8c37f7d74a1`.
|
||||
|
||||
CPU parallelism was deliberately not added: the remaining Track Builder Task
|
||||
is only tens of seconds, while the profile gives no material CPU-bound graph
|
||||
phase whose parallelization would repay the added determinism, ownership and
|
||||
memory-risk surface. Production still uses its one admitted CPU worker; this
|
||||
is a measured operational decision, not an arbitrary CPU1 policy. No GPU,
|
||||
scratch lease, Sparse SfM or downstream work ran, and no swap was observed.
|
||||
|
|
|
|||
172
src/project_db.c
172
src/project_db.c
|
|
@ -8452,77 +8452,149 @@ Lardon3DProjectDbResult lardon3d_project_db_list_track_sets(
|
|||
return result;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
uint64_t feature_set_id;
|
||||
uint32_t feature_index;
|
||||
uint32_t track_index;
|
||||
} TrackObservationValidationKey;
|
||||
|
||||
typedef struct {
|
||||
uint64_t image_id;
|
||||
uint32_t track_index;
|
||||
uint32_t reserved;
|
||||
} TrackImageValidationKey;
|
||||
|
||||
_Static_assert(sizeof(TrackObservationValidationKey) == 16,
|
||||
"Track publication validation must retain a compact key");
|
||||
_Static_assert(sizeof(TrackImageValidationKey) == 16,
|
||||
"Track publication validation must retain a compact image key");
|
||||
|
||||
static int compare_track_observation_validation_key(const void *left, const void *right) {
|
||||
const TrackObservationValidationKey *a = left;
|
||||
const TrackObservationValidationKey *b = right;
|
||||
if (a->feature_set_id != b->feature_set_id)
|
||||
return a->feature_set_id < b->feature_set_id ? -1 : 1;
|
||||
if (a->feature_index != b->feature_index)
|
||||
return a->feature_index < b->feature_index ? -1 : 1;
|
||||
return a->track_index == b->track_index ? 0 :
|
||||
(a->track_index < b->track_index ? -1 : 1);
|
||||
}
|
||||
|
||||
static int compare_track_image_validation_key(const void *left, const void *right) {
|
||||
const TrackImageValidationKey *a = left;
|
||||
const TrackImageValidationKey *b = right;
|
||||
if (a->track_index != b->track_index)
|
||||
return a->track_index < b->track_index ? -1 : 1;
|
||||
return a->image_id == b->image_id ? 0 : (a->image_id < b->image_id ? -1 : 1);
|
||||
}
|
||||
|
||||
static Lardon3DProjectDbResult validate_track_observations_locked(
|
||||
Lardon3DProjectDb *database, const Lardon3DProjectDbTrack *tracks, size_t track_count) {
|
||||
size_t observation_count = 0;
|
||||
if (track_count > UINT32_MAX)
|
||||
return LARDON3D_PROJECT_DB_INVALID_ARGUMENT;
|
||||
for (size_t track_index = 0; track_index < track_count; ++track_index) {
|
||||
const Lardon3DProjectDbTrack *track = &tracks[track_index];
|
||||
if (!track->observations || track->observation_count < 2) {
|
||||
if (!track->observations || track->observation_count < 2 ||
|
||||
track->observation_count > SIZE_MAX - observation_count)
|
||||
return LARDON3D_PROJECT_DB_INVALID_ARGUMENT;
|
||||
observation_count += track->observation_count;
|
||||
}
|
||||
/* CONTRACT: an empty Track Set is a valid complete result when every graph
|
||||
* component conflicts; only non-empty publication needs membership checks. */
|
||||
if (observation_count == 0)
|
||||
return LARDON3D_PROJECT_DB_OK;
|
||||
if (observation_count > SIZE_MAX / sizeof(TrackObservationValidationKey))
|
||||
return LARDON3D_PROJECT_DB_INVALID_ARGUMENT;
|
||||
|
||||
/* WHY/CONTRACT: publication must independently reject duplicate memberships
|
||||
* and repeated images, even though Track Builder normally supplies canonical
|
||||
* output. Sorting compact transient keys is O(V log V), deterministic, and
|
||||
* preserves the same rejection relation as the former all-pairs scan. The
|
||||
* arrays are released before COMMIT and are never persistent state. */
|
||||
TrackObservationValidationKey *observations =
|
||||
malloc(observation_count * sizeof(*observations));
|
||||
TrackImageValidationKey *images = malloc(observation_count * sizeof(*images));
|
||||
if (!observations || !images) {
|
||||
free(observations);
|
||||
free(images);
|
||||
return LARDON3D_PROJECT_DB_IO_ERROR;
|
||||
}
|
||||
size_t offset = 0;
|
||||
for (size_t track_index = 0; track_index < track_count; ++track_index) {
|
||||
const Lardon3DProjectDbTrack *track = &tracks[track_index];
|
||||
for (uint32_t position = 0; position < track->observation_count; ++position) {
|
||||
const Lardon3DProjectDbTrackObservation *observation = &track->observations[position];
|
||||
if (observation->position_in_track != position || observation->feature_set_id == 0) {
|
||||
free(observations);
|
||||
free(images);
|
||||
return LARDON3D_PROJECT_DB_INVALID_ARGUMENT;
|
||||
}
|
||||
for (uint32_t observation_index = 0; observation_index < track->observation_count;
|
||||
++observation_index) {
|
||||
const Lardon3DProjectDbTrackObservation *observation = &track->observations[observation_index];
|
||||
if (observation->position_in_track != observation_index || observation->feature_set_id == 0) {
|
||||
return LARDON3D_PROJECT_DB_INVALID_ARGUMENT;
|
||||
observations[offset++] = (TrackObservationValidationKey) {
|
||||
observation->feature_set_id, observation->feature_index, (uint32_t)track_index};
|
||||
}
|
||||
}
|
||||
qsort(observations, observation_count, sizeof(*observations),
|
||||
compare_track_observation_validation_key);
|
||||
for (size_t index = 1; index < observation_count; ++index) {
|
||||
if (observations[index - 1].feature_set_id == observations[index].feature_set_id &&
|
||||
observations[index - 1].feature_index == observations[index].feature_index) {
|
||||
free(observations);
|
||||
free(images);
|
||||
return LARDON3D_PROJECT_DB_CONSTRAINT;
|
||||
}
|
||||
}
|
||||
|
||||
sqlite3_stmt *statement = NULL;
|
||||
Lardon3DProjectDbResult result = prepare(
|
||||
database, "SELECT image_id,feature_count FROM feature_sets WHERE feature_set_id=?1",
|
||||
&statement);
|
||||
database, "SELECT image_id,feature_count FROM feature_sets WHERE feature_set_id=?1", &statement);
|
||||
if (result != LARDON3D_PROJECT_DB_OK) {
|
||||
free(observations);
|
||||
free(images);
|
||||
return result;
|
||||
}
|
||||
(void)sqlite3_bind_int64(statement, 1, (sqlite3_int64)observation->feature_set_id);
|
||||
for (size_t begin = 0; result == LARDON3D_PROJECT_DB_OK && begin < observation_count;) {
|
||||
size_t end = begin + 1;
|
||||
while (end < observation_count &&
|
||||
observations[end].feature_set_id == observations[begin].feature_set_id)
|
||||
++end;
|
||||
sqlite3_reset(statement);
|
||||
sqlite3_clear_bindings(statement);
|
||||
(void)sqlite3_bind_int64(statement, 1, (sqlite3_int64)observations[begin].feature_set_id);
|
||||
int code = sqlite3_step(statement);
|
||||
sqlite3_int64 image_id = code == SQLITE_ROW ? sqlite3_column_int64(statement, 0) : 0;
|
||||
sqlite3_int64 feature_count = code == SQLITE_ROW ? sqlite3_column_int64(statement, 1) : 0;
|
||||
(void)sqlite3_finalize(statement);
|
||||
if (code != SQLITE_ROW) {
|
||||
return code == SQLITE_DONE ? LARDON3D_PROJECT_DB_NOT_FOUND
|
||||
result = code == SQLITE_DONE ? LARDON3D_PROJECT_DB_NOT_FOUND
|
||||
: sqlite_result(database, code, "validate track feature set");
|
||||
} else if (image_id <= 0 || feature_count < 0) {
|
||||
result = LARDON3D_PROJECT_DB_CONSTRAINT;
|
||||
} else {
|
||||
for (size_t index = begin; index < end; ++index) {
|
||||
if ((uint64_t)observations[index].feature_index >= (uint64_t)feature_count) {
|
||||
result = LARDON3D_PROJECT_DB_CONSTRAINT;
|
||||
break;
|
||||
}
|
||||
if (image_id <= 0 || feature_count < 0 || (uint64_t)observation->feature_index >=
|
||||
(uint64_t)feature_count) {
|
||||
return LARDON3D_PROJECT_DB_CONSTRAINT;
|
||||
images[index] = (TrackImageValidationKey) {
|
||||
(uint64_t)image_id, observations[index].track_index, 0};
|
||||
}
|
||||
for (uint32_t prior = 0; prior < observation_index; ++prior) {
|
||||
const Lardon3DProjectDbTrackObservation *old = &track->observations[prior];
|
||||
if (old->feature_set_id == observation->feature_set_id &&
|
||||
old->feature_index == observation->feature_index) {
|
||||
return LARDON3D_PROJECT_DB_CONSTRAINT;
|
||||
}
|
||||
sqlite3_stmt *image_statement = NULL;
|
||||
result = prepare(database, "SELECT image_id FROM feature_sets WHERE feature_set_id=?1",
|
||||
&image_statement);
|
||||
if (result != LARDON3D_PROJECT_DB_OK) {
|
||||
begin = end;
|
||||
}
|
||||
(void)sqlite3_finalize(statement);
|
||||
if (result == LARDON3D_PROJECT_DB_OK) {
|
||||
qsort(images, observation_count, sizeof(*images), compare_track_image_validation_key);
|
||||
for (size_t index = 1; index < observation_count; ++index) {
|
||||
if (images[index - 1].track_index == images[index].track_index &&
|
||||
images[index - 1].image_id == images[index].image_id) {
|
||||
result = LARDON3D_PROJECT_DB_CONSTRAINT;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
free(observations);
|
||||
free(images);
|
||||
return result;
|
||||
}
|
||||
(void)sqlite3_bind_int64(image_statement, 1, (sqlite3_int64)old->feature_set_id);
|
||||
code = sqlite3_step(image_statement);
|
||||
sqlite3_int64 old_image = code == SQLITE_ROW ? sqlite3_column_int64(image_statement, 0) : 0;
|
||||
(void)sqlite3_finalize(image_statement);
|
||||
if (code != SQLITE_ROW) {
|
||||
return LARDON3D_PROJECT_DB_CORRUPT;
|
||||
}
|
||||
if (old_image == image_id) {
|
||||
return LARDON3D_PROJECT_DB_CONSTRAINT;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (size_t first = 0; first < track_count; ++first) {
|
||||
for (size_t second = first + 1; second < track_count; ++second) {
|
||||
for (uint32_t a = 0; a < tracks[first].observation_count; ++a) {
|
||||
for (uint32_t b = 0; b < tracks[second].observation_count; ++b) {
|
||||
if (tracks[first].observations[a].feature_set_id == tracks[second].observations[b].feature_set_id &&
|
||||
tracks[first].observations[a].feature_index == tracks[second].observations[b].feature_index) {
|
||||
return LARDON3D_PROJECT_DB_CONSTRAINT;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return LARDON3D_PROJECT_DB_OK;
|
||||
}
|
||||
|
||||
Lardon3DProjectDbResult lardon3d_project_db_create_track_set(
|
||||
|
|
|
|||
|
|
@ -106,9 +106,16 @@ uint32_t tb::CompactGraph::register_feature(const FeatureMetadata &m) {
|
|||
void tb::CompactGraph::insert_identity(const Node &node, uint32_t index) {
|
||||
const size_t mask = identity_.size() - 1U;
|
||||
size_t slot = static_cast<size_t>(hash_key(node.feature_set_id, node.feature_index)) & mask;
|
||||
while (identity_[slot].node_plus_one) slot = (slot + 1U) & mask;
|
||||
uint64_t probes = 1;
|
||||
while (identity_[slot].node_plus_one) {
|
||||
slot = (slot + 1U) & mask;
|
||||
++probes;
|
||||
}
|
||||
identity_[slot] = {node.feature_set_id, node.feature_index, index + 1U};
|
||||
++identity_size_;
|
||||
++profile_.identity_inserts;
|
||||
profile_.identity_probes += probes;
|
||||
profile_.identity_max_probe = std::max(profile_.identity_max_probe, probes);
|
||||
}
|
||||
|
||||
uint32_t tb::CompactGraph::resolve_node(uint32_t metadata_index,
|
||||
|
|
@ -118,21 +125,29 @@ uint32_t tb::CompactGraph::resolve_node(uint32_t metadata_index,
|
|||
const uint64_t set = metadata_[metadata_index].feature_set_id;
|
||||
const size_t mask = identity_.size() - 1U;
|
||||
size_t slot = static_cast<size_t>(hash_key(set, feature_index)) & mask;
|
||||
uint64_t probes = 1;
|
||||
while (identity_[slot].node_plus_one) {
|
||||
const IdentitySlot &found = identity_[slot];
|
||||
if (found.feature_set_id == set && found.feature_index == feature_index) {
|
||||
const uint32_t index = found.node_plus_one - 1U;
|
||||
if (nodes_[index].metadata_index != metadata_index)
|
||||
throw std::invalid_argument("contradictory observation");
|
||||
++profile_.identity_lookups;
|
||||
profile_.identity_probes += probes;
|
||||
profile_.identity_max_probe = std::max(profile_.identity_max_probe, probes);
|
||||
return index;
|
||||
}
|
||||
slot = (slot + 1U) & mask;
|
||||
++probes;
|
||||
}
|
||||
if (nodes_.size() == UINT32_MAX || identity_size_ == identity_.size())
|
||||
throw std::bad_alloc();
|
||||
nodes_.push_back({set, feature_index, metadata_index});
|
||||
const uint32_t index = static_cast<uint32_t>(nodes_.size() - 1U);
|
||||
insert_identity(nodes_.back(), index);
|
||||
++profile_.identity_lookups;
|
||||
profile_.identity_probes += probes;
|
||||
profile_.identity_max_probe = std::max(profile_.identity_max_probe, probes);
|
||||
return index;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -55,6 +55,16 @@ struct Output {
|
|||
std::vector<TrackRange> tracks;
|
||||
};
|
||||
|
||||
/* WHY: large real scopes need diagnosis without changing their durable task or
|
||||
* scientific result. These counters describe only transient hash-table work;
|
||||
* they are never serialized, fingerprinted, or consulted by the Builder. */
|
||||
struct Profile {
|
||||
uint64_t identity_lookups = 0;
|
||||
uint64_t identity_probes = 0;
|
||||
uint64_t identity_max_probe = 0;
|
||||
uint64_t identity_inserts = 0;
|
||||
};
|
||||
|
||||
/* WHY: Project construction must have one owner for nodes, identity lookup and
|
||||
* indexed edges. Transporting pointer edges through the public ABI recreated
|
||||
* the complete graph twice before DSU. This private type is deliberately not
|
||||
|
|
@ -70,6 +80,7 @@ class CompactGraph {
|
|||
uint64_t node_image(uint32_t node_index) const;
|
||||
size_t node_count() const { return nodes_.size(); }
|
||||
uint64_t raw_edge_count() const { return raw_edge_count_; }
|
||||
const Profile &profile() const { return profile_; }
|
||||
|
||||
private:
|
||||
uint32_t resolve_node(uint32_t metadata_index, uint32_t feature_index);
|
||||
|
|
@ -81,6 +92,7 @@ class CompactGraph {
|
|||
std::vector<IdentitySlot> identity_;
|
||||
size_t identity_size_ = 0;
|
||||
uint64_t raw_edge_count_ = 0;
|
||||
Profile profile_;
|
||||
};
|
||||
|
||||
using Checkpoint = bool (*)(void *userdata);
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <chrono>
|
||||
#include <cstring>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <fcntl.h>
|
||||
#include <limits>
|
||||
#include <new>
|
||||
|
|
@ -25,6 +27,44 @@ extern "C" {
|
|||
namespace {
|
||||
namespace internal = lardon3d::track_builder_internal;
|
||||
|
||||
bool profile_enabled() {
|
||||
const char *value = std::getenv("LARDON3D_TRACK_BUILDER_PROFILE");
|
||||
return value != nullptr && std::strcmp(value, "1") == 0;
|
||||
}
|
||||
|
||||
double elapsed_seconds(const std::chrono::steady_clock::time_point &start) {
|
||||
return std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count();
|
||||
}
|
||||
|
||||
/* CONTRACT: profile output is opt-in diagnostic stderr only. It carries no
|
||||
* task identity or scientific state, so a restarted task remains exactly the
|
||||
* same durable operation whether profiling is enabled or absent. */
|
||||
void report_profile(bool enabled, uint64_t gvr_count, uint64_t raw_edge_hint,
|
||||
const internal::CompactGraph &graph, double hint_seconds,
|
||||
double graph_seconds, double resolve_seconds, double core_seconds,
|
||||
double serialize_seconds, double revalidate_seconds) {
|
||||
if (!enabled) return;
|
||||
const internal::Profile &profile = graph.profile();
|
||||
const uint64_t identity_operations = profile.identity_lookups + profile.identity_inserts;
|
||||
const double mean_probe = identity_operations == 0
|
||||
? 0.0
|
||||
: static_cast<double>(profile.identity_probes) /
|
||||
static_cast<double>(identity_operations);
|
||||
std::fprintf(stderr,
|
||||
"track_builder_profile gvr=%llu raw_edge_hint=%llu nodes=%zu raw_edges=%llu "
|
||||
"hint_s=%.3f graph_s=%.3f resolve_s=%.3f core_s=%.3f serialize_s=%.3f "
|
||||
"revalidate_s=%.3f identity_lookups=%llu identity_inserts=%llu "
|
||||
"identity_mean_probe=%.3f identity_max_probe=%llu\n",
|
||||
static_cast<unsigned long long>(gvr_count),
|
||||
static_cast<unsigned long long>(raw_edge_hint), graph.node_count(),
|
||||
static_cast<unsigned long long>(graph.raw_edge_count()), hint_seconds,
|
||||
graph_seconds, resolve_seconds, core_seconds, serialize_seconds,
|
||||
revalidate_seconds,
|
||||
static_cast<unsigned long long>(profile.identity_lookups),
|
||||
static_cast<unsigned long long>(profile.identity_inserts), mean_probe,
|
||||
static_cast<unsigned long long>(profile.identity_max_probe));
|
||||
}
|
||||
|
||||
#ifdef LARDON3D_TRACK_BUILDER_PROJECT_TESTING
|
||||
extern "C" void lardon3d_track_builder_project_test_publication_capacities(
|
||||
uint64_t node_count, uint64_t serialization_capacity_bytes);
|
||||
|
|
@ -225,6 +265,7 @@ Lardon3DTrackBuilderProjectStatus internal::build_project(
|
|||
return LARDON3D_TRACK_BUILDER_PROJECT_INVALID_ARGUMENT;
|
||||
}
|
||||
try {
|
||||
const bool profiling = profile_enabled();
|
||||
std::vector<uint64_t> owned_ids(request->gvr_ids,
|
||||
request->gvr_ids + request->gvr_count);
|
||||
Lardon3DTrackBuilderProjectRequest owned_request = *request;
|
||||
|
|
@ -259,6 +300,7 @@ Lardon3DTrackBuilderProjectStatus internal::build_project(
|
|||
}
|
||||
if (db != LARDON3D_PROJECT_DB_NOT_FOUND) return map_db(db);
|
||||
|
||||
const auto hint_start = std::chrono::steady_clock::now();
|
||||
uint64_t raw_edge_hint = 0;
|
||||
for (size_t i = 0; i < request->gvr_count; ++i) {
|
||||
Lardon3DProjectDbGeometricVerificationResult gvr{};
|
||||
|
|
@ -269,8 +311,12 @@ Lardon3DTrackBuilderProjectStatus internal::build_project(
|
|||
return LARDON3D_TRACK_BUILDER_PROJECT_OUT_OF_MEMORY;
|
||||
raw_edge_hint += gvr.inlier_count;
|
||||
}
|
||||
const double hint_seconds = elapsed_seconds(hint_start);
|
||||
const auto graph_start = std::chrono::steady_clock::now();
|
||||
internal::CompactGraph graph(raw_edge_hint);
|
||||
const double graph_seconds = elapsed_seconds(graph_start);
|
||||
std::unordered_map<uint64_t, FeatureCacheEntry> cache;
|
||||
const auto resolve_start = std::chrono::steady_clock::now();
|
||||
for (size_t i = 0; i < request->gvr_count; ++i) {
|
||||
auto status = resolve_gvr(*request, request->gvr_ids[i], cache, graph);
|
||||
if (status != LARDON3D_TRACK_BUILDER_PROJECT_OK) return status;
|
||||
|
|
@ -280,6 +326,8 @@ Lardon3DTrackBuilderProjectStatus internal::build_project(
|
|||
if (checkpoint && !checkpoint(checkpoint_userdata))
|
||||
return LARDON3D_TRACK_BUILDER_PROJECT_INTERRUPTED;
|
||||
}
|
||||
const double resolve_seconds = elapsed_seconds(resolve_start);
|
||||
const auto core_start = std::chrono::steady_clock::now();
|
||||
internal::Output core;
|
||||
auto core_status = graph.build(&core);
|
||||
if (core_status != LARDON3D_TRACK_BUILDER_OK) {
|
||||
|
|
@ -287,6 +335,7 @@ Lardon3DTrackBuilderProjectStatus internal::build_project(
|
|||
? LARDON3D_TRACK_BUILDER_PROJECT_OUT_OF_MEMORY
|
||||
: LARDON3D_TRACK_BUILDER_PROJECT_CORE_ERROR;
|
||||
}
|
||||
const double core_seconds = elapsed_seconds(core_start);
|
||||
/* GATE D CONTRACT: Gate B (DSU/canonicalization) is deliberately
|
||||
* non-preemptible, but a pause/cancel raised during it must be observed
|
||||
* before any publication preparation can become durable. */
|
||||
|
|
@ -296,6 +345,7 @@ Lardon3DTrackBuilderProjectStatus internal::build_project(
|
|||
* canonical compact membership to Track Model rows. position_in_track is
|
||||
* the already-canonical flat order; DB publication below remains the sole
|
||||
* atomic, owner-only persistence point. */
|
||||
const auto serialize_start = std::chrono::steady_clock::now();
|
||||
std::vector<std::vector<Lardon3DProjectDbTrackObservation>> stored(core.tracks.size());
|
||||
std::vector<Lardon3DProjectDbTrack> publish(core.tracks.size());
|
||||
for (size_t i = 0; i < core.tracks.size(); ++i) {
|
||||
|
|
@ -308,6 +358,7 @@ Lardon3DTrackBuilderProjectStatus internal::build_project(
|
|||
}
|
||||
publish[i] = {0, 0, static_cast<uint32_t>(stored[i].size()), stored[i].data()};
|
||||
}
|
||||
const double serialize_seconds = elapsed_seconds(serialize_start);
|
||||
#ifdef LARDON3D_TRACK_BUILDER_PROJECT_TESTING
|
||||
uint64_t stored_observation_capacity = 0;
|
||||
for (const auto &observations : stored)
|
||||
|
|
@ -321,12 +372,17 @@ Lardon3DTrackBuilderProjectStatus internal::build_project(
|
|||
lardon3d_track_builder_project_test_before_revalidation(
|
||||
request->database, request->gvr_ids, request->gvr_count);
|
||||
#endif
|
||||
const auto revalidate_start = std::chrono::steady_clock::now();
|
||||
for (size_t i = 0; i < request->gvr_count; ++i) {
|
||||
db = validate_gvr_snapshot(*request, request->gvr_ids[i]);
|
||||
if (db != LARDON3D_PROJECT_DB_OK) {
|
||||
return map_db(db);
|
||||
}
|
||||
}
|
||||
const double revalidate_seconds = elapsed_seconds(revalidate_start);
|
||||
report_profile(profiling, request->gvr_count, raw_edge_hint, graph, hint_seconds,
|
||||
graph_seconds, resolve_seconds, core_seconds, serialize_seconds,
|
||||
revalidate_seconds);
|
||||
/* GATE D CONTRACT: this is the final interruptible boundary. Refusal
|
||||
* guarantees zero publication; create_track_set below remains one
|
||||
* non-preemptible atomic publication. */
|
||||
|
|
|
|||
|
|
@ -66,15 +66,16 @@ bool compact_memory_estimate(uint64_t raw_edges, uint64_t feature_sets,
|
|||
}
|
||||
uint64_t graph_and_peak_node_bytes = 0, edge_bytes = 0;
|
||||
uint64_t identity_bytes = 0, feature_bytes = 0, match_file_peak_bytes = 0;
|
||||
/* WHY/ACCOUNTING: 101 B/node is the simultaneous retained node (16),
|
||||
* DSU/group/conflict scratch (17), flat canonical output (24) and worst
|
||||
/* WHY/ACCOUNTING: 133 B/node is the simultaneous retained node (16),
|
||||
* DSU/group/conflict scratch (17), flat canonical output (24), worst
|
||||
* per-track publication serialization (44: stored observation capacity 16,
|
||||
* per-track vector capacity/objects up to 12, and publish rows 16). Identity slots and raw indexed
|
||||
* edges are charged by their actual reserved capacities. 640 B/Feature Set
|
||||
* per-track vector capacity/objects up to 12, and publish rows 16), plus
|
||||
* two 16-byte sorted transient publication-validation keys. Identity slots
|
||||
* and raw indexed edges are charged by their actual reserved capacities. 640 B/Feature Set
|
||||
* bounds the metadata projection plus the adapter cache/hash allocation.
|
||||
* WHY/CONTRACT: resolve_gvr materializes exactly one Match File at a time;
|
||||
* charge its largest entry vector as a peak, not the sum across the scope. */
|
||||
if (!checked_mul(nodes, 101U, &graph_and_peak_node_bytes) ||
|
||||
if (!checked_mul(nodes, 133U, &graph_and_peak_node_bytes) ||
|
||||
!checked_mul(raw_edges, sizeof(lardon3d::track_builder_internal::Edge), &edge_bytes) ||
|
||||
!checked_mul(slots, sizeof(lardon3d::track_builder_internal::IdentitySlot),
|
||||
&identity_bytes) ||
|
||||
|
|
|
|||
|
|
@ -742,7 +742,10 @@ void run_durable_task_case() {
|
|||
uint64_t task_id = 0;
|
||||
CHECK(lardon3d_project_enqueue_track_builder_task(&state, &configuration, &task_id));
|
||||
Lardon3DTaskSnapshot snapshot{};
|
||||
for (size_t attempt = 0; attempt < 200; ++attempt) {
|
||||
/* WHY: ASan/UBSan make the durable worker's SQLite startup materially slower
|
||||
* on loaded hosts. This is an observation deadline only; it neither changes
|
||||
* Task cancellation semantics nor permits a non-terminal publication. */
|
||||
for (size_t attempt = 0; attempt < 500; ++attempt) {
|
||||
CHECK(lardon3d_task_queue_get(queue, task_id, &snapshot));
|
||||
if (snapshot.state == TASK_COMPLETED || snapshot.state == TASK_FAILED ||
|
||||
snapshot.state == TASK_CANCELLED)
|
||||
|
|
@ -836,7 +839,9 @@ void run_crash_recovery_case() {
|
|||
setenv("LARDON3D_TRACK_BUILDER_TEST_SKIP_FINISHED", "1", 1);
|
||||
CHECK(lardon3d_project_enqueue_track_builder_task(&state, &configuration, &task_id));
|
||||
Lardon3DTaskSnapshot snapshot{};
|
||||
for (size_t attempt = 0; attempt < 200; ++attempt) {
|
||||
/* Same bounded sanitizer/loaded-host observation allowance as the durable
|
||||
* enqueue path above; recovery semantics remain asserted after completion. */
|
||||
for (size_t attempt = 0; attempt < 500; ++attempt) {
|
||||
CHECK(lardon3d_task_queue_get(queue, task_id, &snapshot));
|
||||
if (snapshot.state == TASK_COMPLETED) break;
|
||||
usleep(10000);
|
||||
|
|
@ -1027,7 +1032,7 @@ void run_s21_admission_estimate_case() {
|
|||
constexpr uint64_t historical_envelope = 19546898688ULL;
|
||||
uint64_t compact = 0;
|
||||
CHECK(lardon3d_track_builder_task_memory_estimate(edges, 0, &compact));
|
||||
CHECK(compact == 1932981756ULL && compact < capacity_after_canonical_reserve &&
|
||||
CHECK(compact == 2357184892ULL && compact < capacity_after_canonical_reserve &&
|
||||
historical_envelope > capacity_after_canonical_reserve);
|
||||
CHECK(!lardon3d_track_builder_task_memory_estimate(
|
||||
static_cast<uint64_t>(UINT32_MAX) / 2U + 1U, 0, &compact));
|
||||
|
|
@ -1155,7 +1160,7 @@ void run_disjoint_publication_capacity_case() {
|
|||
while (slots < required_slots) slots *= 2U;
|
||||
const uint64_t legacy_undercharge = 4ULL * 1024ULL * 1024ULL +
|
||||
85ULL * g_publication_nodes + 8ULL * track_count + 16ULL * slots + 640ULL * 2U;
|
||||
CHECK(estimate == legacy_undercharge + 16ULL * g_publication_nodes);
|
||||
CHECK(estimate == legacy_undercharge + 48ULL * g_publication_nodes);
|
||||
|
||||
Lardon3DHardwareProfile profile{};
|
||||
char error[128]{};
|
||||
|
|
|
|||
Loading…
Reference in a new issue