From 3e54f669c0da843c3e39a96e179a2b49025b2064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yi=C4=9Fithan=20Yi=C4=9Fit?= Date: Sun, 2 Feb 2025 04:49:41 +0300 Subject: [PATCH 1/5] fix: bugfix for adding/removing models. --- libvmaf/src/feature/feature_collector.c | 41 +++++---- libvmaf/test/test_feature_collector.c | 109 ++++++++++++++++++++---- 2 files changed, 117 insertions(+), 33 deletions(-) diff --git a/libvmaf/src/feature/feature_collector.c b/libvmaf/src/feature/feature_collector.c index 2eb2ec643..8e0105c22 100644 --- a/libvmaf/src/feature/feature_collector.c +++ b/libvmaf/src/feature/feature_collector.c @@ -242,17 +242,17 @@ int vmaf_feature_collector_mount_model(VmafFeatureCollector *feature_collector, VmafPredictModel *m = malloc(sizeof(VmafPredictModel)); if (!m) return -ENOMEM; + m->model = model; m->next = NULL; - VmafPredictModel **head = &feature_collector->models; - while (*head && (*head)->next != NULL) - *head = (*head)->next; - - if (!(*head)) - *head = m; - else - (*head)->next = m; + VmafPredictModel *head = feature_collector->models; + if (!head) { + feature_collector->models = m; + } else { + while (head->next) head = head->next; + head->next = m; + } return 0; } @@ -263,17 +263,24 @@ int vmaf_feature_collector_unmount_model(VmafFeatureCollector *feature_collector if (!feature_collector) return -EINVAL; if (!model) return -EINVAL; - VmafPredictModel **head = &feature_collector->models; - while (*head && (*head)->model != model) - head = &(*head)->next; + VmafPredictModel *head = feature_collector->models; + VmafPredictModel *prev = NULL; - if (!(*head)) return -EINVAL; - - VmafPredictModel *m = *head; - *head = m->next; - free(m); + while (head) { + if (head->model == model) { + if (prev) { + prev->next = head->next; + } else { + feature_collector->models = head->next; + } + free(head); + return 0; + } + prev = head; + head = head->next; + } - return 0; + return -ENOENT; } int vmaf_feature_collector_register_metadata(VmafFeatureCollector *feature_collector, diff --git a/libvmaf/test/test_feature_collector.c b/libvmaf/test/test_feature_collector.c index 647d07703..e30073a35 100644 --- a/libvmaf/test/test_feature_collector.c +++ b/libvmaf/test/test_feature_collector.c @@ -57,21 +57,58 @@ static char* test_model_mount() err = vmaf_feature_collector_init(&feature_collector); mu_assert("problem during vmaf_feature_collector_init", !err); - VmafModelConfig model_cfg = { 0 }; - VmafModel *model; - vmaf_model_load(&model, &model_cfg, "vmaf_v0.6.1"); - mu_assert("problem during vmaf_model_load", model); + char *model_names[] = {"vmaf_0", "vmaf_1", "vmaf_2"}; + + VmafModelConfig model_cfg = { + .name = model_names[0], + .flags = VMAF_MODEL_FLAGS_DEFAULT, + }; + VmafModelConfig model_cfg1 = { + .name = model_names[1], + .flags = VMAF_MODEL_FLAGS_DEFAULT, + }; - err = vmaf_feature_collector_mount_model(feature_collector, model); + VmafModelConfig model_cfg2 = { + .name = model_names[2], + .flags = VMAF_MODEL_FLAGS_DEFAULT, + }; + + VmafModel *model0, *model1, *model2; + vmaf_model_load(&model0, &model_cfg, "vmaf_v0.6.1"); + mu_assert("problem during vmaf_model_load", model0); + + vmaf_model_load(&model1, &model_cfg1, "vmaf_v0.6.1"); + mu_assert("problem during vmaf_model_load", model0); + + vmaf_model_load(&model2, &model_cfg2, "vmaf_v0.6.1"); + mu_assert("problem during vmaf_model_load", model0); + + err = vmaf_feature_collector_mount_model(feature_collector, model0); mu_assert("problem during vmaf_model_mount", feature_collector->models); - err = vmaf_feature_collector_mount_model(feature_collector, model); + err = vmaf_feature_collector_mount_model(feature_collector, model1); mu_assert("problem during vmaf_model_mount", feature_collector->models->next); - vmaf_model_destroy(model); + err = vmaf_feature_collector_mount_model(feature_collector, model2); + mu_assert("problem during vmaf_model_mount", + feature_collector->models->next); + + VmafPredictModel *model_iter = feature_collector->models; + unsigned i = 0; + + while (model_iter) { + mu_assert("model name does not match", !strcmp(model_iter->model->name, + model_names[i])); + model_iter = model_iter->next; + i++; + } + + vmaf_model_destroy(model0); + vmaf_model_destroy(model1); + vmaf_model_destroy(model2); vmaf_feature_collector_destroy(feature_collector); return NULL; @@ -85,20 +122,60 @@ static char* test_model_unmount() err = vmaf_feature_collector_init(&feature_collector); mu_assert("problem during vmaf_feature_collector_init", !err); - VmafModelConfig model_cfg = { 0 }; - VmafModel *model; - vmaf_model_load(&model, &model_cfg, "vmaf_v0.6.1"); - mu_assert("problem during vmaf_model_load", model); + char *model_names[] = {"vmaf_0", "vmaf_1", "vmaf_2"}; + + VmafModelConfig model_cfg = { + .name = model_names[0], + .flags = VMAF_MODEL_FLAGS_DEFAULT, + }; + + VmafModelConfig model_cfg1 = { + .name = model_names[1], + .flags = VMAF_MODEL_FLAGS_DEFAULT, + }; + + VmafModelConfig model_cfg2 = { + .name = model_names[2], + .flags = VMAF_MODEL_FLAGS_DEFAULT, + }; + + VmafModel *model0, *model1, *model2; + vmaf_model_load(&model0, &model_cfg, "vmaf_v0.6.1"); + mu_assert("problem during vmaf_model_load", model0); - err = vmaf_feature_collector_mount_model(feature_collector, model); + vmaf_model_load(&model1, &model_cfg1, "vmaf_v0.6.1"); + mu_assert("problem during vmaf_model_load", model0); + + vmaf_model_load(&model2, &model_cfg2, "vmaf_v0.6.1"); + mu_assert("problem during vmaf_model_load", model0); + + err = vmaf_feature_collector_mount_model(feature_collector, model0); mu_assert("problem during vmaf_model_mount", feature_collector->models); - err = vmaf_feature_collector_unmount_model(feature_collector, model); - mu_assert("problem during vmaf_model_unmount", - !feature_collector->models); + err = vmaf_feature_collector_mount_model(feature_collector, model1); + mu_assert("problem during vmaf_model_mount", + feature_collector->models->next); - vmaf_model_destroy(model); + err = vmaf_feature_collector_mount_model(feature_collector, model2); + mu_assert("problem during vmaf_model_mount", + feature_collector->models->next); + + err = vmaf_feature_collector_unmount_model(feature_collector, model0); + mu_assert("problem during vmaf_model_unmount", !err); + + err = vmaf_feature_collector_unmount_model(feature_collector, model1); + mu_assert("problem during vmaf_model_unmount", !err); + + err = vmaf_feature_collector_unmount_model(feature_collector, model2); + mu_assert("problem during vmaf_model_unmount", !err); + + mu_assert("feature_collector->models should be NULL", + !feature_collector->models); + + vmaf_model_destroy(model0); + vmaf_model_destroy(model1); + vmaf_model_destroy(model2); vmaf_feature_collector_destroy(feature_collector); return NULL; From 556a7cb0903c9d5fb138a2b90181227708a04ab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yi=C4=9Fithan=20Yi=C4=9Fit?= Date: Sat, 1 Feb 2025 18:33:16 +0300 Subject: [PATCH 2/5] refactor: cleanup unnecessary fields --- libvmaf/src/metadata_handler.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/libvmaf/src/metadata_handler.h b/libvmaf/src/metadata_handler.h index f5f781fa4..1a21e17f6 100644 --- a/libvmaf/src/metadata_handler.h +++ b/libvmaf/src/metadata_handler.h @@ -23,8 +23,6 @@ typedef struct VmafCallbackItem { VmafMetadataConfiguration metadata_cfg; - void (*callback)(void *, VmafMetadata *); - void *data; struct VmafCallbackItem *next; } VmafCallbackItem; From dd4d749f79e677abd1ca7e5d77620ccb3b7a72ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yi=C4=9Fithan=20Yi=C4=9Fit?= Date: Sat, 1 Feb 2025 18:35:55 +0300 Subject: [PATCH 3/5] refactor: Move tests from `test_predict.c` to `test_propagate_metadata.c`. Implement new tests for non-monotnic frames and multiple callbacks. --- libvmaf/test/meson.build | 9 +- libvmaf/test/test_predict.c | 101 +----- libvmaf/test/test_propagate_metadata.c | 413 ++++++++++++++++++++++++- 3 files changed, 416 insertions(+), 107 deletions(-) diff --git a/libvmaf/test/meson.build b/libvmaf/test/meson.build index e8d943e98..2ace210d9 100644 --- a/libvmaf/test/meson.build +++ b/libvmaf/test/meson.build @@ -18,8 +18,11 @@ test_picture = executable('test_picture', ) test_propagate_metadata = executable('test_propagate_metadata', - ['test.c', 'test_propagate_metadata.c', '../src/metadata_handler.c'], + ['test.c', 'test_propagate_metadata.c', '../src/metadata_handler.c','../src/dict.c', + '../src/feature/feature_collector.c', '../src/feature/alias.c', '../src/model.c', '../src/svm.cpp', '../src/log.c', + '../src/read_json_model.c', '../src/pdjson.c', json_model_c_sources, '../src/feature/feature_name.c', '../src/feature/feature_extractor.c'], include_directories : [libvmaf_inc, test_inc, include_directories('../src/')], + link_with : get_option('default_library') == 'both' ? libvmaf.get_static_lib() : libvmaf, ) test_feature_collector = executable('test_feature_collector', @@ -50,8 +53,8 @@ test_model = executable('test_model', ) test_predict = executable('test_predict', - ['test.c', 'test_predict.c', '../src/dict.c', '../src/metadata_handler.c', - '../src/feature/feature_collector.c', '../src/feature/alias.c', '../src/model.c', '../src/svm.cpp', '../src/log.c', + ['test.c', 'test_predict.c', '../src/dict.c', '../src/feature/feature_collector.c', + '../src/feature/alias.c', '../src/model.c', '../src/svm.cpp', '../src/log.c', '../src/read_json_model.c', '../src/pdjson.c', json_model_c_sources, '../src/feature/feature_name.c', '../src/feature/feature_extractor.c',], include_directories : [libvmaf_inc, test_inc, include_directories('../src/')], link_with : get_option('default_library') == 'both' ? libvmaf.get_static_lib() : libvmaf, diff --git a/libvmaf/test/test_predict.c b/libvmaf/test/test_predict.c index 3fe970047..dcebda55e 100644 --- a/libvmaf/test/test_predict.c +++ b/libvmaf/test/test_predict.c @@ -19,19 +19,13 @@ #include #include "feature/feature_collector.h" -#include "metadata_handler.h" #include "test.h" #include "predict.h" #include "predict.c" #include #include - -typedef struct { - VmafDictionary **metadata; - int flags; -} MetaStruct; - +#include static char *test_predict_score_at_index() { @@ -65,96 +59,6 @@ static char *test_predict_score_at_index() return NULL; } - -void set_meta(void *data, VmafMetadata *metadata) -{ - if (!data) return; - MetaStruct *meta = data; - char key[128], value[128]; - snprintf(key, sizeof(value), "%s_%d", metadata->feature_name, - metadata->picture_index); - snprintf(value, sizeof(value), "%f", metadata->score); - vmaf_dictionary_set(meta->metadata, key, value, meta->flags); -} - -static char* test_propagate_metadata() -{ - int err; - - VmafDictionary *dict = NULL; - MetaStruct meta_data = { - .metadata = &dict, - .flags = 0, - }; - - VmafMetadataConfiguration m = { - .feature_name = "vmaf", - .callback = set_meta, - .data = &meta_data, - }; - - VmafFeatureCollector *feature_collector; - err = vmaf_feature_collector_init(&feature_collector); - mu_assert("problem during vmaf_feature_collector_init", !err); - - err = vmaf_feature_collector_register_metadata(feature_collector, m); - mu_assert("problem during vmaf_feature_collector_register_metadata_0", !err); - - VmafModel *model; - VmafModelConfig cfg = { - .name = "vmaf", - .flags = VMAF_MODEL_FLAGS_DEFAULT, - }; - err = vmaf_model_load(&model, &cfg, "vmaf_v0.6.1"); - mu_assert("problem during vmaf_model_load", !err); - err = vmaf_feature_collector_mount_model(feature_collector, model); - mu_assert("problem during vmaf_mount_model", !err); - - for (unsigned i = 0; i < model->n_features; i++) { - err = vmaf_feature_collector_append(feature_collector, - model->feature[i].name, 60., 0); - mu_assert("problem during vmaf_feature_collector_append", !err); - } - - VmafDictionaryEntry *e = vmaf_dictionary_get(&dict, "vmaf_0", 0); - mu_assert("error on propagaton metadata: propagated key not found!", - e); - mu_assert("error on propagaton metadata: propagated key wrong!", - !strcmp(e->key, "vmaf_0")); - mu_assert("error on propagaton metadata: propagated data wrong!", - !strcmp(e->val, "100.000000")); - - vmaf_feature_collector_destroy(feature_collector); - - m.data = NULL; - err = vmaf_feature_collector_init(&feature_collector); - mu_assert("problem during vmaf_feature_collector_init", !err); - - err = vmaf_feature_collector_register_metadata(feature_collector, m); - mu_assert("problem during vmaf_feature_collector_register_metadata_1", !err); - - for (unsigned i = 0; i < model->n_features; i++) { - err = vmaf_feature_collector_append(feature_collector, - model->feature[i].name, 60., 0); - mu_assert("problem during vmaf_feature_collector_append", !err); - } - - vmaf_feature_collector_destroy(feature_collector); - - m.callback = NULL; - err = vmaf_feature_collector_init(&feature_collector); - mu_assert("problem during vmaf_feature_collector_init", !err); - - err = vmaf_feature_collector_register_metadata(feature_collector, m); - mu_assert("problem during vmaf_feature_collector_register_metadata_2", err); - - vmaf_feature_collector_destroy(feature_collector); - - vmaf_model_destroy(model); - return NULL; - -} - static char *test_find_linear_function_parameters() { int err; @@ -271,6 +175,5 @@ char *run_tests() mu_run_test(test_predict_score_at_index); mu_run_test(test_find_linear_function_parameters); mu_run_test(test_piecewise_linear_mapping); - mu_run_test(test_propagate_metadata); return NULL; -} +} \ No newline at end of file diff --git a/libvmaf/test/test_propagate_metadata.c b/libvmaf/test/test_propagate_metadata.c index ee4faced3..85116084b 100644 --- a/libvmaf/test/test_propagate_metadata.c +++ b/libvmaf/test/test_propagate_metadata.c @@ -16,10 +16,52 @@ * */ -#include "metadata_handler.h" #include "test.h" +#include "metadata_handler.h" +#include "feature/feature_collector.h" +#include "predict.h" +#include "predict.c" + +#include +#include +#include +#include +#include + +typedef struct { + VmafDictionary **metadata; + int flags; +} MetaStruct; + +static void set_meta(void *data, VmafMetadata *metadata) +{ + if (!data) return; + MetaStruct *meta = data; + char key[128], value[128]; + snprintf(key, sizeof(value), "%s_%d", metadata->feature_name, + metadata->picture_index); + snprintf(value, sizeof(value), "%f", metadata->score); + vmaf_dictionary_set(meta->metadata, key, value, meta->flags); +} + +static int g_callback_order[4] = {-1, -1, -1, -1}; +static int g_callback_count = 0; + +static void test_non_monotonic_callback(void *data, VmafMetadata *m) +{ + if (!data) return; + MetaStruct *meta = data; + + if (!strcmp("vmaf", m->feature_name)) + // Track callback order + g_callback_order[m->picture_index] = g_callback_count++; -void set_meta() {} + // Store in dictionary for verification + char key[32], value[32]; + snprintf(key, sizeof(key), "vmaf_%d", m->picture_index); + snprintf(value, sizeof(value), "%f", m->score); + vmaf_dictionary_set(meta->metadata, key, value, meta->flags); +} static char *test_propagate_metadata_init() { @@ -55,9 +97,8 @@ static char *test_propagate_metadata_append() int err = vmaf_metadata_init(&propagate_metadata); mu_assert("problem during vmaf_propagate_metadata_init", !err); - VmafMetadataConfiguration metadata_config; + VmafMetadataConfiguration metadata_config = {0}; metadata_config.callback = set_meta; - metadata_config.data = NULL; err = vmaf_metadata_append(propagate_metadata, metadata_config); mu_assert("problem during vmaf_propagate_metadata_append", !err); @@ -77,10 +118,372 @@ static char *test_propagate_metadata_append() return NULL; } +static char* test_propagate_metadata() +{ + int err; + + VmafDictionary *dict = NULL; + MetaStruct meta_data = { + .metadata = &dict, + .flags = 0, + }; + + VmafMetadataConfiguration m = { + .feature_name = "vmaf", + .callback = set_meta, + .data = &meta_data, + }; + + VmafFeatureCollector *feature_collector; + err = vmaf_feature_collector_init(&feature_collector); + mu_assert("problem during vmaf_feature_collector_init", !err); + + err = vmaf_feature_collector_register_metadata(feature_collector, m); + mu_assert("problem during vmaf_feature_collector_register_metadata_0", !err); + + VmafModel *model; + VmafModelConfig cfg = { + .name = "vmaf", + .flags = VMAF_MODEL_FLAGS_DEFAULT, + }; + err = vmaf_model_load(&model, &cfg, "vmaf_v0.6.1"); + mu_assert("problem during vmaf_model_load", !err); + err = vmaf_feature_collector_mount_model(feature_collector, model); + mu_assert("problem during vmaf_mount_model", !err); + + for (unsigned i = 0; i < model->n_features; i++) { + err = vmaf_feature_collector_append(feature_collector, + model->feature[i].name, 60., 0); + mu_assert("problem during vmaf_feature_collector_append", !err); + } + + VmafDictionaryEntry *e = vmaf_dictionary_get(&dict, "vmaf_0", 0); + mu_assert("error on propagaton metadata: propagated key not found!", + e); + mu_assert("error on propagaton metadata: propagated key wrong!", + !strcmp(e->key, "vmaf_0")); + mu_assert("error on propagaton metadata: propagated data wrong!", + !strcmp(e->val, "100.000000")); + + vmaf_feature_collector_destroy(feature_collector); + + m.data = NULL; + m.feature_name = "vmaf"; + err = vmaf_feature_collector_init(&feature_collector); + mu_assert("problem during vmaf_feature_collector_init", !err); + + err = vmaf_feature_collector_register_metadata(feature_collector, m); + mu_assert("problem during vmaf_feature_collector_register_metadata_1", !err); + + for (unsigned i = 0; i < model->n_features; i++) { + err = vmaf_feature_collector_append(feature_collector, + model->feature[i].name, 60., 0); + mu_assert("problem during vmaf_feature_collector_append", !err); + } + + vmaf_feature_collector_destroy(feature_collector); + + m.callback = NULL; + m.feature_name = "vmaf"; + err = vmaf_feature_collector_init(&feature_collector); + mu_assert("problem during vmaf_feature_collector_init", !err); + + err = vmaf_feature_collector_register_metadata(feature_collector, m); + mu_assert("problem during vmaf_feature_collector_register_metadata_2", err); + + vmaf_feature_collector_destroy(feature_collector); + + vmaf_model_destroy(model); + return NULL; + +} + +static char *test_propagate_metadata_non_monotonic() +{ + int err; + + // Reset global counters + g_callback_count = 0; + for (int i = 0; i < 4; i++) { + g_callback_order[i] = -1; + } + + // Setup dictionary to store callback results + VmafDictionary *dict = NULL; + MetaStruct meta_data = { + .metadata = &dict, + .flags = 0, + }; + + VmafMetadataConfiguration m = { + .feature_name = strdup("vmaf"), + .callback = test_non_monotonic_callback, + .data = &meta_data, + }; + + // Initialize feature collector + VmafFeatureCollector *feature_collector; + err = vmaf_feature_collector_init(&feature_collector); + mu_assert("problem during vmaf_feature_collector_init", !err); + + err = vmaf_feature_collector_register_metadata(feature_collector, m); + mu_assert("problem during vmaf_feature_collector_register_metadata", !err); + + // Load VMAF model + VmafModel *model; + VmafModelConfig cfg = { + .name = "vmaf", + .flags = VMAF_MODEL_FLAGS_DEFAULT, + }; + err = vmaf_model_load(&model, &cfg, "vmaf_v0.6.1"); + mu_assert("problem during vmaf_model_load", !err); + err = vmaf_feature_collector_mount_model(feature_collector, model); + mu_assert("problem during vmaf_mount_model", !err); + + // Simulate non-monotonic VMAF score computations + // Frame order: 3, 0, 2, 1 + for (unsigned i = 0; i < model->n_features; i++) { + // Frame 3 + err = vmaf_feature_collector_append(feature_collector, + model->feature[i].name, 60., 3); + mu_assert("problem appending frame 3", !err); + + // Frame 0 + err = vmaf_feature_collector_append(feature_collector, + model->feature[i].name, 70., 0); + mu_assert("problem appending frame 0", !err); + + // Frame 2 + err = vmaf_feature_collector_append(feature_collector, + model->feature[i].name, 80., 2); + mu_assert("problem appending frame 2", !err); + + // Frame 1 + err = vmaf_feature_collector_append(feature_collector, + model->feature[i].name, 90., 1); + mu_assert("problem appending frame 1", !err); + } + + // Verify callback order is monotonic regardless of computation order + mu_assert("Frame 0 callback not first", g_callback_order[0] == 0); + mu_assert("Frame 1 callback not second", g_callback_order[1] == 1); + mu_assert("Frame 2 callback not third", g_callback_order[2] == 2); + mu_assert("Frame 3 callback not fourth", g_callback_order[3] == 3); + + // Verify all frame scores were propagated + for (int i = 0; i < 4; i++) { + char key[32]; + snprintf(key, sizeof(key), "vmaf_%d", i); + VmafDictionaryEntry *e = vmaf_dictionary_get(&dict, key, 0); + mu_assert("Missing frame score in metadata", e != NULL); + } + + vmaf_feature_collector_destroy(feature_collector); + vmaf_model_destroy(model); + return NULL; +} + +// Structure to track callback invocations for multiple callbacks +typedef struct { + int callback_id; + int call_count; + VmafDictionary **metadata; +} MultiCallbackData; + +static void multi_callback(void *data, VmafMetadata *metadata) +{ + if (!data) return; + MultiCallbackData *cb_data = data; + + char key[128], value[128]; + snprintf(key, sizeof(key), "cb%d_%s_%d", + cb_data->callback_id, + metadata->feature_name, + metadata->picture_index); + snprintf(value, sizeof(value), "%f", metadata->score); + cb_data->call_count++; + vmaf_dictionary_set(cb_data->metadata, key, value, 0); +} + +static char *test_multiple_callbacks() +{ + int err; + VmafDictionary *dict1 = NULL; + VmafDictionary *dict2 = NULL; + + // Setup two different callback data structures + MultiCallbackData cb_data1 = { + .callback_id = 1, + .call_count = 0, + .metadata = &dict1 + }; + + MultiCallbackData cb_data2 = { + .callback_id = 2, + .call_count = 0, + .metadata = &dict2 + }; + + // Register two different callbacks + VmafMetadataConfiguration m1 = { + .feature_name = "vmaf", + .callback = multi_callback, + .data = &cb_data1, + }; + + VmafMetadataConfiguration m2 = { + .feature_name = "vmaf", + .callback = multi_callback, + .data = &cb_data2, + }; + + VmafFeatureCollector *feature_collector; + err = vmaf_feature_collector_init(&feature_collector); + mu_assert("problem during vmaf_feature_collector_init", !err); + + err = vmaf_feature_collector_register_metadata(feature_collector, m1); + mu_assert("problem registering first callback", !err); + + err = vmaf_feature_collector_register_metadata(feature_collector, m2); + mu_assert("problem registering second callback", !err); + + // Load and mount VMAF model + VmafModel *model; + VmafModelConfig cfg = { + .name = "vmaf", + .flags = VMAF_MODEL_FLAGS_DEFAULT, + }; + err = vmaf_model_load(&model, &cfg, "vmaf_v0.6.1"); + mu_assert("problem during vmaf_model_load", !err); + + err = vmaf_feature_collector_mount_model(feature_collector, model); + mu_assert("problem mounting model", !err); + + // Add some feature data + for (unsigned i = 0; i < model->n_features; i++) { + err = vmaf_feature_collector_append(feature_collector, + model->feature[i].name, 60., 0); + mu_assert("problem appending features", !err); + } + + // Verify both callbacks were called + mu_assert("first callback not called", cb_data1.call_count > 0); + mu_assert("second callback not called", cb_data2.call_count > 0); + mu_assert("callbacks called different number of times", + cb_data1.call_count == cb_data2.call_count); + + // Verify data in both dictionaries + VmafDictionaryEntry *e1 = vmaf_dictionary_get(&dict1, "cb1_vmaf_0", 0); + VmafDictionaryEntry *e2 = vmaf_dictionary_get(&dict2, "cb2_vmaf_0", 0); + + mu_assert("first callback data missing", e1 != NULL); + mu_assert("second callback data missing", e2 != NULL); + mu_assert("callback data mismatch", strcmp(e1->val, e2->val) == 0); + + vmaf_feature_collector_destroy(feature_collector); + vmaf_model_destroy(model); + return NULL; +} + +static char *test_multiple_callbacks_non_monotonic() +{ + int err; + VmafDictionary *dict1 = NULL; + VmafDictionary *dict2 = NULL; + + // Setup callback data + MultiCallbackData cb_data1 = { + .callback_id = 1, + .call_count = 0, + .metadata = &dict1 + }; + + MultiCallbackData cb_data2 = { + .callback_id = 2, + .call_count = 0, + .metadata = &dict2 + }; + + VmafMetadataConfiguration m1 = { + .feature_name = "vmaf", + .callback = multi_callback, + .data = &cb_data1, + }; + + VmafMetadataConfiguration m2 = { + .feature_name = "vmaf", + .callback = multi_callback, + .data = &cb_data2, + }; + + VmafFeatureCollector *feature_collector; + err = vmaf_feature_collector_init(&feature_collector); + mu_assert("problem during init", !err); + + err = vmaf_feature_collector_register_metadata(feature_collector, m1); + err |= vmaf_feature_collector_register_metadata(feature_collector, m2); + mu_assert("problem registering callbacks", !err); + + VmafModel *model; + VmafModelConfig cfg = { + .name = "vmaf", + .flags = VMAF_MODEL_FLAGS_DEFAULT, + }; + err = vmaf_model_load(&model, &cfg, "vmaf_v0.6.1"); + mu_assert("problem loading model", !err); + + err = vmaf_feature_collector_mount_model(feature_collector, model); + mu_assert("problem mounting model", !err); + + + for (unsigned i = 0; i < model->n_features; i++) { + // Frame 2 + err = vmaf_feature_collector_append(feature_collector, + model->feature[i].name, 60., 2); + mu_assert("problem appending frame 2", !err); + + // Frame 0 + err = vmaf_feature_collector_append(feature_collector, + model->feature[i].name, 60., 0); + mu_assert("problem appending frame 0", !err); + + // Frame 3 + err = vmaf_feature_collector_append(feature_collector, + model->feature[i].name, 60., 3); + mu_assert("problem appending frame 3", !err); + + // Frame 1 + err = vmaf_feature_collector_append(feature_collector, + model->feature[i].name, 60., 1); + mu_assert("problem appending frame 1", !err); + } + + for (int i = 0; i < 4; i++) { + char key1[32], key2[32]; + snprintf(key1, sizeof(key1), "cb1_vmaf_%d", i); + snprintf(key2, sizeof(key2), "cb2_vmaf_%d", i); + + VmafDictionaryEntry *e1 = vmaf_dictionary_get(&dict1, key1, 0); + VmafDictionaryEntry *e2 = vmaf_dictionary_get(&dict2, key2, 0); + + mu_assert("missing callback 1 data", e1 != NULL); + mu_assert("missing callback 2 data", e2 != NULL); + + mu_assert("callback data mismatch", strcmp(e1->val, e2->val) == 0); + } + vmaf_feature_collector_destroy(feature_collector); + vmaf_model_destroy(model); + return NULL; +} + char *run_tests() { mu_run_test(test_propagate_metadata_init); mu_run_test(test_propagate_metadata_destroy); mu_run_test(test_propagate_metadata_append); + mu_run_test(test_propagate_metadata); + mu_run_test(test_propagate_metadata_non_monotonic); + mu_run_test(test_multiple_callbacks); + mu_run_test(test_multiple_callbacks_non_monotonic); return NULL; -} +} \ No newline at end of file From 1246b726e4565f2e275573bbecae0669cb3fb652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yi=C4=9Fithan=20Yi=C4=9Fit?= Date: Sat, 1 Feb 2025 18:36:32 +0300 Subject: [PATCH 4/5] libvmaf: Implement non monotonic feature propagation. --- libvmaf/src/feature/feature_collector.c | 73 +++++---- libvmaf/src/feature/feature_collector.h | 2 + libvmaf/src/metadata_handler.c | 3 + libvmaf/src/metadata_handler.h | 1 + libvmaf/test/test_propagate_metadata.c | 196 ++++++++++++++++++++++-- 5 files changed, 237 insertions(+), 38 deletions(-) diff --git a/libvmaf/src/feature/feature_collector.c b/libvmaf/src/feature/feature_collector.c index 8e0105c22..e06c850e8 100644 --- a/libvmaf/src/feature/feature_collector.c +++ b/libvmaf/src/feature/feature_collector.c @@ -31,6 +31,8 @@ #include "log.h" #include "predict.h" +#define MAX(a, b) ((a) > (b) ? (a) : (b)) + static int aggregate_vector_init(AggregateVector *aggregate_vector) { if (!aggregate_vector) return -EINVAL; @@ -245,6 +247,8 @@ int vmaf_feature_collector_mount_model(VmafFeatureCollector *feature_collector, m->model = model; m->next = NULL; + m->last_highest_seen_index = 0; + m->last_lowest_seen_index = 0; VmafPredictModel *head = feature_collector->models; if (!head) { @@ -354,30 +358,12 @@ int vmaf_feature_collector_append(VmafFeatureCollector *feature_collector, int res = 0; - VmafCallbackItem *metadata_iter = feature_collector->metadata ? - feature_collector->metadata->head : NULL; - while (metadata_iter) { - // Check current feature name is the same as the metadata feature name - if (!strcmp(metadata_iter->metadata_cfg.feature_name, feature_name)) { - - // Call the callback function with the metadata feature name - VmafMetadata data = { - .feature_name = metadata_iter->metadata_cfg.feature_name, - .picture_index = picture_index, - .score = score, - }; - metadata_iter->metadata_cfg.callback(metadata_iter->metadata_cfg.data, &data); - // Move to the next metadata - goto next_metadata; - } - - VmafPredictModel *model_iter = feature_collector->models; + VmafPredictModel *model_iter = feature_collector->models; - // If metadata feature name is not the same as the current feature feature_name - // Check if metadata feature name is the predicted feature - while (model_iter) { - VmafModel *model = model_iter->model; + while (model_iter && feature_collector->metadata->cnt) { + VmafModel *model = model_iter->model; + if (strcmp(model->name, feature_name)) { pthread_mutex_unlock(&(feature_collector->lock)); res = vmaf_feature_collector_get_score(feature_collector, model->name, &score, picture_index); @@ -389,11 +375,46 @@ int vmaf_feature_collector_append(VmafFeatureCollector *feature_collector, picture_index, &score, true, true, 0); pthread_mutex_lock(&(feature_collector->lock)); } - model_iter = model_iter->next; - } + } else { + model_iter->last_highest_seen_index = MAX(model_iter->last_highest_seen_index, picture_index); + unsigned process_index = model_iter->last_lowest_seen_index; + double temp_score = 0.0; + bool frame_ready = true; - next_metadata: - metadata_iter = metadata_iter->next; + while (process_index <= model_iter->last_highest_seen_index && frame_ready) { + pthread_mutex_unlock(&(feature_collector->lock)); + res = vmaf_feature_collector_get_score(feature_collector, + model->name, &temp_score, process_index); + frame_ready = !res; + pthread_mutex_lock(&(feature_collector->lock)); + + VmafCallbackItem *metadata_iter = feature_collector->metadata->cnt ? + feature_collector->metadata->head : NULL; + + while (metadata_iter && frame_ready) { + VmafMetadata data = {0}; + for (unsigned i = 0; i < model->n_features; i++) { + data.feature_name = model->feature[i].name; + data.picture_index = process_index; + data.score = find_feature_vector(feature_collector, model->feature[i].name)->score[process_index].value; + + metadata_iter->metadata_cfg.callback(metadata_iter->metadata_cfg.data, &data); + } + data.feature_name = model->name; + data.picture_index = process_index; + data.score = temp_score; + metadata_iter->metadata_cfg.callback(metadata_iter->metadata_cfg.data, &data); + + metadata_iter = metadata_iter->next; + } + + if (frame_ready) { + process_index++; + model_iter->last_lowest_seen_index = process_index; + } + } + } + model_iter = model_iter->next; } unlock: diff --git a/libvmaf/src/feature/feature_collector.h b/libvmaf/src/feature/feature_collector.h index 83745bf8c..5f3d89cb4 100644 --- a/libvmaf/src/feature/feature_collector.h +++ b/libvmaf/src/feature/feature_collector.h @@ -46,6 +46,8 @@ typedef struct { typedef struct VmafPredictModel { VmafModel *model; + unsigned last_highest_seen_index; + unsigned last_lowest_seen_index; struct VmafPredictModel *next; } VmafPredictModel; diff --git a/libvmaf/src/metadata_handler.c b/libvmaf/src/metadata_handler.c index 99f69c212..9d231e05f 100644 --- a/libvmaf/src/metadata_handler.c +++ b/libvmaf/src/metadata_handler.c @@ -31,6 +31,7 @@ int vmaf_metadata_init(VmafCallbackList **const metadata) if (!metadata_s) goto fail; metadata_s->head = NULL; + metadata_s->cnt = 0; return 0; @@ -56,6 +57,8 @@ int vmaf_metadata_append(VmafCallbackList *metadata, const VmafMetadataConfigura iter->next = node; } + metadata->cnt++; + return 0; fail: diff --git a/libvmaf/src/metadata_handler.h b/libvmaf/src/metadata_handler.h index 1a21e17f6..12acde136 100644 --- a/libvmaf/src/metadata_handler.h +++ b/libvmaf/src/metadata_handler.h @@ -28,6 +28,7 @@ typedef struct VmafCallbackItem { typedef struct VmafCallbackList{ VmafCallbackItem *head; + unsigned cnt; } VmafCallbackList; int vmaf_metadata_init(VmafCallbackList **const metadata); diff --git a/libvmaf/test/test_propagate_metadata.c b/libvmaf/test/test_propagate_metadata.c index 85116084b..384b7405f 100644 --- a/libvmaf/test/test_propagate_metadata.c +++ b/libvmaf/test/test_propagate_metadata.c @@ -16,14 +16,13 @@ * */ +#include "dict.h" #include "test.h" #include "metadata_handler.h" #include "feature/feature_collector.h" -#include "predict.h" #include "predict.c" #include -#include #include #include #include @@ -202,13 +201,11 @@ static char *test_propagate_metadata_non_monotonic() { int err; - // Reset global counters g_callback_count = 0; for (int i = 0; i < 4; i++) { g_callback_order[i] = -1; } - // Setup dictionary to store callback results VmafDictionary *dict = NULL; MetaStruct meta_data = { .metadata = &dict, @@ -221,7 +218,6 @@ static char *test_propagate_metadata_non_monotonic() .data = &meta_data, }; - // Initialize feature collector VmafFeatureCollector *feature_collector; err = vmaf_feature_collector_init(&feature_collector); mu_assert("problem during vmaf_feature_collector_init", !err); @@ -229,7 +225,6 @@ static char *test_propagate_metadata_non_monotonic() err = vmaf_feature_collector_register_metadata(feature_collector, m); mu_assert("problem during vmaf_feature_collector_register_metadata", !err); - // Load VMAF model VmafModel *model; VmafModelConfig cfg = { .name = "vmaf", @@ -264,7 +259,6 @@ static char *test_propagate_metadata_non_monotonic() mu_assert("problem appending frame 1", !err); } - // Verify callback order is monotonic regardless of computation order mu_assert("Frame 0 callback not first", g_callback_order[0] == 0); mu_assert("Frame 1 callback not second", g_callback_order[1] == 1); mu_assert("Frame 2 callback not third", g_callback_order[2] == 2); @@ -283,7 +277,6 @@ static char *test_propagate_metadata_non_monotonic() return NULL; } -// Structure to track callback invocations for multiple callbacks typedef struct { int callback_id; int call_count; @@ -302,16 +295,16 @@ static void multi_callback(void *data, VmafMetadata *metadata) metadata->picture_index); snprintf(value, sizeof(value), "%f", metadata->score); cb_data->call_count++; - vmaf_dictionary_set(cb_data->metadata, key, value, 0); + vmaf_dictionary_set(cb_data->metadata, key, value, 1); } + static char *test_multiple_callbacks() { int err; VmafDictionary *dict1 = NULL; VmafDictionary *dict2 = NULL; - // Setup two different callback data structures MultiCallbackData cb_data1 = { .callback_id = 1, .call_count = 0, @@ -324,7 +317,6 @@ static char *test_multiple_callbacks() .metadata = &dict2 }; - // Register two different callbacks VmafMetadataConfiguration m1 = { .feature_name = "vmaf", .callback = multi_callback, @@ -476,6 +468,184 @@ static char *test_multiple_callbacks_non_monotonic() return NULL; } + +static char *test_multiple_models_metadata() +{ + int err; + VmafDictionary *dict = NULL; + + // Setup callback data + MultiCallbackData cb_data = { + .callback_id = 1, + .call_count = 0, + .metadata = &dict + }; + + VmafMetadataConfiguration m = { + .callback = multi_callback, + .data = &cb_data, + }; + + VmafFeatureCollector *feature_collector; + err = vmaf_feature_collector_init(&feature_collector); + mu_assert("problem during feature collector init", !err); + + err = vmaf_feature_collector_register_metadata(feature_collector, m); + mu_assert("problem registering metadata callback", !err); + + VmafModel *model1; + VmafModelConfig cfg1 = { + .name = "vmaf_0", + .flags = VMAF_MODEL_FLAGS_DEFAULT, + }; + err = vmaf_model_load(&model1, &cfg1, "vmaf_v0.6.1"); + mu_assert("problem loading first model", !err); + + err = vmaf_feature_collector_mount_model(feature_collector, model1); + mu_assert("problem mounting first model", !err); + + VmafModel *model2; + VmafModelConfig cfg2 = { + .name = "vmaf_1", + .flags = VMAF_MODEL_FLAGS_DEFAULT, + }; + err = vmaf_model_load(&model2, &cfg2, "vmaf_v0.6.1"); + mu_assert("problem loading second model", !err); + + err = vmaf_feature_collector_mount_model(feature_collector, model2); + mu_assert("problem mounting second model", !err); + + for (unsigned frame = 0; frame < 4; frame++) { + for (unsigned i = 0; i < model1->n_features; i++) { + err = vmaf_feature_collector_append(feature_collector, + model1->feature[i].name, + 60.0 + frame * 10.0, + frame); + mu_assert("problem appending features", !err); + } + } + + const char *model_names[] = {"vmaf_0", "vmaf_1"}; + + for (unsigned frame = 0; frame < 4; frame++) { + for (unsigned i = 0; i < 2; i++) { + char key[128]; + snprintf(key, sizeof(key), "cb1_%s_%d", model_names[i], frame); + VmafDictionaryEntry *e = vmaf_dictionary_get(&dict, key, 0); + mu_assert("missing model score callback", e != NULL); + } + } + + vmaf_feature_collector_destroy(feature_collector); + vmaf_model_destroy(model1); + vmaf_model_destroy(model2); + return NULL; +} + +static char *test_multiple_models_multiple_metadata_configs() +{ + int err; + VmafDictionary *dict1 = NULL; + VmafDictionary *dict2 = NULL; + VmafDictionary *dict3 = NULL; + + MultiCallbackData cb_data1 = { + .callback_id = 1, + .call_count = 0, + .metadata = &dict1 + }; + + MultiCallbackData cb_data2 = { + .callback_id = 2, + .call_count = 0, + .metadata = &dict2 + }; + + MultiCallbackData cb_data3 = { + .callback_id = 3, + .call_count = 0, + .metadata = &dict3 + }; + + VmafMetadataConfiguration m1 = { + .callback = multi_callback, + .data = &cb_data1, + }; + + VmafMetadataConfiguration m2 = { + .callback = multi_callback, + .data = &cb_data2, + }; + + VmafMetadataConfiguration m3 = { + .callback = multi_callback, + .data = &cb_data3, + }; + + VmafFeatureCollector *feature_collector; + err = vmaf_feature_collector_init(&feature_collector); + mu_assert("problem during init", !err); + + err = vmaf_feature_collector_register_metadata(feature_collector, m1); + mu_assert("problem registering first metadata config", !err); + + err = vmaf_feature_collector_register_metadata(feature_collector, m2); + mu_assert("problem registering second metadata config", !err); + + err = vmaf_feature_collector_register_metadata(feature_collector, m3); + mu_assert("problem registering third metadata config", !err); + + VmafModel *models[3]; + const char *names[] = {"vmaf-0", "vmaf-1", "vmaf-2"}; + + for (int i = 0; i < 3; i++) { + VmafModelConfig cfg = { + .name = names[i], + .flags = VMAF_MODEL_FLAGS_DEFAULT, + }; + err = vmaf_model_load(&models[i], &cfg, "vmaf_v0.6.1"); + mu_assert("problem loading model", !err); + + err = vmaf_feature_collector_mount_model(feature_collector, models[i]); + mu_assert("problem mounting model", !err); + } + + for (unsigned frame = 0; frame < 4; frame++) { + for (unsigned i = 0; i < models[0]->n_features; i++) { + err = vmaf_feature_collector_append(feature_collector, + models[0]->feature[i].name, + 60.0, + frame); + mu_assert("problem appending features", !err); + } + } + + VmafDictionary *dicts[] = {dict1, dict2, dict3}; + for (unsigned frame = 0; frame < 4; frame++) { + for (int model = 0; model < 3; model++) { + char key[128]; + snprintf(key, sizeof(key), "cb%d_%s_%d", model + 1, names[model], frame); + printf("key: %s %p %p\n", key, &dicts[model], dicts[model]); + VmafDictionaryEntry *e = vmaf_dictionary_get(&dicts[model], key, 0); + printf("e: %p\n", e); + mu_assert("missing metadata for model", e != NULL); + } + } + + mu_assert("callback count mismatch for first config", + cb_data1.call_count > 0); + mu_assert("callback count mismatch for second config", + cb_data2.call_count > 0); + mu_assert("callback count mismatch for third config", + cb_data3.call_count > 0); + + for (int i = 0; i < 3; i++) { + vmaf_model_destroy(models[i]); + } + vmaf_feature_collector_destroy(feature_collector); + return NULL; +} + char *run_tests() { mu_run_test(test_propagate_metadata_init); @@ -485,5 +655,7 @@ char *run_tests() mu_run_test(test_propagate_metadata_non_monotonic); mu_run_test(test_multiple_callbacks); mu_run_test(test_multiple_callbacks_non_monotonic); + mu_run_test(test_multiple_models_metadata); + mu_run_test(test_multiple_models_multiple_metadata_configs); return NULL; -} \ No newline at end of file +} From a696d7cb22a54568c97c9f0562b6c34142a89acf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yi=C4=9Fithan=20Yi=C4=9Fit?= Date: Sat, 1 Feb 2025 22:48:33 +0300 Subject: [PATCH 5/5] refactor: remove feature name from VmafMetadataConfiguration --- libvmaf/include/libvmaf/libvmaf.h | 1 - libvmaf/src/feature/feature_collector.c | 1 - libvmaf/test/test_propagate_metadata.c | 8 -------- 3 files changed, 10 deletions(-) diff --git a/libvmaf/include/libvmaf/libvmaf.h b/libvmaf/include/libvmaf/libvmaf.h index 11d0d1190..6e489f8e3 100644 --- a/libvmaf/include/libvmaf/libvmaf.h +++ b/libvmaf/include/libvmaf/libvmaf.h @@ -280,7 +280,6 @@ typedef struct VmafMetadata { * @param data User data to pass to the callback. */ typedef struct VmafMetadataConfiguration { - char *feature_name; void (*callback)(void *data, VmafMetadata *metadata); void *data; } VmafMetadataConfiguration; diff --git a/libvmaf/src/feature/feature_collector.c b/libvmaf/src/feature/feature_collector.c index e06c850e8..4b0efdcc2 100644 --- a/libvmaf/src/feature/feature_collector.c +++ b/libvmaf/src/feature/feature_collector.c @@ -291,7 +291,6 @@ int vmaf_feature_collector_register_metadata(VmafFeatureCollector *feature_colle VmafMetadataConfiguration metadata_cfg) { if (!feature_collector) return -EINVAL; - if (!metadata_cfg.feature_name) return -EINVAL; if (!metadata_cfg.callback) return -EINVAL; VmafCallbackList *metadata = feature_collector->metadata; diff --git a/libvmaf/test/test_propagate_metadata.c b/libvmaf/test/test_propagate_metadata.c index 384b7405f..d81fdf168 100644 --- a/libvmaf/test/test_propagate_metadata.c +++ b/libvmaf/test/test_propagate_metadata.c @@ -128,7 +128,6 @@ static char* test_propagate_metadata() }; VmafMetadataConfiguration m = { - .feature_name = "vmaf", .callback = set_meta, .data = &meta_data, }; @@ -167,7 +166,6 @@ static char* test_propagate_metadata() vmaf_feature_collector_destroy(feature_collector); m.data = NULL; - m.feature_name = "vmaf"; err = vmaf_feature_collector_init(&feature_collector); mu_assert("problem during vmaf_feature_collector_init", !err); @@ -183,7 +181,6 @@ static char* test_propagate_metadata() vmaf_feature_collector_destroy(feature_collector); m.callback = NULL; - m.feature_name = "vmaf"; err = vmaf_feature_collector_init(&feature_collector); mu_assert("problem during vmaf_feature_collector_init", !err); @@ -213,7 +210,6 @@ static char *test_propagate_metadata_non_monotonic() }; VmafMetadataConfiguration m = { - .feature_name = strdup("vmaf"), .callback = test_non_monotonic_callback, .data = &meta_data, }; @@ -318,13 +314,11 @@ static char *test_multiple_callbacks() }; VmafMetadataConfiguration m1 = { - .feature_name = "vmaf", .callback = multi_callback, .data = &cb_data1, }; VmafMetadataConfiguration m2 = { - .feature_name = "vmaf", .callback = multi_callback, .data = &cb_data2, }; @@ -397,13 +391,11 @@ static char *test_multiple_callbacks_non_monotonic() }; VmafMetadataConfiguration m1 = { - .feature_name = "vmaf", .callback = multi_callback, .data = &cb_data1, }; VmafMetadataConfiguration m2 = { - .feature_name = "vmaf", .callback = multi_callback, .data = &cb_data2, };