Change match_empty_frames to empty_is_annotated

changelog.d/20241229_221630_mzhiltso_empty_is_annotated.md

@@ -0,0 +1,4 @@
+### Changed
+
+- The `match_empty_frames` quality setting is renamed to `empty_is_annotated`
+  (<https://github.com/cvat-ai/cvat/pull/8888>)

cvat-core/src/quality-settings.ts

@@ -38,7 +38,7 @@ export default class QualitySettings {
     #objectVisibilityThreshold: number;
     #panopticComparison: boolean;
     #compareAttributes: boolean;
-    #matchEmptyFrames: boolean;
+    #emptyIsAnnotated: boolean;
     #descriptions: Record<string, string>;
 
     constructor(initialData: SerializedQualitySettingsData) {
@@ -60,7 +60,7 @@ export default class QualitySettings {
         this.#objectVisibilityThreshold = initialData.object_visibility_threshold;
         this.#panopticComparison = initialData.panoptic_comparison;
         this.#compareAttributes = initialData.compare_attributes;
-        this.#matchEmptyFrames = initialData.match_empty_frames;
+        this.#emptyIsAnnotated = initialData.empty_is_annotated;
         this.#descriptions = initialData.descriptions;
     }
 
@@ -200,12 +200,12 @@ export default class QualitySettings {
         this.#maxValidationsPerJob = newVal;
     }
 
-    get matchEmptyFrames(): boolean {
-        return this.#matchEmptyFrames;
+    get emptyIsAnnotated(): boolean {
+        return this.#emptyIsAnnotated;
     }
 
-    set matchEmptyFrames(newVal: boolean) {
-        this.#matchEmptyFrames = newVal;
+    set emptyIsAnnotated(newVal: boolean) {
+        this.#emptyIsAnnotated = newVal;
     }
 
     get descriptions(): Record<string, string> {
@@ -236,7 +236,7 @@ export default class QualitySettings {
             target_metric: this.#targetMetric,
             target_metric_threshold: this.#targetMetricThreshold,
             max_validations_per_job: this.#maxValidationsPerJob,
-            match_empty_frames: this.#matchEmptyFrames,
+            empty_is_annotated: this.#emptyIsAnnotated,
         };
 
         return result;

cvat-core/src/server-response-types.ts

@@ -258,7 +258,7 @@ export interface SerializedQualitySettingsData {
     object_visibility_threshold?: number;
     panoptic_comparison?: boolean;
     compare_attributes?: boolean;
-    match_empty_frames?: boolean;
+    empty_is_annotated?: boolean;
     descriptions?: Record<string, string>;
 }
 

cvat-ui/src/components/quality-control/quality-control-page.tsx

@@ -223,7 +223,7 @@ function QualityControlPage(): JSX.Element {
                 settings.lowOverlapThreshold = values.lowOverlapThreshold / 100;
                 settings.iouThreshold = values.iouThreshold / 100;
                 settings.compareAttributes = values.compareAttributes;
-                settings.matchEmptyFrames = values.matchEmptyFrames;
+                settings.emptyIsAnnotated = values.emptyIsAnnotated;
 
                 settings.oksSigma = values.oksSigma / 100;
                 settings.pointSizeBase = values.pointSizeBase;

cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx

@@ -34,7 +34,7 @@ export default function QualitySettingsForm(props: Readonly<Props>): JSX.Element
         lowOverlapThreshold: settings.lowOverlapThreshold * 100,
         iouThreshold: settings.iouThreshold * 100,
         compareAttributes: settings.compareAttributes,
-        matchEmptyFrames: settings.matchEmptyFrames,
+        emptyIsAnnotated: settings.emptyIsAnnotated,
 
         oksSigma: settings.oksSigma * 100,
         pointSizeBase: settings.pointSizeBase,
@@ -81,7 +81,7 @@ export default function QualitySettingsForm(props: Readonly<Props>): JSX.Element
             {makeTooltipFragment('Target metric', targetMetricDescription)}
             {makeTooltipFragment('Target metric threshold', settings.descriptions.targetMetricThreshold)}
             {makeTooltipFragment('Compare attributes', settings.descriptions.compareAttributes)}
-            {makeTooltipFragment('Match empty frames', settings.descriptions.matchEmptyFrames)}
+            {makeTooltipFragment('Empty frames are annotated', settings.descriptions.emptyIsAnnotated)}
         </>,
     );
 
@@ -198,12 +198,12 @@ export default function QualitySettingsForm(props: Readonly<Props>): JSX.Element
                 </Col>
                 <Col span={12}>
                     <Form.Item
-                        name='matchEmptyFrames'
+                        name='emptyIsAnnotated'
                         valuePropName='checked'
                         rules={[{ required: true }]}
                     >
                         <Checkbox>
-                            <Text className='cvat-text-color'>Match empty frames</Text>
+                            <Text className='cvat-text-color'>Empty frames are annotated</Text>
                         </Checkbox>
                     </Form.Item>
                 </Col>

cvat/apps/quality_control/migrations/0006_rename_match_empty_frames_qualitysettings_empty_is_annotated.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.15 on 2024-12-29 19:08
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("quality_control", "0005_qualitysettings_match_empty"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="qualitysettings",
+            old_name="match_empty_frames",
+            new_name="empty_is_annotated",
+        ),
+    ]

cvat/apps/quality_control/models.py

@@ -235,7 +235,7 @@ class QualitySettings(models.Model):
 
     compare_attributes = models.BooleanField()
 
-    match_empty_frames = models.BooleanField(default=False)
+    empty_is_annotated = models.BooleanField(default=False)
 
     target_metric = models.CharField(
         max_length=32,

cvat/apps/quality_control/quality_reports.py

@@ -215,10 +215,11 @@ class ComparisonParameters(_Serializable):
     panoptic_comparison: bool = True
     "Use only the visible part of the masks and polygons in comparisons"
 
-    match_empty_frames: bool = False
+    empty_is_annotated: bool = False
     """
-    Consider unannotated (empty) frames as matching. If disabled, quality metrics, such as accuracy,
-    will be 0 if both GT and DS frames have no annotations. When enabled, they will be 1 instead.
+    Consider unannotated (empty) frames virtually annotated as "nothing".
+    If disabled, quality metrics, such as accuracy, will be 0 if both GT and DS frames
+    have no annotations. When enabled, they will be 1 instead.
     This will also add virtual annotations to empty frames in the comparison results.
     """
 
@@ -1977,15 +1978,22 @@ def _find_closest_unmatched_shape(shape: dm.Annotation):
             gt_label_idx = label_id_map[gt_ann.label] if gt_ann else self._UNMATCHED_IDX
             confusion_matrix[ds_label_idx, gt_label_idx] += 1
 
-        if self.settings.match_empty_frames and not gt_item.annotations and not ds_item.annotations:
+        if self.settings.empty_is_annotated and (
+            not gt_item.annotations or not ds_item.annotations
+        ):
             # Add virtual annotations for empty frames
-            valid_labels_count = 1
-            total_labels_count = 1
+            if not gt_item.annotations and not ds_item.annotations:
+                valid_labels_count = 1
+                total_labels_count = 1
+
+                valid_shapes_count = 1
+                total_shapes_count = 1
 
-            valid_shapes_count = 1
-            total_shapes_count = 1
-            ds_shapes_count = 1
-            gt_shapes_count = 1
+            if not ds_item.annotations:
+                ds_shapes_count = 1
+
+            if not gt_item.annotations:
+                gt_shapes_count = 1
 
         self._frame_results[frame_id] = ComparisonReportFrameSummary(
             annotations=self._generate_frame_annotations_summary(
@@ -2078,12 +2086,17 @@ def _generate_frame_annotations_summary(
     ) -> ComparisonReportAnnotationsSummary:
         summary = self._compute_annotations_summary(confusion_matrix, confusion_matrix_labels)
 
-        if self.settings.match_empty_frames and summary.total_count == 0:
+        if self.settings.empty_is_annotated and (not summary.ds_count or not summary.gt_count):
             # Add virtual annotations for empty frames
-            summary.valid_count = 1
-            summary.total_count = 1
-            summary.ds_count = 1
-            summary.gt_count = 1
+            if not summary.total_count:
+                summary.valid_count = 1
+                summary.total_count = 1
+
+            if not summary.ds_count:
+                summary.ds_count = 1
+
+            if not summary.gt_count:
+                summary.gt_count = 1
 
         return summary
 
@@ -2108,14 +2121,26 @@ def _generate_dataset_annotations_summary(
             ),
         )
         mean_ious = []
-        empty_frame_count = 0
+        empty_gt_frames = set()
+        empty_ds_frames = set()
         confusion_matrix_labels, confusion_matrix, _ = self._make_zero_confusion_matrix()
 
-        for frame_result in frame_summaries.values():
+        for frame_id, frame_result in frame_summaries.items():
             confusion_matrix += frame_result.annotations.confusion_matrix.rows
 
-            if not np.any(frame_result.annotations.confusion_matrix.rows):
-                empty_frame_count += 1
+            if self.settings.empty_is_annotated and not np.any(
+                frame_result.annotations.confusion_matrix.rows[
+                    np.triu_indices_from(frame_result.annotations.confusion_matrix.rows)
+                ]
+            ):
+                empty_ds_frames.add(frame_id)
+
+            if self.settings.empty_is_annotated and not np.any(
+                frame_result.annotations.confusion_matrix.rows[
+                    np.tril_indices_from(frame_result.annotations.confusion_matrix.rows)
+                ]
+            ):
+                empty_gt_frames.add(frame_id)
 
             if annotation_components is None:
                 annotation_components = deepcopy(frame_result.annotation_components)
@@ -2128,13 +2153,13 @@ def _generate_dataset_annotations_summary(
             confusion_matrix, confusion_matrix_labels
         )
 
-        if self.settings.match_empty_frames and empty_frame_count:
+        if self.settings.empty_is_annotated:
             # Add virtual annotations for empty frames,
             # they are not included in the confusion matrix
-            annotation_summary.valid_count += empty_frame_count
-            annotation_summary.total_count += empty_frame_count
-            annotation_summary.ds_count += empty_frame_count
-            annotation_summary.gt_count += empty_frame_count
+            annotation_summary.valid_count += len(empty_ds_frames & empty_gt_frames)
+            annotation_summary.total_count += len(empty_ds_frames | empty_gt_frames)
+            annotation_summary.ds_count += len(empty_ds_frames)
+            annotation_summary.gt_count += len(empty_gt_frames)
 
         # Cannot be computed in accumulate()
         annotation_components.shape.mean_iou = np.mean(mean_ious)

cvat/apps/quality_control/serializers.py

@@ -92,15 +92,15 @@ class Meta:
             "object_visibility_threshold",
             "panoptic_comparison",
             "compare_attributes",
-            "match_empty_frames",
+            "empty_is_annotated",
         )
         read_only_fields = (
             "id",
             "task_id",
         )
 
         extra_kwargs = {k: {"required": False} for k in fields}
-        extra_kwargs.setdefault("match_empty_frames", {}).setdefault("default", False)
+        extra_kwargs.setdefault("empty_is_annotated", {}).setdefault("default", False)
 
         for field_name, help_text in {
             "target_metric": "The primary metric used for quality estimation",
@@ -166,9 +166,9 @@ class Meta:
                 Use only the visible part of the masks and polygons in comparisons
             """,
             "compare_attributes": "Enables or disables annotation attribute comparison",
-            "match_empty_frames": """
-                Count empty frames as matching. This affects target metrics like accuracy in cases
-                there are no annotations. If disabled, frames without annotations
+            "empty_is_annotated": """
+                Consider empty frames annotated as "empty". This affects target metrics like
+                accuracy in cases there are no annotations. If disabled, frames without annotations
                 are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead.
                 This will also add virtual annotations to empty frames in the comparison results.
             """,

cvat/schema.yml

@@ -9775,12 +9775,12 @@ components:
         compare_attributes:
           type: boolean
           description: Enables or disables annotation attribute comparison
-        match_empty_frames:
+        empty_is_annotated:
           type: boolean
           default: false
           description: |
-            Count empty frames as matching. This affects target metrics like accuracy in cases
-            there are no annotations. If disabled, frames without annotations
+            Consider empty frames annotated as "empty". This affects target metrics like
+            accuracy in cases there are no annotations. If disabled, frames without annotations
             are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead.
             This will also add virtual annotations to empty frames in the comparison results.
     PatchedTaskValidationLayoutWriteRequest:
@@ -10282,12 +10282,12 @@ components:
         compare_attributes:
           type: boolean
           description: Enables or disables annotation attribute comparison
-        match_empty_frames:
+        empty_is_annotated:
           type: boolean
           default: false
           description: |
-            Count empty frames as matching. This affects target metrics like accuracy in cases
-            there are no annotations. If disabled, frames without annotations
+            Consider empty frames annotated as "empty". This affects target metrics like
+            accuracy in cases there are no annotations. If disabled, frames without annotations
             are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead.
             This will also add virtual annotations to empty frames in the comparison results.
     RegisterSerializerEx:

site/content/en/docs/manual/advanced/analytics-and-monitoring/auto-qa.md

@@ -385,7 +385,7 @@ Annotation quality settings have the following parameters:
 | - | - |
 | Min overlap threshold | Min overlap threshold used for the distinction between matched and unmatched shapes. Used to match all types of annotations. It corresponds to the Intersection over union (IoU) for spatial annotations, such as bounding boxes and masks. |
 | Low overlap threshold | Low overlap threshold used for the distinction between strong and weak matches. Only affects _Low overlap_ warnings. It's supposed that _Min similarity threshold_ <= _Low overlap threshold_. |
-| Match empty frames | Consider frames matched if there are no annotations both on GT and regular job frames |
+| Empty frames are annotated | Consider frames annotated as "empty" if there are no annotations on a frame. If a frame is empty in both GT and job annotations, it will be considered a matching annotation. |
 
 | _Point and Skeleton matching_ | |
 | - | - |

tests/python/rest_api/test_quality_control.py

@@ -1213,7 +1213,7 @@ def test_modified_task_produces_different_metrics(
             "compare_line_orientation",
             "panoptic_comparison",
             "point_size_base",
-            "match_empty_frames",
+            "empty_is_annotated",
         ],
     )
     def test_settings_affect_metrics(
@@ -1246,8 +1246,11 @@ def test_settings_affect_metrics(
             )
 
         new_report = self.create_quality_report(admin_user, task_id)
-        if parameter == "match_empty_frames":
+        if parameter == "empty_is_annotated":
             assert new_report["summary"]["valid_count"] != old_report["summary"]["valid_count"]
+            assert new_report["summary"]["total_count"] != old_report["summary"]["total_count"]
+            assert new_report["summary"]["ds_count"] != old_report["summary"]["ds_count"]
+            assert new_report["summary"]["gt_count"] != old_report["summary"]["gt_count"]
         else:
             assert (
                 new_report["summary"]["conflict_count"] != old_report["summary"]["conflict_count"]