Skip to content

Commit

Permalink
fix(SUP-1444): bug in annotation export json file (#1658)
Browse files Browse the repository at this point in the history 
Co-authored-by: Luca Marongiu <[email protected]>
  • Loading branch information
@AyloSrd
AyloSrd and Luca Marongiu authored Mar 11, 2024
1 parent 32a8d4d commit 7097402
Show file tree
Hide file tree
Showing 7 changed files with 636 additions and 17 deletions.
95 changes: 80 additions & 15 deletions src/kili/adapters/kili_api_gateway/label/annotation_to_json_response.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,13 @@ def _classic_annotations_to_json_response(
def _key_annotations_iterator(
annotation: VideoTranscriptionAnnotation,
) -> Generator[
Tuple[VideoTranscriptionKeyAnnotation, int, int, Optional[VideoTranscriptionKeyAnnotation]],
Tuple[
VideoTranscriptionKeyAnnotation,
int,
int,
Optional[VideoTranscriptionKeyAnnotation],
Optional[int],
],
None,
None,
]:
Expand All @@ -220,7 +226,13 @@ def _key_annotations_iterator(
def _key_annotations_iterator(
annotation: VideoClassificationAnnotation,
) -> Generator[
Tuple[VideoClassificationKeyAnnotation, int, int, Optional[VideoClassificationKeyAnnotation]],
Tuple[
VideoClassificationKeyAnnotation,
int,
int,
Optional[VideoClassificationKeyAnnotation],
Optional[int],
],
None,
None,
]:
Expand All @@ -231,7 +243,13 @@ def _key_annotations_iterator(
def _key_annotations_iterator(
annotation: VideoObjectDetectionAnnotation,
) -> Generator[
Tuple[VideoObjectDetectionKeyAnnotation, int, int, Optional[VideoObjectDetectionKeyAnnotation]],
Tuple[
VideoObjectDetectionKeyAnnotation,
int,
int,
Optional[VideoObjectDetectionKeyAnnotation],
Optional[int],
],
None,
None,
]:
Expand All @@ -246,19 +264,38 @@ def _key_annotations_iterator(annotation: VideoAnnotation) -> Generator:
sorted_key_annotations = sorted(
annotation["keyAnnotations"], key=lambda key_ann: int(key_ann["frame"])
)

# previous_key_ann is used to keep track of the previous key annotation
# in case where keyframe is not present in the current frame range
previous_key_ann = {}
previous_key_ann_index = 0
# iterate over the frame ranges of the annotation
for frame_interval in annotation["frames"]:
frame_range = range(frame_interval["start"], frame_interval["end"] + 1)
# has_key_annotation is used to keep track of whether the current frame range
# has a key annotation or not. It could be that the current frame range does not
# have a key annotation, or that the first keyframe is after the start of the frame range
has_key_annotation = False
for key_ann_index, key_ann in enumerate(sorted_key_annotations):
# skip the key annotation if the key annotation start frame
# is not in current frame range
if key_ann["frame"] not in frame_range:
continue

if key_ann["frame"] > frame_interval["start"] and not has_key_annotation:
# if the key annotation start frame is after the start of the frame range,
# then we need to yield the previous key annotation
key_ann_frame = previous_key_ann["frame"]
yield (
previous_key_ann,
frame_interval["start"],
key_ann["frame"],
key_ann,
key_ann_frame,
)
# compute the key annotation frame range
# the start frame of key annotation is given, but not the end frame
key_ann_start = key_ann["frame"]
key_ann_frame = key_ann["frame"]
key_ann_end = min(
frame_interval["end"] + 1,
sorted_key_annotations[key_ann_index + 1]["frame"]
Expand All @@ -273,7 +310,26 @@ def _key_annotations_iterator(annotation: VideoAnnotation) -> Generator:
else None
)

yield key_ann, key_ann_start, key_ann_end, next_key_ann
has_key_annotation = True
previous_key_ann = key_ann
previous_key_ann_index = key_ann_index

yield key_ann, key_ann_start, key_ann_end, next_key_ann, key_ann_frame

if not has_key_annotation:
key_ann_frame = previous_key_ann["frame"]
next_key_ann = (
sorted_key_annotations[previous_key_ann_index + 1]
if previous_key_ann_index + 1 < len(sorted_key_annotations)
else None
)
yield (
previous_key_ann,
frame_interval["start"],
frame_interval["end"] + 1,
next_key_ann,
key_ann_frame,
)


def _ranking_annotation_to_json_response(
Expand Down Expand Up @@ -319,10 +375,12 @@ def _video_transcription_annotation_to_json_response(
"""
json_resp: Dict[str, Dict[JobName, Dict]] = defaultdict(dict)

for key_ann, key_ann_start, key_ann_end, _ in _key_annotations_iterator(annotation):
for key_ann, key_ann_start, key_ann_end, _, key_ann_frame in _key_annotations_iterator(
annotation
):
for frame_id in range(key_ann_start, key_ann_end):
json_resp[str(frame_id)][annotation["job"]] = {
"isKeyFrame": frame_id == key_ann_start,
"isKeyFrame": frame_id == key_ann_frame,
"text": key_ann["annotationValue"]["text"],
}

Expand Down Expand Up @@ -406,12 +464,14 @@ def _video_classification_annotation_to_json_response(

json_resp: Dict[str, Dict[JobName, Dict]] = defaultdict(dict)

for key_ann, key_ann_start, key_ann_end, _ in _key_annotations_iterator(annotation):
for key_ann, key_ann_start, key_ann_end, _, key_ann_frame in _key_annotations_iterator(
annotation
):
for frame_id in range(key_ann_start, key_ann_end):
# initialize the frame json response
json_resp[str(frame_id)][annotation["job"]] = {
"categories": [],
"isKeyFrame": frame_id == key_ann_start,
"isKeyFrame": frame_id == key_ann_frame,
}

# get the frame json response of child jobs
Expand Down Expand Up @@ -461,22 +521,27 @@ def _video_object_detection_annotation_to_json_response(

json_resp = defaultdict(dict)

for key_ann, key_ann_start, key_ann_end, next_key_ann in _key_annotations_iterator(annotation):
for (
key_ann,
key_ann_start,
key_ann_end,
next_key_ann,
key_ann_frame,
) in _key_annotations_iterator(annotation):
for frame_id in range(key_ann_start, key_ann_end):
# get the frame json response of child jobs
child_jobs_frame_json_resp = json_resp_child_jobs.get(str(frame_id), {})

annotation_dict = {
"children": child_jobs_frame_json_resp,
"isKeyFrame": frame_id == key_ann_start,
"isKeyFrame": frame_id == key_ann_frame,
"categories": [{"name": annotation["category"]}],
"mid": annotation["mid"],
"type": json_interface["jobs"][annotation["job"]]["tools"][0],
}

if frame_id == key_ann_start or next_key_ann is None:
if frame_id == key_ann_frame or next_key_ann is None:
norm_vertices = key_ann["annotationValue"]["vertices"]

# between two key frame annotations, an object (point, bbox, polygon) is
# interpolated in the UI
else:
Expand All @@ -485,9 +550,9 @@ def _video_object_detection_annotation_to_json_response(
norm_vertices = _interpolate_object(
object_type=json_interface["jobs"][annotation["job"]]["tools"][0],
object_initial_state=object_inital_state,
initial_state_frame_index=key_ann_start,
initial_state_frame_index=key_ann["frame"],
object_final_state=object_final_state,
final_state_frame_index=key_ann_end,
final_state_frame_index=next_key_ann["frame"],
at_frame=frame_id,
)

Expand Down
3 changes: 2 additions & 1 deletion src/kili/core/graphql/graphql_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
from gql.transport.requests import RequestsHTTPTransport
from gql.transport.requests import log as gql_requests_logger
from graphql import DocumentNode, print_schema
from pyrate_limiter import Duration, Limiter, Rate
from pyrate_limiter import Duration, Rate
from pyrate_limiter.limiter import Limiter
from tenacity import (
retry,
retry_all,
Expand Down
127 changes: 127 additions & 0 deletions tests/unit/adapters/kili_api_gateway/label/test_data/test_case_15.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
"""Mocks for split video export."""

from collections import defaultdict

json_interface = {
"jobs": {
"JOB_0": {
"content": {
"categories": {
"OBJECT_A": {"children": ["JOB_1"], "name": "Train", "color": "#733AFB"}
},
"input": "radio",
},
"instruction": "Track objects A and B",
"isChild": False,
"tools": ["rectangle"],
"mlTask": "OBJECT_DETECTION",
"models": {"tracking": {}},
"isVisible": True,
"required": 0,
},
"JOB_1": {
"content": {
"categories": {
"IS_THE OBJECT OCCLUDED?": {"children": [], "name": "Is the object occluded?"}
},
"input": "checkbox",
},
"instruction": "",
"isChild": True,
"mlTask": "CLASSIFICATION",
"models": {},
"isVisible": True,
"required": 0,
},
}
}
fake_bbox_norm_vertices = [
{"x": 0.35, "y": 0.54},
{"x": 0.35, "y": 0.46},
{"x": 0.40, "y": 0.46},
{"x": 0.40, "y": 0.54},
]

annotations = [
{
"__typename": "VideoObjectDetectionAnnotation",
"id": "55ad2c18-c063-4064-9f46-2e3fa3eed483",
"job": "JOB_0",
"path": [],
"labelId": "cltftiw7n000qslwu4o1d9rj3",
"frames": [{"start": 0, "end": 1}, {"start": 4, "end": 5}],
"keyAnnotations": [
{
"id": "55ad2c18-c063-4064-9f46-2e3fa3eed483-0",
"frame": 0,
"annotationValue": {"vertices": [[fake_bbox_norm_vertices]]},
}
],
"name": "Train 2",
"mid": "20240306141046672-1",
"category": "OBJECT_A",
}
]

expected_json_resp = {
"0": {
"ANNOTATION_JOB_COUNTER": {"JOB_0": defaultdict(int, {"OBJECT_A": 1})},
"ANNOTATION_NAMES_JOB": {"20240306141046672-1": "Train 2"},
"JOB_0": {
"annotations": [
{
"children": {},
"isKeyFrame": True,
"categories": [{"name": "OBJECT_A"}],
"mid": "20240306141046672-1",
"type": "rectangle",
"boundingPoly": [{"normalizedVertices": fake_bbox_norm_vertices}],
}
],
},
},
"1": {
"JOB_0": {
"annotations": [
{
"children": {},
"isKeyFrame": False,
"categories": [{"name": "OBJECT_A"}],
"mid": "20240306141046672-1",
"type": "rectangle",
"boundingPoly": [{"normalizedVertices": fake_bbox_norm_vertices}],
}
]
}
},
"2": {},
"3": {},
"4": {
"JOB_0": {
"annotations": [
{
"children": {},
"isKeyFrame": False,
"categories": [{"name": "OBJECT_A"}],
"mid": "20240306141046672-1",
"type": "rectangle",
"boundingPoly": [{"normalizedVertices": fake_bbox_norm_vertices}],
}
]
}
},
"5": {
"JOB_0": {
"annotations": [
{
"children": {},
"isKeyFrame": False,
"categories": [{"name": "OBJECT_A"}],
"mid": "20240306141046672-1",
"type": "rectangle",
"boundingPoly": [{"normalizedVertices": fake_bbox_norm_vertices}],
}
]
}
},
}
Loading

0 comments on commit 7097402

Please sign in to comment.