From c53e0620e0edcce1c23d2f2ed69747d97fa974f7 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 22 Jun 2021 10:13:01 +0300 Subject: [PATCH 01/63] init --- .vscode/settings.json | 12 +- cvat/apps/dataset_manager/annotation.py | 3 + cvat/apps/dataset_manager/bindings.py | 732 +++++++++++++----- cvat/apps/dataset_manager/formats/camvid.py | 14 +- cvat/apps/dataset_manager/formats/coco.py | 14 +- cvat/apps/dataset_manager/formats/icdar.py | 32 +- cvat/apps/dataset_manager/formats/imagenet.py | 12 +- cvat/apps/dataset_manager/formats/labelme.py | 12 +- .../dataset_manager/formats/market1501.py | 12 +- cvat/apps/dataset_manager/formats/mask.py | 14 +- .../dataset_manager/formats/pascal_voc.py | 19 +- cvat/apps/dataset_manager/formats/tfrecord.py | 12 +- cvat/apps/dataset_manager/formats/vggface2.py | 12 +- .../apps/dataset_manager/formats/widerface.py | 12 +- cvat/apps/dataset_manager/formats/yolo.py | 8 +- cvat/apps/dataset_manager/project.py | 71 ++ cvat/apps/dataset_manager/views.py | 62 +- cvat/apps/engine/views.py | 89 ++- 18 files changed, 845 insertions(+), 297 deletions(-) create mode 100644 cvat/apps/dataset_manager/project.py diff --git a/.vscode/settings.json b/.vscode/settings.json index cb78ca045e6..ce5a350e8f3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,4 @@ { - "eslint.enable": true, "eslint.probe": [ "javascript", "typescript", @@ -24,5 +23,14 @@ "python.linting.pycodestyleEnabled": false, "licenser.license": "Custom", "licenser.customHeader": "Copyright (C) @YEAR@ Intel Corporation\n\nSPDX-License-Identifier: MIT", - "files.trimTrailingWhitespace": true + "files.trimTrailingWhitespace": true, + "python.pythonPath": ".env/bin/python", + "sqltools.connections": [ + { + "previewLimit": 50, + "driver": "SQLite", + "name": "cvat", + "database": "${workspaceFolder:cvat}/db.sqlite3" + } + ] } diff --git a/cvat/apps/dataset_manager/annotation.py b/cvat/apps/dataset_manager/annotation.py index 95c127c6085..741334879db 100644 --- a/cvat/apps/dataset_manager/annotation.py +++ b/cvat/apps/dataset_manager/annotation.py @@ -42,6 +42,9 @@ def data(self): def __getitem__(self, key): return getattr(self, key) + def __setitem__(self, key, value): + return setattr(self, key, value) + @data.setter def data(self, data): self.version = data['version'] diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index b800e18ca3c..786e12c837a 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -4,49 +4,32 @@ # SPDX-License-Identifier: MIT import os.path as osp -from collections import OrderedDict, namedtuple +from collections import namedtuple +from typing import Any, Callable, Dict, List, Mapping, NamedTuple, OrderedDict, Tuple, Union from pathlib import Path from django.utils import timezone import datumaro.components.extractor as datumaro from cvat.apps.engine.frame_provider import FrameProvider -from cvat.apps.engine.models import AttributeType, ShapeType +from cvat.apps.engine.models import AttributeType, LabeledShape, ShapeType, Project, Task, Label from datumaro.util import cast from datumaro.util.image import ByteImage, Image -from .annotation import AnnotationManager, TrackManager +from .annotation import AnnotationManager, TrackManager, AnnotationIR -class TaskData: +class InstanceLabelData: Attribute = namedtuple('Attribute', 'name, value') - LabeledShape = namedtuple( - 'LabeledShape', 'type, frame, label, points, occluded, attributes, source, group, z_order') - LabeledShape.__new__.__defaults__ = (0, 0) - TrackedShape = namedtuple( - 'TrackedShape', 'type, frame, points, occluded, outside, keyframe, attributes, source, group, z_order, label, track_id') - TrackedShape.__new__.__defaults__ = ('manual', 0, 0, None, 0) - Track = namedtuple('Track', 'label, group, source, shapes') - Tag = namedtuple('Tag', 'frame, label, attributes, source, group') - Tag.__new__.__defaults__ = (0, ) - Frame = namedtuple( - 'Frame', 'idx, frame, name, width, height, labeled_shapes, tags') - def __init__(self, annotation_ir, db_task, host='', create_callback=None): - self._annotation_ir = annotation_ir - self._db_task = db_task - self._host = host - self._create_callback = create_callback - self._MAX_ANNO_SIZE = 30000 - self._frame_info = {} - self._frame_mapping = {} - self._frame_step = db_task.data.get_frame_step() + def __init__(self, instance: Union[Task, Project]) -> None: + instance = instance.project if isinstance(instance, Task) and instance.project_id is not None else instance - db_labels = (self._db_task.project if self._db_task.project_id else self._db_task).label_set.all().prefetch_related( - 'attributespec_set').order_by('pk') + db_labels = instance.label_set.all().prefetch_related('attributespec_set').order_by('pk') - self._label_mapping = OrderedDict( - (db_label.id, db_label) for db_label in db_labels) + self._label_mapping = OrderedDict[int, Label]( + ((db_label.id, db_label) for db_label in db_labels), + ) self._attribute_mapping = {db_label.id: { 'mutable': {}, 'immutable': {}, 'spec': {}} @@ -67,9 +50,6 @@ def __init__(self, annotation_ir, db_task, host='', create_callback=None): **attr_mapping['immutable'], } - self._init_frame_info() - self._init_meta() - def _get_label_id(self, label_name): for db_label in self._label_mapping.values(): if label_name == db_label.name: @@ -101,6 +81,69 @@ def _get_mutable_attribute_id(self, label_id, attribute_name): def _get_immutable_attribute_id(self, label_id, attribute_name): return self._get_attribute_id(label_id, attribute_name, 'immutable') + def _import_attribute(self, label_id, attribute): + spec_id = self._get_attribute_id(label_id, attribute.name) + value = attribute.value + + if spec_id: + spec = self._attribute_mapping[label_id]['spec'][spec_id] + + try: + if spec.input_type == AttributeType.NUMBER: + pass # no extra processing required + elif spec.input_type == AttributeType.CHECKBOX: + if isinstance(value, str): + value = value.lower() + assert value in {'true', 'false'} + elif isinstance(value, (bool, int, float)): + value = 'true' if value else 'false' + else: + raise ValueError("Unexpected attribute value") + except Exception as e: + raise Exception("Failed to convert attribute '%s'='%s': %s" % + (self._get_label_name(label_id), value, e)) + + return { 'spec_id': spec_id, 'value': value } + + def _export_attributes(self, attributes): + exported_attributes = [] + for attr in attributes: + attribute_name = self._get_attribute_name(attr["spec_id"]) + exported_attributes.append(TaskData.Attribute( + name=attribute_name, + value=attr["value"], + )) + return exported_attributes + + +class TaskData(InstanceLabelData): + LabeledShape = namedtuple( + 'LabeledShape', 'type, frame, label, points, occluded, attributes, source, group, z_order') + LabeledShape.__new__.__defaults__ = (0, 0) + TrackedShape = namedtuple( + 'TrackedShape', 'type, frame, points, occluded, outside, keyframe, attributes, source, group, z_order, label, track_id') + TrackedShape.__new__.__defaults__ = ('manual', 0, 0, None, 0) + Track = namedtuple('Track', 'label, group, source, shapes') + Tag = namedtuple('Tag', 'frame, label, attributes, source, group') + Tag.__new__.__defaults__ = (0, ) + Frame = namedtuple( + 'Frame', 'idx, frame, name, width, height, labeled_shapes, tags') + + def __init__(self, annotation_ir, db_task, host='', create_callback=None): + self._annotation_ir = annotation_ir + self._db_task = db_task + self._host = host + self._create_callback = create_callback + self._MAX_ANNO_SIZE = 30000 + self._frame_info = {} + self._frame_mapping = {} + self._frame_step = db_task.data.get_frame_step() + + InstanceLabelData.__init__(self, db_task) + + self._init_frame_info() + self._init_meta() + def abs_frame_id(self, relative_id): if relative_id not in range(0, self._db_task.data.size): raise ValueError("Unknown internal frame id %s" % relative_id) @@ -132,79 +175,80 @@ def _init_frame_info(self): for frame_number, info in self._frame_info.items() } - def _init_meta(self): - db_segments = self._db_task.segment_set.all().prefetch_related('job_set') - self._meta = OrderedDict([ - ("task", OrderedDict([ - ("id", str(self._db_task.id)), - ("name", self._db_task.name), - ("size", str(self._db_task.data.size)), - ("mode", self._db_task.mode), - ("overlap", str(self._db_task.overlap)), - ("bugtracker", self._db_task.bug_tracker), - ("created", str(timezone.localtime(self._db_task.created_date))), - ("updated", str(timezone.localtime(self._db_task.updated_date))), - ("start_frame", str(self._db_task.data.start_frame)), - ("stop_frame", str(self._db_task.data.stop_frame)), - ("frame_filter", self._db_task.data.frame_filter), - - ("labels", [ - ("label", OrderedDict([ - ("name", db_label.name), - ("color", db_label.color), - ("attributes", [ - ("attribute", OrderedDict([ - ("name", db_attr.name), - ("mutable", str(db_attr.mutable)), - ("input_type", db_attr.input_type), - ("default_value", db_attr.default_value), - ("values", db_attr.values)])) - for db_attr in db_label.attributespec_set.all()]) - ])) for db_label in self._label_mapping.values() - ]), + @staticmethod + def meta_for_task(db_task, host, label_mapping=None): + db_segments = db_task.segment_set.all().prefetch_related('job_set') + + meta = OrderedDict([ + ("id", str(db_task.id)), + ("name", db_task.name), + ("size", str(db_task.data.size)), + ("mode", db_task.mode), + ("overlap", str(db_task.overlap)), + ("bugtracker", db_task.bug_tracker), + ("created", str(timezone.localtime(db_task.created_date))), + ("updated", str(timezone.localtime(db_task.updated_date))), + ("subset", db_task.subset or datumaro.DEFAULT_SUBSET_NAME), + ("start_frame", str(db_task.data.start_frame)), + ("stop_frame", str(db_task.data.stop_frame)), + ("frame_filter", db_task.data.frame_filter), + + ("segments", [ + ("segment", OrderedDict([ + ("id", str(db_segment.id)), + ("start", str(db_segment.start_frame)), + ("stop", str(db_segment.stop_frame)), + ("url", "{}/?id={}".format( + host, db_segment.job_set.all()[0].id))] + )) for db_segment in db_segments + ]), + + ("owner", OrderedDict([ + ("username", db_task.owner.username), + ("email", db_task.owner.email) + ]) if db_task.owner else ""), + + ("assignee", OrderedDict([ + ("username", db_task.assignee.username), + ("email", db_task.assignee.email) + ]) if db_task.assignee else ""), + ]) - ("segments", [ - ("segment", OrderedDict([ - ("id", str(db_segment.id)), - ("start", str(db_segment.start_frame)), - ("stop", str(db_segment.stop_frame)), - ("url", "{}/?id={}".format( - self._host, db_segment.job_set.all()[0].id))] - )) for db_segment in db_segments - ]), + if label_mapping is not None: + meta['labels'] = [ + ("label", OrderedDict([ + ("name", db_label.name), + ("color", db_label.color), + ("attributes", [ + ("attribute", OrderedDict([ + ("name", db_attr.name), + ("mutable", str(db_attr.mutable)), + ("input_type", db_attr.input_type), + ("default_value", db_attr.default_value), + ("values", db_attr.values)])) + for db_attr in db_label.attributespec_set.all()]) + ])) for db_label in label_mapping.values() + ] + + if hasattr(db_task.data, "video"): + meta["original_size"] = OrderedDict([ + ("width", str(db_task.data.video.width)), + ("height", str(db_task.data.video.height)) + ]) - ("owner", OrderedDict([ - ("username", self._db_task.owner.username), - ("email", self._db_task.owner.email) - ]) if self._db_task.owner else ""), + return meta - ("assignee", OrderedDict([ - ("username", self._db_task.assignee.username), - ("email", self._db_task.assignee.email) - ]) if self._db_task.assignee else ""), - ])), + def _init_meta(self): + self._meta = OrderedDict([ + ("task", self.meta_for_task(self._db_task, self._host, self._label_mapping)), ("dumped", str(timezone.localtime(timezone.now()))) ]) if hasattr(self._db_task.data, "video"): - self._meta["task"]["original_size"] = OrderedDict([ - ("width", str(self._db_task.data.video.width)), - ("height", str(self._db_task.data.video.height)) - ]) # Add source to dumped file self._meta["source"] = str( osp.basename(self._db_task.data.video.path)) - def _export_attributes(self, attributes): - exported_attributes = [] - for attr in attributes: - attribute_name = self._get_attribute_name(attr["spec_id"]) - exported_attributes.append(TaskData.Attribute( - name=attribute_name, - value=attr["value"], - )) - return exported_attributes - def _export_tracked_shape(self, shape): return TaskData.TrackedShape( type=shape["type"], @@ -330,30 +374,6 @@ def _import_tag(self, tag): if self._get_attribute_id(label_id, attrib.name)] return _tag - def _import_attribute(self, label_id, attribute): - spec_id = self._get_attribute_id(label_id, attribute.name) - value = attribute.value - - if spec_id: - spec = self._attribute_mapping[label_id]['spec'][spec_id] - - try: - if spec.input_type == AttributeType.NUMBER: - pass # no extra processing required - elif spec.input_type == AttributeType.CHECKBOX: - if isinstance(value, str): - value = value.lower() - assert value in {'true', 'false'} - elif isinstance(value, (bool, int, float)): - value = 'true' if value else 'false' - else: - raise ValueError("Unexpected attribute value") - except Exception as e: - raise Exception("Failed to convert attribute '%s'='%s': %s" % - (self._get_label_name(label_id), value, e)) - - return { 'spec_id': spec_id, 'value': value } - def _import_shape(self, shape): _shape = shape._asdict() label_id = self._get_label_id(_shape.pop('label')) @@ -456,6 +476,264 @@ def match_frame_fuzzy(self, path): return v return None +class ProjectData(InstanceLabelData): + # TODO: strictify + LabledShape = NamedTuple('LabledShape', [('type',Any), ('frame',Any), ('label',Any), ('points',Any), ('occluded',Any), ('attributes',Any), ('source',Any), ('group',Any), ('z_order',Any)]) + LabledShape.__new__.__defaults__ = (0,0) + TrackedShape = NamedTuple('TrackedShape', + [('type',Any), ('frame',Any), ('points',Any), ('occluded',Any), ('outside',Any), ('keyframe',Any), ('attributes',Any), ('source',Any), ('group',Any), ('z_order',Any), ('label',Any), ('track_id',Any)], + ) + TrackedShape.__new__.__defaults__ = ('manual', 0, 0, None, 0) + Track = NamedTuple('Track', [('label',Any), ('attributes',Any), ('source',Any), ('group',Any)]) + Tag = NamedTuple('Tag', [('frame',Any), ('label',Any), ('attributes',Any), ('source',Any), ('group',Any)]) + Tag.__new__.__defaults__ = (0, ) + Frame = NamedTuple('Frame', [('task_id', int), ('subset', str), ('idx', int), ('frame', int), ('name', str), ('width', int), ('height', int), ('labeled_shapes', List[Union[LabeledShape, TrackedShape]]), ('tags', List[Tag])]) + + def __init__(self, annotation_irs: Mapping[str, AnnotationIR], db_project: Project, host: str, create_callback: Callable = None): + self._annotation_irs = annotation_irs + self._db_project = db_project + self._db_tasks: OrderedDict[int, Task] = OrderedDict( + ((db_task.id, db_task) for db_task in db_project.tasks.order_by("subset","id").all()) + ) + self._subsets = set() + self._host = host + self._create_callback = create_callback + self._MAX_ANNO_SIZE = 30000 + self._frame_info: Dict[Tuple[int, int], dict] = dict() + self._frame_mapping: Dict[Tuple[str, str], Tuple[str, str]] = dict() + self._frame_step_mapping = None + + for task in self._db_tasks.values(): + self._subsets.add(task.subset) + self._subsets: List[str] = list(self._subsets) + + InstanceLabelData.__init__(self, db_project) + + self._init_task_frame_offsets() + self._init_frame_info() + self._init_meta() + + def abs_frame_id(self, task_id: int, relative_id: int) -> int: + task = self._db_tasks[task_id] + if relative_id not in range(0, task.data.size): + raise ValueError(f"Unknown internal frame id {relative_id}") + return relative_id * task.data.get_frame_step() + task.data.start_frame + self._task_frame_offsets[task_id] + + def rel_frame_id(self, task_id: int, absolute_id: int) -> int: + task = self._db_tasks[task_id] + d, m = divmod( + absolute_id - self._task_frame_offsets[task_id] - task.data.start_frame, self._frame_step) + if m or d not in range(0, task.data.size): + raise ValueError(f"Unknown frame {absolute_id}") + return d + + def _init_task_frame_offsets(self): + self._task_frame_offsets: Dict[int, int] = dict() + s = 0 + subset = None + + for task in self._db_tasks.values(): + if subset != task.subset: + s = 0 + subset = task.subset + self._task_frame_offsets[task.id] = s + s += task.data.start_frame + task.data.get_frame_step() * task.data.size + + + def _init_frame_info(self): + self._frame_info = dict() + for task in self._db_tasks.values(): + if hasattr(task.data, 'video'): + self._frame_info.update({(task.id, frame): { + "path": "frame_{:06d}".format(self.abs_frame_id(task.id, frame)), + "width": task.data.video.width, + "height": task.data.video.height, + "subset": task.subset, + } for frame in range(task.data.size)}) + else: + self._frame_info.update({(task.id, self.rel_frame_id(task.id, db_image.frame)): { + "path": db_image.path, + "width": db_image.width, + "height": db_image.height, + "subset": task.subset + } for db_image in task.data.images.all()}) + + self._frame_mapping = { + (self._db_tasks[frame_ident[0]].subset, self._get_filename(info["path"])): frame_ident + for frame_ident, info in self._frame_info.items() + } + + def _init_meta(self): + self._meta = OrderedDict([ + ('project', OrderedDict([ + ('id', str(self._db_project.id)), + ('name', self._db_project.name), + ("bugtracker", self._db_project.bug_tracker), + ("created", str(timezone.localtime(self._db_project.created_date))), + ("updated", str(timezone.localtime(self._db_project.updated_date))), + ("tasks", [ + ('task', + TaskData.meta_for_task(db_task, self._host) + ) for db_task in self._db_project.tasks.all() + ]), + + ("labels", [ + ("label", OrderedDict([ + ("name", db_label.name), + ("color", db_label.color), + ("attributes", [ + ("attribute", OrderedDict([ + ("name", db_attr.name), + ("mutable", str(db_attr.mutable)), + ("input_type", db_attr.input_type), + ("default_value", db_attr.default_value), + ("values", db_attr.values)])) + for db_attr in db_label.attributespec_set.all()]) + ])) for db_label in self._label_mapping.values() + ]), + + ("owner", OrderedDict([ + ("username", self._db_project.owner.username), + ("email", self._db_project.owner.email), + ]) if self._db_project.owner else ""), + + ("assignee", OrderedDict([ + ("username", self._db_project.assignee.username), + ("email", self._db_project.assignee.email), + ]) if self._db_project.assignee else ""), + ])), + ("dumped", str(timezone.localtime(timezone.now()))) + ]) + + def _export_tracked_shape(self, shape: dict, task_id: int) -> TrackedShape: + return ProjectData.TrackedShape( + type=shape["type"], + frame=self.abs_frame_id(task_id, shape["frame"]), + label=self._get_label_name(shape["label_id"]), + points=shape["points"], + occluded=shape["occluded"], + z_order=shape.get("z_order", 0), + group=shape.get("group", 0), + outside=shape.get("outside", False), + keyframe=shape.get("keyframe", True), + track_id=shape["track_id"], + source=shape.get("source", "manual"), + attributes=self._export_attributes(shape["attributes"]), + ) + + def _export_labeled_shape(self, shape: dict, task_id: int) -> LabeledShape: + return ProjectData.LabeledShape( + type=shape["type"], + label=self._get_label_name(shape["label_id"]), + frame=self.abs_frame_id(task_id, shape["frame"]), + points=shape["points"], + occluded=shape["occluded"], + z_order=shape.get("z_order", 0), + group=shape.get("group", 0), + source=shape["source"], + attributes=self._export_attributes(shape["attributes"]), + ) + + def _export_tag(self, tag: dict, task_id: int) -> Tag: + return ProjectData.Tag( + frame=self.abs_frame_id(task_id, tag["frame"]), + label=self._get_label_name(tag["label_id"]), + group=tag.get("group", 0), + source=tag["source"], + attributes=self._export_attributes(tag["attributes"]), + ) + + def group_by_frame(self, include_empty=False): + frames: Dict[Tuple[str, int], ProjectData.Frame] = {} + def get_frame(task_id: int, idx: int) -> ProjectData.Frame: + frame_info = self._frame_info[(task_id, idx)] + abs_frame = self.abs_frame_id(task_id, idx) + if (frame_info["subset"], abs_frame) not in frames: + frames[(frame_info["subset"], abs_frame)] = ProjectData.Frame( + task_id=task_id, + subset=frame_info["subset"], + idx=idx, + frame=abs_frame, + name=frame_info["path"], + height=frame_info["height"], + width=frame_info["width"], + labeled_shapes=[], + tags=[], + ) + return frames[(frame_info["subset"], abs_frame)] + + if include_empty: + for ident in self._frame_info: + get_frame(*ident) + + for task in self._db_tasks.values(): + anno_manager = AnnotationManager(self._annotation_irs[task.id]) + for shape in sorted(anno_manager.to_shapes(task.data.size), + key=lambda shape: shape.get("z_order", 0)): + if (task.id, shape['frame']) not in self._frame_info: + continue + if 'track_id' in shape: + if shape['outside']: + continue + exported_shape = self._export_tracked_shape(shape, task.id) + else: + exported_shape = self._export_labeled_shape(shape, task.id) + get_frame(task.id, shape['frame']).labeled_shapes.append(exported_shape) + + for tag in self._annotation_irs[task.id].tags: + get_frame(task.id, tag['frame']).tags.append(self._export_tag(tag, task.id)) + + return iter(frames.values()) + + @property + def shapes(self): + for task in self._db_tasks.values(): + for shape in self._annotation_irs[task.id].shapes: + yield self._export_labeled_shape(task.id, shape) + + @property + def tracks(self): + raise NotImplementedError() + + @property + def tags(self): + for task in self._db_tasks.values(): + for tag in self._annotation_irs[task.id].tags: + yield self._export_tag(task.id, tag) + + @property + def meta(self): + return self._meta + + @property + def data(self): + raise NotImplementedError() + + @property + def frame_info(self): + raise NotImplementedError() + + @property + def frame_step(self): + raise NotImplementedError() + + @property + def db_project(self): + raise NotImplementedError() + + @property + def subsets(self) -> List[str]: + return self._subsets + + @property + def tasks(self): + return list(self._db_tasks.values()) + + @staticmethod + def _get_filename(path): + return osp.splitext(path)[0] + + class CvatTaskDataExtractor(datumaro.SourceExtractor): def __init__(self, task_data, include_images=False): super().__init__() @@ -528,8 +806,6 @@ def _load_categories(cvat_anno): return categories def _read_cvat_anno(self, cvat_frame_anno, task_data): - item_anno = [] - categories = self.categories() label_cat = categories[datumaro.AnnotationType.label] def map_label(name): return label_cat.find(name)[0] @@ -538,73 +814,171 @@ def map_label(name): return label_cat.find(name)[0] for _, label in task_data.meta['task']['labels'] } - def convert_attrs(label, cvat_attrs): - cvat_attrs = {a.name: a.value for a in cvat_attrs} - dm_attr = dict() - for _, a_desc in label_attrs[label]: - a_name = a_desc['name'] - a_value = cvat_attrs.get(a_name, a_desc['default_value']) - try: - if a_desc['input_type'] == AttributeType.NUMBER: - a_value = float(a_value) - elif a_desc['input_type'] == AttributeType.CHECKBOX: - a_value = (a_value.lower() == 'true') - dm_attr[a_name] = a_value - except Exception as e: - raise Exception( - "Failed to convert attribute '%s'='%s': %s" % - (a_name, a_value, e)) - return dm_attr - - for tag_obj in cvat_frame_anno.tags: - anno_group = tag_obj.group or 0 - anno_label = map_label(tag_obj.label) - anno_attr = convert_attrs(tag_obj.label, tag_obj.attributes) - - anno = datumaro.Label(label=anno_label, - attributes=anno_attr, group=anno_group) - item_anno.append(anno) - - for shape_obj in cvat_frame_anno.labeled_shapes: - anno_group = shape_obj.group or 0 - anno_label = map_label(shape_obj.label) - anno_attr = convert_attrs(shape_obj.label, shape_obj.attributes) - anno_attr['occluded'] = shape_obj.occluded - - if hasattr(shape_obj, 'track_id'): - anno_attr['track_id'] = shape_obj.track_id - anno_attr['keyframe'] = shape_obj.keyframe - - anno_points = shape_obj.points - if shape_obj.type == ShapeType.POINTS: - anno = datumaro.Points(anno_points, - label=anno_label, attributes=anno_attr, group=anno_group, - z_order=shape_obj.z_order) - elif shape_obj.type == ShapeType.POLYLINE: - anno = datumaro.PolyLine(anno_points, - label=anno_label, attributes=anno_attr, group=anno_group, - z_order=shape_obj.z_order) - elif shape_obj.type == ShapeType.POLYGON: - anno = datumaro.Polygon(anno_points, - label=anno_label, attributes=anno_attr, group=anno_group, - z_order=shape_obj.z_order) - elif shape_obj.type == ShapeType.RECTANGLE: - x0, y0, x1, y1 = anno_points - anno = datumaro.Bbox(x0, y0, x1 - x0, y1 - y0, - label=anno_label, attributes=anno_attr, group=anno_group, - z_order=shape_obj.z_order) - elif shape_obj.type == ShapeType.CUBOID: - continue # Datumaro does not support cuboids + return convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label) + +class CVATProjectDataExtractor(datumaro.SourceExtractor): + def __init__(self, project_data: ProjectData, include_images=False): + super().__init__() + self._categories = self._load_categories(project_data) + + dm_items: List[datumaro.DatasetItem] = [] + + ext_per_task: Dict[int, str] = {} + image_maker_per_task: Dict[int, Callable] = {} + + for task in project_data.tasks: + is_video = task.mode == 'interpolation' + ext_per_task[task.id] = FrameProvider.VIDEO_FRAME_EXT if is_video else '' + if include_images: + frame_provider = FrameProvider(task.data) + if is_video: + # optimization for videos: use numpy arrays instead of bytes + # some formats or transforms can require image data + def _make_image(i, **kwargs): + loader = lambda _: frame_provider.get_frame(i, + quality=frame_provider.Quality.ORIGINAL, + out_type=frame_provider.Type.NUMPY_ARRAY)[0] + return Image(loader=loader, **kwargs) + else: + # for images use encoded data to avoid recoding + def _make_image(i, **kwargs): + loader = lambda _: frame_provider.get_frame(i, + quality=frame_provider.Quality.ORIGINAL, + out_type=frame_provider.Type.BUFFER)[0].getvalue() + return ByteImage(data=loader, **kwargs) + image_maker_per_task[task.id] = _make_image + + for frame_data in project_data.group_by_frame(include_empty=True): + image_args = { + 'path': frame_data.name + ext_per_task[frame_data.task_id], + 'size': (frame_data.height, frame_data.width), + } + if include_images: + dm_image = _make_image(frame_data.idx, **image_args) else: - raise Exception("Unknown shape type '%s'" % shape_obj.type) + dm_image = Image(**image_args) + dm_anno = self._read_cvat_anno(frame_data, project_data) + dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0], + annotation=dm_anno, image=dm_image, subset=frame_data.subset, + attributes={'frame': frame_data.frame} + ) + dm_items.append(dm_item) - item_anno.append(anno) + self._items = dm_items - return item_anno + def __iter__(self): + for item in self._items: + yield item + + def __len__(self): + return len(self._items) + + def categories(self): + return self._categories + + + @staticmethod + def _load_categories(project_data: ProjectData): + categories: Dict[datumaro.AnnotationType, datumaro.LabelCategories] = {} + + label_categories = datumaro.LabelCategories(attributes=['occluded']) + + for _, label in project_data.meta['project']['labels']: + label_categories.add(label['name']) + for _, attr in label['attributes']: + label_categories.attributes.add(attr['name']) + + categories[datumaro.AnnotationType.label] = label_categories + + return categories + + def _read_cvat_anno(self, cvat_frame_anno: ProjectData.Frame, project_data: ProjectData): + categories = self.categories() + label_cat = categories[datumaro.AnnotationType.label] + def map_label(name: str): return label_cat.find(name)[0] + label_attrs = { + label['name']: label['attributes'] + for _, label in project_data.meta['project']['labels'] + } + + return convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label) + +def CVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_images: bool=False): + if isinstance(instance_data, ProjectData): + return CVATProjectDataExtractor(instance_data, include_images) + else: + return CvatTaskDataExtractor(instance_data, include_images) class CvatImportError(Exception): pass +def convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label): + item_anno = [] + + def convert_attrs(label, cvat_attrs): + cvat_attrs = {a.name: a.value for a in cvat_attrs} + dm_attr = dict() + for _, a_desc in label_attrs[label]: + a_name = a_desc['name'] + a_value = cvat_attrs.get(a_name, a_desc['default_value']) + try: + if a_desc['input_type'] == AttributeType.NUMBER: + a_value = float(a_value) + elif a_desc['input_type'] == AttributeType.CHECKBOX: + a_value = (a_value.lower() == 'true') + dm_attr[a_name] = a_value + except Exception as e: + raise Exception( + "Failed to convert attribute '%s'='%s': %s" % + (a_name, a_value, e)) + return dm_attr + + for tag_obj in cvat_frame_anno.tags: + anno_group = tag_obj.group or 0 + anno_label = map_label(tag_obj.label) + anno_attr = convert_attrs(tag_obj.label, tag_obj.attributes) + + anno = datumaro.Label(label=anno_label, + attributes=anno_attr, group=anno_group) + item_anno.append(anno) + + for shape_obj in cvat_frame_anno.labeled_shapes: + anno_group = shape_obj.group or 0 + anno_label = map_label(shape_obj.label) + anno_attr = convert_attrs(shape_obj.label, shape_obj.attributes) + anno_attr['occluded'] = shape_obj.occluded + + if hasattr(shape_obj, 'track_id'): + anno_attr['track_id'] = shape_obj.track_id + anno_attr['keyframe'] = shape_obj.keyframe + + anno_points = shape_obj.points + if shape_obj.type == ShapeType.POINTS: + anno = datumaro.Points(anno_points, + label=anno_label, attributes=anno_attr, group=anno_group, + z_order=shape_obj.z_order) + elif shape_obj.type == ShapeType.POLYLINE: + anno = datumaro.PolyLine(anno_points, + label=anno_label, attributes=anno_attr, group=anno_group, + z_order=shape_obj.z_order) + elif shape_obj.type == ShapeType.POLYGON: + anno = datumaro.Polygon(anno_points, + label=anno_label, attributes=anno_attr, group=anno_group, + z_order=shape_obj.z_order) + elif shape_obj.type == ShapeType.RECTANGLE: + x0, y0, x1, y1 = anno_points + anno = datumaro.Bbox(x0, y0, x1 - x0, y1 - y0, + label=anno_label, attributes=anno_attr, group=anno_group, + z_order=shape_obj.z_order) + elif shape_obj.type == ShapeType.CUBOID: + continue # Datumaro does not support cuboids + else: + raise Exception("Unknown shape type '%s'" % shape_obj.type) + + item_anno.append(anno) + + return item_anno + def match_dm_item(item, task_data, root_hint=None): is_video = task_data.meta['task']['mode'] == 'interpolation' diff --git a/cvat/apps/dataset_manager/formats/camvid.py b/cvat/apps/dataset_manager/formats/camvid.py index a8fb50592e9..d8d8c4d9041 100644 --- a/cvat/apps/dataset_manager/formats/camvid.py +++ b/cvat/apps/dataset_manager/formats/camvid.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive @@ -16,13 +16,13 @@ @exporter(name='CamVid', ext='ZIP', version='1.0') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(CVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) dataset.transform('polygons_to_masks') dataset.transform('boxes_to_masks') dataset.transform('merge_instance_segments') - label_map = make_colormap(task_data) + label_map = make_colormap(instance_data) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'camvid', save_images=save_images, apply_colormap=True, @@ -31,10 +31,10 @@ def _export(dst_file, task_data, save_images=False): make_zip_archive(temp_dir, dst_file) @importer(name='CamVid', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: Archive(src_file.name).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'camvid', env=dm_env) dataset.transform('masks_to_polygons') - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/coco.py b/cvat/apps/dataset_manager/formats/coco.py index 3e4fb223fdb..dbd4a4946f0 100644 --- a/cvat/apps/dataset_manager/formats/coco.py +++ b/cvat/apps/dataset_manager/formats/coco.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor, \ +from cvat.apps.dataset_manager.bindings import CVATDataExtractor, \ import_dm_annotations from cvat.apps.dataset_manager.util import make_zip_archive @@ -15,9 +15,9 @@ @exporter(name='COCO', ext='ZIP', version='1.0') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(CVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'coco_instances', save_images=save_images, merge_images=True) @@ -25,14 +25,14 @@ def _export(dst_file, task_data, save_images=False): make_zip_archive(temp_dir, dst_file) @importer(name='COCO', ext='JSON, ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): if zipfile.is_zipfile(src_file): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'coco', env=dm_env) - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) else: dataset = Dataset.import_from(src_file.name, 'coco_instances', env=dm_env) - import_dm_annotations(dataset, task_data) \ No newline at end of file + import_dm_annotations(dataset, instance_data) \ No newline at end of file diff --git a/cvat/apps/dataset_manager/formats/icdar.py b/cvat/apps/dataset_manager/formats/icdar.py index 8df8f3e40fd..8845ab411db 100644 --- a/cvat/apps/dataset_manager/formats/icdar.py +++ b/cvat/apps/dataset_manager/formats/icdar.py @@ -9,7 +9,7 @@ from datumaro.components.extractor import (AnnotationType, Caption, Label, LabelCategories, Transform) -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive @@ -75,45 +75,45 @@ def transform_item(self, item): return item.wrap(annotations=annotations) @exporter(name='ICDAR Recognition', ext='ZIP', version='1.0') -def _export_recognition(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export_recognition(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(CVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) dataset.transform(LabelToCaption) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'icdar_word_recognition', save_images=save_images) make_zip_archive(temp_dir, dst_file) @importer(name='ICDAR Recognition', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'icdar_word_recognition', env=dm_env) dataset.transform(CaptionToLabel, 'icdar') - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) @exporter(name='ICDAR Localization', ext='ZIP', version='1.0') -def _export_localization(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export_localization(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(CVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'icdar_text_localization', save_images=save_images) make_zip_archive(temp_dir, dst_file) @importer(name='ICDAR Localization', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'icdar_text_localization', env=dm_env) dataset.transform(AddLabelToAnns, 'icdar') - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) @exporter(name='ICDAR Segmentation', ext='ZIP', version='1.0') -def _export_segmentation(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export_segmentation(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(CVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.transform('polygons_to_masks') dataset.transform('boxes_to_masks') @@ -122,10 +122,10 @@ def _export_segmentation(dst_file, task_data, save_images=False): make_zip_archive(temp_dir, dst_file) @importer(name='ICDAR Segmentation', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'icdar_text_segmentation', env=dm_env) dataset.transform(AddLabelToAnns, 'icdar') dataset.transform('masks_to_polygons') - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/imagenet.py b/cvat/apps/dataset_manager/formats/imagenet.py index 2ed0cb47484..8720a6cdf64 100644 --- a/cvat/apps/dataset_manager/formats/imagenet.py +++ b/cvat/apps/dataset_manager/formats/imagenet.py @@ -9,7 +9,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor, \ +from cvat.apps.dataset_manager.bindings import CVATDataExtractor, \ import_dm_annotations from cvat.apps.dataset_manager.util import make_zip_archive @@ -17,9 +17,9 @@ @exporter(name='ImageNet', ext='ZIP', version='1.0') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(CVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: if save_images: dataset.export(temp_dir, 'imagenet', save_images=save_images) @@ -29,11 +29,11 @@ def _export(dst_file, task_data, save_images=False): make_zip_archive(temp_dir, dst_file) @importer(name='ImageNet', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) if glob(osp.join(tmp_dir, '*.txt')): dataset = Dataset.import_from(tmp_dir, 'imagenet_txt', env=dm_env) else: dataset = Dataset.import_from(tmp_dir, 'imagenet', env=dm_env) - import_dm_annotations(dataset, task_data) \ No newline at end of file + import_dm_annotations(dataset, instance_data) \ No newline at end of file diff --git a/cvat/apps/dataset_manager/formats/labelme.py b/cvat/apps/dataset_manager/formats/labelme.py index 744b11faab0..5a65b407942 100644 --- a/cvat/apps/dataset_manager/formats/labelme.py +++ b/cvat/apps/dataset_manager/formats/labelme.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive @@ -15,19 +15,19 @@ @exporter(name='LabelMe', ext='ZIP', version='3.0') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(CVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'label_me', save_images=save_images) make_zip_archive(temp_dir, dst_file) @importer(name='LabelMe', ext='ZIP', version='3.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: Archive(src_file.name).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'label_me', env=dm_env) dataset.transform('masks_to_polygons') - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/market1501.py b/cvat/apps/dataset_manager/formats/market1501.py index 1b6e24f43f1..ce170db719a 100644 --- a/cvat/apps/dataset_manager/formats/market1501.py +++ b/cvat/apps/dataset_manager/formats/market1501.py @@ -9,7 +9,7 @@ from datumaro.components.extractor import (AnnotationType, Label, LabelCategories, Transform) -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive @@ -59,19 +59,19 @@ def transform_item(self, item): @exporter(name='Market-1501', ext='ZIP', version='1.0') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(CVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.transform(LabelAttrToAttr, 'market-1501') dataset.export(temp_dir, 'market1501', save_images=save_images) make_zip_archive(temp_dir, dst_file) @importer(name='Market-1501', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'market1501', env=dm_env) dataset.transform(AttrToLabelAttr, 'market-1501') - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/mask.py b/cvat/apps/dataset_manager/formats/mask.py index 3e3780e8c6a..97ec836ef8b 100644 --- a/cvat/apps/dataset_manager/formats/mask.py +++ b/cvat/apps/dataset_manager/formats/mask.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive @@ -16,23 +16,23 @@ @exporter(name='Segmentation mask', ext='ZIP', version='1.1') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(CVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) dataset.transform('polygons_to_masks') dataset.transform('boxes_to_masks') dataset.transform('merge_instance_segments') with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'voc_segmentation', save_images=save_images, - apply_colormap=True, label_map=make_colormap(task_data)) + apply_colormap=True, label_map=make_colormap(instance_data)) make_zip_archive(temp_dir, dst_file) @importer(name='Segmentation mask', ext='ZIP', version='1.1') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: Archive(src_file.name).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'voc', env=dm_env) dataset.transform('masks_to_polygons') - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/pascal_voc.py b/cvat/apps/dataset_manager/formats/pascal_voc.py index 3f10b93aa93..3e040200836 100644 --- a/cvat/apps/dataset_manager/formats/pascal_voc.py +++ b/cvat/apps/dataset_manager/formats/pascal_voc.py @@ -11,17 +11,17 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, - import_dm_annotations) +from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, + ProjectData, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive from .registry import dm_env, exporter, importer @exporter(name='PASCAL VOC', ext='ZIP', version='1.1') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(CVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'voc', save_images=save_images, label_map='source') @@ -29,15 +29,16 @@ def _export(dst_file, task_data, save_images=False): make_zip_archive(temp_dir, dst_file) @importer(name='PASCAL VOC', ext='ZIP', version='1.1') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: Archive(src_file.name).extractall(tmp_dir) # put label map from the task if not present labelmap_file = osp.join(tmp_dir, 'labelmap.txt') if not osp.isfile(labelmap_file): - labels = (label['name'] + ':::' - for _, label in task_data.meta['task']['labels']) + labels_meta = instance_data.meta['project']['labels'] \ + if isinstance(instance_data, ProjectData) else instance_data.meta['task']['labels'] + labels = (label['name'] + ':::' for _, label in labels_meta) with open(labelmap_file, 'w') as f: f.write('\n'.join(labels)) @@ -57,4 +58,4 @@ def _import(src_file, task_data): dataset = Dataset.import_from(tmp_dir, 'voc', env=dm_env) dataset.transform('masks_to_polygons') - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/tfrecord.py b/cvat/apps/dataset_manager/formats/tfrecord.py index 9847bf61b66..29270f09b23 100644 --- a/cvat/apps/dataset_manager/formats/tfrecord.py +++ b/cvat/apps/dataset_manager/formats/tfrecord.py @@ -6,7 +6,7 @@ from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive from datumaro.components.project import Dataset @@ -23,18 +23,18 @@ @exporter(name='TFRecord', ext='ZIP', version='1.0', enabled=tf_available) -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(CVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'tf_detection_api', save_images=save_images) make_zip_archive(temp_dir, dst_file) @importer(name='TFRecord', ext='ZIP', version='1.0', enabled=tf_available) -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: Archive(src_file.name).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'tf_detection_api', env=dm_env) - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/vggface2.py b/cvat/apps/dataset_manager/formats/vggface2.py index 528f52c76d7..5f618739c1c 100644 --- a/cvat/apps/dataset_manager/formats/vggface2.py +++ b/cvat/apps/dataset_manager/formats/vggface2.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor, \ +from cvat.apps.dataset_manager.bindings import CVATDataExtractor, \ import_dm_annotations from cvat.apps.dataset_manager.util import make_zip_archive @@ -15,18 +15,18 @@ @exporter(name='VGGFace2', ext='ZIP', version='1.0') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(CVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'vgg_face2', save_images=save_images) make_zip_archive(temp_dir, dst_file) @importer(name='VGGFace2', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'vgg_face2', env=dm_env) - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/widerface.py b/cvat/apps/dataset_manager/formats/widerface.py index 7f120ffe215..4dc3b9ab355 100644 --- a/cvat/apps/dataset_manager/formats/widerface.py +++ b/cvat/apps/dataset_manager/formats/widerface.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor, \ +from cvat.apps.dataset_manager.bindings import CVATDataExtractor, \ import_dm_annotations from cvat.apps.dataset_manager.util import make_zip_archive @@ -15,18 +15,18 @@ @exporter(name='WiderFace', ext='ZIP', version='1.0') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(CVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'wider_face', save_images=save_images) make_zip_archive(temp_dir, dst_file) @importer(name='WiderFace', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'wider_face', env=dm_env) - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/yolo.py b/cvat/apps/dataset_manager/formats/yolo.py index 0df6f5fe27a..76251b29ba7 100644 --- a/cvat/apps/dataset_manager/formats/yolo.py +++ b/cvat/apps/dataset_manager/formats/yolo.py @@ -8,7 +8,7 @@ from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, import_dm_annotations, match_dm_item, find_dataset_root) from cvat.apps.dataset_manager.util import make_zip_archive from datumaro.components.extractor import DatasetItem @@ -19,9 +19,9 @@ @exporter(name='YOLO', ext='ZIP', version='1.1') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(CVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'yolo', save_images=save_images) diff --git a/cvat/apps/dataset_manager/project.py b/cvat/apps/dataset_manager/project.py new file mode 100644 index 00000000000..8ec138f4a9f --- /dev/null +++ b/cvat/apps/dataset_manager/project.py @@ -0,0 +1,71 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from typing import Callable + +from django.db import transaction + +from cvat.apps.engine import models +from cvat.apps.dataset_manager.task import TaskAnnotation + +from .annotation import AnnotationIR +from .bindings import ProjectData +from .formats.registry import make_exporter + +def export_project(project_id, dst_file, format_name, + server_url=None, save_images=False): + # For big tasks dump function may run for a long time and + # we dont need to acquire lock after the task has been initialized from DB. + # But there is the bug with corrupted dump file in case 2 or + # more dump request received at the same time: + # https://github.com/opencv/cvat/issues/217 + with transaction.atomic(): + project = TaskAnnotation(project_id) + project.init_from_db() + + exporter = make_exporter(format_name) + with open(dst_file, 'wb') as f: + project.export(f, exporter, host=server_url, save_images=save_images) + +class ProjectAnnotation: + def __init__(self, pk: int): + self.db_project = models.Project.objects.get(id=pk) + self.db_tasks = models.Task.objects.filter(project__id=pk).order_by('id') + + self.annotation_irs: dict[int, AnnotationIR] = dict() + + def reset(self): + for annotation_ir in self.annotation_irs.values(): + annotation_ir.reset() + + def put(self, data): + raise NotImplementedError() + + def create(self, data): + raise NotImplementedError() + + def update(self, data): + raise NotImplementedError() + + def delete(self, data=None): + raise NotImplementedError() + + def init_from_db(self): + self.reset() + + for task in self.db_tasks: + annotation = TaskAnnotation(pk=task.id) + annotation.init_from_db() + self.annotation_irs[task.id] = annotation.ir_data + + def export(self, dst_file: str, exporter: Callable, host: str='', **options): + project_data = ProjectData( + annotation_irs=self.annotation_irs, + db_project=self.db_project, + host=host + ) + exporter(dst_file, project_data, **options) + @property + def data(self) -> dict: + raise NotImplementedError() \ No newline at end of file diff --git a/cvat/apps/dataset_manager/views.py b/cvat/apps/dataset_manager/views.py index b622eaa65af..faf1a8c1f0b 100644 --- a/cvat/apps/dataset_manager/views.py +++ b/cvat/apps/dataset_manager/views.py @@ -11,8 +11,9 @@ from django.utils import timezone import cvat.apps.dataset_manager.task as task +import cvat.apps.dataset_manager.project as project from cvat.apps.engine.log import slogger -from cvat.apps.engine.models import Task +from cvat.apps.engine.models import Project, Task from datumaro.cli.util import make_file_name from datumaro.util import to_snake_case @@ -29,22 +30,32 @@ def log_exception(logger=None, exc_info=True): exc_info=exc_info) -def get_export_cache_dir(db_task): - task_dir = osp.abspath(db_task.get_task_dirname()) - if osp.isdir(task_dir): - return osp.join(task_dir, 'export_cache') +def get_export_cache_dir(db_instance): + base_dir = osp.abspath(db_instance.get_project_dirname() if isinstance(db_instance, Project) else db_instance.get_task_dirname()) + if osp.isdir(base_dir): + return osp.join(base_dir, 'export_cache') else: - raise Exception('Task dir {} does not exist'.format(task_dir)) + raise Exception('{} dir {} does not exist'.format("Project" if isinstance(db_instance, Project) else "Task", base_dir)) DEFAULT_CACHE_TTL = timedelta(hours=10) -CACHE_TTL = DEFAULT_CACHE_TTL +TASK_CACHE_TTL = DEFAULT_CACHE_TTL +PROJECT_CACHE_TTL = DEFAULT_CACHE_TTL / 3 -def export_task(task_id, dst_format, server_url=None, save_images=False): +def export(dst_format, task_id=None, project_id=None, server_url=None, save_images=False): try: - db_task = Task.objects.get(pk=task_id) - - cache_dir = get_export_cache_dir(db_task) + if task_id is not None: + db_instance = Task.objects.get(pk=task_id) + logger = slogger.task[task_id] + cache_ttl = TASK_CACHE_TTL + export_fn = task.export_task + else: + db_instance = Project.objects.get(pk=project_id) + logger = slogger.project[project_id] + cache_ttl = PROJECT_CACHE_TTL + export_fn = project.export_project + + cache_dir = get_export_cache_dir(db_instance) exporter = EXPORT_FORMATS[dst_format] output_base = '%s_%s' % ('dataset' if save_images else 'annotations', @@ -52,39 +63,48 @@ def export_task(task_id, dst_format, server_url=None, save_images=False): output_path = '%s.%s' % (output_base, exporter.EXT) output_path = osp.join(cache_dir, output_path) - task_time = timezone.localtime(db_task.updated_date).timestamp() + task_time = timezone.localtime(db_instance.updated_date).timestamp() if not (osp.exists(output_path) and \ task_time <= osp.getmtime(output_path)): os.makedirs(cache_dir, exist_ok=True) with tempfile.TemporaryDirectory(dir=cache_dir) as temp_dir: temp_file = osp.join(temp_dir, 'result') - task.export_task(task_id, temp_file, dst_format, + export_fn(db_instance.id, temp_file, dst_format, server_url=server_url, save_images=save_images) os.replace(temp_file, output_path) archive_ctime = osp.getctime(output_path) scheduler = django_rq.get_scheduler() - cleaning_job = scheduler.enqueue_in(time_delta=CACHE_TTL, + cleaning_job = scheduler.enqueue_in(time_delta=cache_ttl, func=clear_export_cache, task_id=task_id, file_path=output_path, file_ctime=archive_ctime) - slogger.task[task_id].info( - "The task '{}' is exported as '{}' at '{}' " + logger.info( + "The {} '{}' is exported as '{}' at '{}' " "and available for downloading for the next {}. " "Export cache cleaning job is enqueued, id '{}'".format( - db_task.name, dst_format, output_path, CACHE_TTL, - cleaning_job.id)) + "project" if isinstance(db_instance, Project) else 'task', + db_instance.name, dst_format, output_path, cache_ttl, + cleaning_job.id + )) return output_path except Exception: - log_exception(slogger.task[task_id]) + log_exception(logger) raise def export_task_as_dataset(task_id, dst_format=None, server_url=None): - return export_task(task_id, dst_format, server_url=server_url, save_images=True) + return export(dst_format, task_id=task_id, server_url=server_url, save_images=True) def export_task_annotations(task_id, dst_format=None, server_url=None): - return export_task(task_id, dst_format, server_url=server_url, save_images=False) + return export(dst_format,task_id=task_id, server_url=server_url, save_images=False) + +def export_project_as_dataset(project_id, dst_format=None, server_url=None): + return export(dst_format, project_id=project_id, server_url=server_url, save_images=True) + + +def export_project_annotation(project_id, dst_format=None, server_url=None): + return export(dst_format, project_id=project_id, server_url=server_url, save_images=False) def clear_export_cache(task_id, file_path, file_ctime): try: diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index b1815d7d1e9..db79a3fbab6 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -291,6 +291,75 @@ def tasks(self, request, pk): return Response(serializer.data) + @swagger_auto_schema(method='get', operation_summary='Export project as a dataset in a specific format', + manual_parameters=[ + openapi.Parameter('format', openapi.IN_QUERY, + description="Desired output format name\nYou can get the list of supported formats at:\n/server/annotation/formats", + type=openapi.TYPE_STRING, required=True), + openapi.Parameter('filename', openapi.IN_QUERY, + description="Desired output file name", + type=openapi.TYPE_STRING, required=False), + openapi.Parameter('action', in_=openapi.IN_QUERY, + description='Used to start downloading process after annotation file had been created', + type=openapi.TYPE_STRING, required=False, enum=['download']) + ], + responses={'202': openapi.Response(description='Exporting has been started'), + '201': openapi.Response(description='Output file is ready for downloading'), + '200': openapi.Response(description='Download of file started'), + '405': openapi.Response(description='Format is not available'), + } + ) + @action(detail=True, methods=['GET'], serializer_class=None, + url_path='dataset') + def dataset_export(self, request, pk): + db_project = self.get_object() # force to call check_object_permissions + + format_name = request.query_params.get("format", "") + return _export_annotations(db_instance=db_project, + rq_id="/api/v1/project/{}/dataset/{}".format(pk, format_name), + request=request, + action=request.query_params.get("action", "").lower(), + callback=dm.views.export_project_as_dataset, + format_name=format_name, + filename=request.query_params.get("filename", "").lower(), + ) + + @swagger_auto_schema(method='get', operation_summary='Method allows to download project annotations', + manual_parameters=[ + openapi.Parameter('format', openapi.IN_QUERY, + description="Desired output format name\nYou can get the list of supported formats at:\n/server/annotation/formats", + type=openapi.TYPE_STRING, required=False), + openapi.Parameter('filename', openapi.IN_QUERY, + description="Desired output file name", + type=openapi.TYPE_STRING, required=False), + openapi.Parameter('action', in_=openapi.IN_QUERY, + description='Used to start downloading process after annotation file had been created', + type=openapi.TYPE_STRING, required=False, enum=['download']) + ], + responses={ + '202': openapi.Response(description='Dump of annotations has been started'), + '201': openapi.Response(description='Annotations file is ready to download'), + '200': openapi.Response(description='Download of file started'), + '405': openapi.Response(description='Format is not available'), + } + ) + @action(detail=True, methods=['GET'], + serializer_class=LabeledDataSerializer) + def annotations(self, request, pk): + db_project = self.get_object() # force to call check_object_permissions + format_name = request.query_params.get('format') + if format_name: + return _export_annotations(db_instance=db_project, + rq_id="/api/v1/projects/{}/annotations/{}".format(pk, format_name), + request=request, + action=request.query_params.get("action", "").lower(), + callback=dm.views.export_project_annotation, + format_name=format_name, + filename=request.query_params.get("filename", "").lower(), + ) + else: + return Response(status=500) + class TaskFilter(filters.FilterSet): project = filters.CharFilter(field_name="project__name", lookup_expr="icontains") name = filters.CharFilter(field_name="name", lookup_expr="icontains") @@ -553,7 +622,7 @@ def annotations(self, request, pk): if request.method == 'GET': format_name = request.query_params.get('format') if format_name: - return _export_annotations(db_task=db_task, + return _export_annotations(db_instance=db_task, rq_id="/api/v1/tasks/{}/annotations/{}".format(pk, format_name), request=request, action=request.query_params.get("action", "").lower(), @@ -676,7 +745,7 @@ def dataset_export(self, request, pk): db_task = self.get_object() # force to call check_object_permissions format_name = request.query_params.get("format", "") - return _export_annotations(db_task=db_task, + return _export_annotations(db_instance=db_task, rq_id="/api/v1/tasks/{}/dataset/{}".format(pk, format_name), request=request, action=request.query_params.get("action", "").lower(), @@ -1048,7 +1117,7 @@ def _import_annotations(request, rq_id, rq_func, pk, format_name): return Response(status=status.HTTP_202_ACCEPTED) -def _export_annotations(db_task, rq_id, request, format_name, action, callback, filename): +def _export_annotations(db_instance, rq_id, request, format_name, action, callback, filename): if action not in {"", "download"}: raise serializers.ValidationError( "Unexpected action specified for the request") @@ -1065,7 +1134,7 @@ def _export_annotations(db_task, rq_id, request, format_name, action, callback, rq_job = queue.fetch_job(rq_id) if rq_job: - last_task_update_time = timezone.localtime(db_task.updated_date) + last_task_update_time = timezone.localtime(db_instance.updated_date) request_time = rq_job.meta.get('request_time', None) if request_time is None or request_time < last_task_update_time: rq_job.cancel() @@ -1079,9 +1148,11 @@ def _export_annotations(db_task, rq_id, request, format_name, action, callback, timestamp = datetime.strftime(last_task_update_time, "%Y_%m_%d_%H_%M_%S") filename = filename or \ - "task_{}-{}-{}{}".format( - db_task.name, timestamp, - format_name, osp.splitext(file_path)[1]) + "{}_{}-{}-{}{}".format( + "project" if isinstance(db_instance, models.Project) else "task", + db_instance.name, timestamp, + format_name, osp.splitext(file_path)[1] + ) return sendfile(request, file_path, attachment=True, attachment_filename=filename.lower()) else: @@ -1102,9 +1173,9 @@ def _export_annotations(db_task, rq_id, request, format_name, action, callback, except Exception: server_address = None - ttl = dm.views.CACHE_TTL.total_seconds() + ttl = (dm.views.PROJECT_CACHE_TTL if isinstance(db_instance, Project) else dm.views.TASK_CACHE_TTL).total_seconds() queue.enqueue_call(func=callback, - args=(db_task.id, format_name, server_address), job_id=rq_id, + args=(db_instance.id, format_name, server_address), job_id=rq_id, meta={ 'request_time': timezone.localtime() }, result_ttl=ttl, failure_ttl=ttl) return Response(status=status.HTTP_202_ACCEPTED) From 46e89b05f8dccd2e99fca4b67be6955b03182b2e Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 23 Jun 2021 06:32:53 +0300 Subject: [PATCH 02/63] Fixed typos --- cvat/apps/dataset_manager/bindings.py | 32 ++++++++++++++++++------ cvat/apps/dataset_manager/formats/mot.py | 8 +++--- cvat/apps/dataset_manager/project.py | 2 +- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 786e12c837a..af9f0ef149d 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: MIT +import sys import os.path as osp from collections import namedtuple from typing import Any, Callable, Dict, List, Mapping, NamedTuple, OrderedDict, Tuple, Union @@ -12,7 +13,7 @@ import datumaro.components.extractor as datumaro from cvat.apps.engine.frame_provider import FrameProvider -from cvat.apps.engine.models import AttributeType, LabeledShape, ShapeType, Project, Task, Label +from cvat.apps.engine.models import AttributeType, ShapeType, Project, Task, Label from datumaro.util import cast from datumaro.util.image import ByteImage, Image @@ -478,8 +479,8 @@ def match_frame_fuzzy(self, path): class ProjectData(InstanceLabelData): # TODO: strictify - LabledShape = NamedTuple('LabledShape', [('type',Any), ('frame',Any), ('label',Any), ('points',Any), ('occluded',Any), ('attributes',Any), ('source',Any), ('group',Any), ('z_order',Any)]) - LabledShape.__new__.__defaults__ = (0,0) + LabeledShape = NamedTuple('LabledShape', [('type',Any), ('frame',Any), ('label',Any), ('points',Any), ('occluded',Any), ('attributes',Any), ('source',Any), ('group',Any), ('z_order',Any)]) + LabeledShape.__new__.__defaults__ = (0,0) TrackedShape = NamedTuple('TrackedShape', [('type',Any), ('frame',Any), ('points',Any), ('occluded',Any), ('outside',Any), ('keyframe',Any), ('attributes',Any), ('source',Any), ('group',Any), ('z_order',Any), ('label',Any), ('track_id',Any)], ) @@ -605,7 +606,7 @@ def _init_meta(self): ("dumped", str(timezone.localtime(timezone.now()))) ]) - def _export_tracked_shape(self, shape: dict, task_id: int) -> TrackedShape: + def _export_tracked_shape(self, shape: dict, task_id: int): return ProjectData.TrackedShape( type=shape["type"], frame=self.abs_frame_id(task_id, shape["frame"]), @@ -621,7 +622,7 @@ def _export_tracked_shape(self, shape: dict, task_id: int) -> TrackedShape: attributes=self._export_attributes(shape["attributes"]), ) - def _export_labeled_shape(self, shape: dict, task_id: int) -> LabeledShape: + def _export_labeled_shape(self, shape: dict, task_id: int): return ProjectData.LabeledShape( type=shape["type"], label=self._get_label_name(shape["label_id"]), @@ -634,7 +635,7 @@ def _export_labeled_shape(self, shape: dict, task_id: int) -> LabeledShape: attributes=self._export_attributes(shape["attributes"]), ) - def _export_tag(self, tag: dict, task_id: int) -> Tag: + def _export_tag(self, tag: dict, task_id: int): return ProjectData.Tag( frame=self.abs_frame_id(task_id, tag["frame"]), label=self._get_label_name(tag["label_id"]), @@ -859,7 +860,8 @@ def _make_image(i, **kwargs): dm_image = Image(**image_args) dm_anno = self._read_cvat_anno(frame_data, project_data) dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0], - annotation=dm_anno, image=dm_image, subset=frame_data.subset, + annotations=dm_anno, image=dm_image, + subset=get_defaulted_subset(frame_data.subset, project_data.subsets), attributes={'frame': frame_data.frame} ) dm_items.append(dm_item) @@ -912,6 +914,22 @@ def CVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_image class CvatImportError(Exception): pass + +def get_defaulted_subset(subset: str, subsets: List[str]) -> str: + if subset: + return subset + else: + if datumaro.DEFAULT_SUBSET_NAME not in subsets: + return datumaro.DEFAULT_SUBSET_NAME + else: + i = 1 + while i < sys.maxsize: + if f'{datumaro.DEFAULT_SUBSET_NAME}_{i}' not in subsets: + return f'{datumaro.DEFAULT_SUBSET_NAME}_{i}' + i += 1 + raise Exception('Cannot find deafoult name for subset') + + def convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label): item_anno = [] diff --git a/cvat/apps/dataset_manager/formats/mot.py b/cvat/apps/dataset_manager/formats/mot.py index 29d5182a674..53b5ccda010 100644 --- a/cvat/apps/dataset_manager/formats/mot.py +++ b/cvat/apps/dataset_manager/formats/mot.py @@ -8,16 +8,16 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor +from cvat.apps.dataset_manager.bindings import CVATDataExtractor from cvat.apps.dataset_manager.util import make_zip_archive from .registry import dm_env, exporter, importer @exporter(name='MOT', ext='ZIP', version='1.1') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(CVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'mot_seq_gt', save_images=save_images) diff --git a/cvat/apps/dataset_manager/project.py b/cvat/apps/dataset_manager/project.py index 8ec138f4a9f..866a75d47e8 100644 --- a/cvat/apps/dataset_manager/project.py +++ b/cvat/apps/dataset_manager/project.py @@ -21,7 +21,7 @@ def export_project(project_id, dst_file, format_name, # more dump request received at the same time: # https://github.com/opencv/cvat/issues/217 with transaction.atomic(): - project = TaskAnnotation(project_id) + project = ProjectAnnotation(project_id) project.init_from_db() exporter = make_exporter(format_name) From 7f4a34932711a6475f96584d70d018336456c183 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 24 Jun 2021 16:32:03 +0300 Subject: [PATCH 03/63] Fixed cache time value --- cvat/apps/engine/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index b44a6474d75..d57b105afe9 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -544,7 +544,7 @@ def retrieve(self, request, pk=None): else: return Response(status=status.HTTP_202_ACCEPTED) - ttl = dm.views.CACHE_TTL.total_seconds() + ttl = dm.views.TASK_CACHE_TTL.total_seconds() queue.enqueue_call( func=dm.views.backup_task, args=(pk, 'task_dump.zip'), From 4a5bc3a2f44ca29bb1f2a3858a08d8b0912ef4c3 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Mon, 28 Jun 2021 13:03:46 +0300 Subject: [PATCH 04/63] Added MOTS PNG --- cvat/apps/dataset_manager/formats/mots.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cvat/apps/dataset_manager/formats/mots.py b/cvat/apps/dataset_manager/formats/mots.py index 22b9dd08c7e..0ed0467fa05 100644 --- a/cvat/apps/dataset_manager/formats/mots.py +++ b/cvat/apps/dataset_manager/formats/mots.py @@ -8,7 +8,7 @@ from datumaro.components.extractor import AnnotationType, Transform from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, find_dataset_root, match_dm_item) from cvat.apps.dataset_manager.util import make_zip_archive @@ -21,9 +21,9 @@ def transform_item(self, item): if 'track_id' in a.attributes]) @exporter(name='MOTS PNG', ext='ZIP', version='1.0') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(CVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) dataset.transform(KeepTracks) # can only export tracks dataset.transform('polygons_to_masks') dataset.transform('boxes_to_masks') From 04d14c5b856ba8b37f3e8186f81f6ca9fcf132a7 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Fri, 2 Jul 2021 17:39:32 +0300 Subject: [PATCH 05/63] temp --- cvat-ui/src/actions/export-actions.ts | 15 ++++++++ .../components/actions-menu/dump-submenu.tsx | 2 +- .../export-dataset/export-dataset-modal.tsx | 34 +++++++++++++++++++ .../components/project-page/project-page.tsx | 2 ++ .../components/projects-page/actions-menu.tsx | 4 ++- .../projects-page/projects-page.tsx | 6 ++-- cvat-ui/src/reducers/export-reducer.ts | 30 ++++++++++++++++ cvat-ui/src/reducers/formats-reducer.ts | 2 +- cvat-ui/src/reducers/interfaces.ts | 21 ++++++++++++ cvat-ui/src/reducers/root-reducer.ts | 4 ++- 10 files changed, 114 insertions(+), 6 deletions(-) create mode 100644 cvat-ui/src/actions/export-actions.ts create mode 100644 cvat-ui/src/components/export-dataset/export-dataset-modal.tsx create mode 100644 cvat-ui/src/reducers/export-reducer.ts diff --git a/cvat-ui/src/actions/export-actions.ts b/cvat-ui/src/actions/export-actions.ts new file mode 100644 index 00000000000..fc4284108cf --- /dev/null +++ b/cvat-ui/src/actions/export-actions.ts @@ -0,0 +1,15 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import { ActionUnion, createAction } from 'utils/redux'; + +export enum ExportActionTypes { + TOGGLE_EXPORT_MODAL_VISIBLE = 'TOGGLE_EXPORT_MODAL_VISIBLE', +} + +export const exportActions = { + toggleExportModalVisible: () => createAction(ExportActionTypes.TOGGLE_EXPORT_MODAL_VISIBLE), +}; + +export type ExportActions = ActionUnion; diff --git a/cvat-ui/src/components/actions-menu/dump-submenu.tsx b/cvat-ui/src/components/actions-menu/dump-submenu.tsx index 91721ac6bb8..e8d6ca1a28e 100644 --- a/cvat-ui/src/components/actions-menu/dump-submenu.tsx +++ b/cvat-ui/src/components/actions-menu/dump-submenu.tsx @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation // // SPDX-License-Identifier: MIT diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx new file mode 100644 index 00000000000..b7f7eac3280 --- /dev/null +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -0,0 +1,34 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import React from 'react'; +import Modal from 'antd/lib/modal'; +import { useSelector, useDispatch } from 'react-redux'; +import Text from 'antd/lib/typography/Text'; + +import { CombinedState } from 'reducers/interfaces'; +import { exportActions } from 'actions/export-actions'; +import getCore from 'cvat-core-wrapper'; + +const core = getCore(); + +type ExportDatasetModalProps = { + instance: any; +}; + +export default function ExportDatasetModal(props: ExportDatasetModalProps): JSX.Element { + const { instance } = props; + const dispatch = useDispatch(); + const modalVisible = useSelector((state: CombinedState) => state.export.modalVisible); + + return ( + dispatch(exportActions.toggleExportModalVisible())} + > + Placeholder + + ); +} diff --git a/cvat-ui/src/components/project-page/project-page.tsx b/cvat-ui/src/components/project-page/project-page.tsx index 385d58c519a..6e83df904ed 100644 --- a/cvat-ui/src/components/project-page/project-page.tsx +++ b/cvat-ui/src/components/project-page/project-page.tsx @@ -16,6 +16,7 @@ import { PlusOutlined } from '@ant-design/icons'; import { CombinedState, Task } from 'reducers/interfaces'; import { getProjectsAsync } from 'actions/projects-actions'; import { cancelInferenceAsync } from 'actions/models-actions'; +import ExportDatasetModal from 'components/export-dataset/export-dataset-modal'; import TaskItem from 'components/tasks-page/task-item'; import DetailsComponent from './details'; import ProjectTopBar from './top-bar'; @@ -109,6 +110,7 @@ export default function ProjectPageComponent(): JSX.Element { ))} + ); } diff --git a/cvat-ui/src/components/projects-page/actions-menu.tsx b/cvat-ui/src/components/projects-page/actions-menu.tsx index 35fefea5ef1..a7c858885f7 100644 --- a/cvat-ui/src/components/projects-page/actions-menu.tsx +++ b/cvat-ui/src/components/projects-page/actions-menu.tsx @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation // // SPDX-License-Identifier: MIT @@ -8,6 +8,7 @@ import Modal from 'antd/lib/modal'; import Menu from 'antd/lib/menu'; import { deleteProjectAsync } from 'actions/projects-actions'; +import { exportActions } from 'actions/export-actions'; interface Props { projectInstance: any; @@ -37,6 +38,7 @@ export default function ProjectActionsMenuComponent(props: Props): JSX.Element { return ( Delete + dispatch(exportActions.toggleExportModalVisible())}>Export dataset ); } diff --git a/cvat-ui/src/components/projects-page/projects-page.tsx b/cvat-ui/src/components/projects-page/projects-page.tsx index 5d4d129a9af..a2842348af5 100644 --- a/cvat-ui/src/components/projects-page/projects-page.tsx +++ b/cvat-ui/src/components/projects-page/projects-page.tsx @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation // // SPDX-License-Identifier: MIT @@ -8,9 +8,10 @@ import { useDispatch, useSelector } from 'react-redux'; import { useLocation, useHistory } from 'react-router'; import Spin from 'antd/lib/spin'; -import FeedbackComponent from 'components/feedback/feedback'; import { CombinedState, ProjectsQuery } from 'reducers/interfaces'; import { getProjectsAsync } from 'actions/projects-actions'; +import FeedbackComponent from 'components/feedback/feedback'; +import ExportDatasetModal from 'components/export-dataset/export-dataset-modal'; import EmptyListComponent from './empty-list'; import TopBarComponent from './top-bar'; import ProjectListComponent from './project-list'; @@ -55,6 +56,7 @@ export default function ProjectsPageComponent(): JSX.Element { {projectsCount ? : } + ); } diff --git a/cvat-ui/src/reducers/export-reducer.ts b/cvat-ui/src/reducers/export-reducer.ts new file mode 100644 index 00000000000..ff84f611b42 --- /dev/null +++ b/cvat-ui/src/reducers/export-reducer.ts @@ -0,0 +1,30 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import { ExportActions, ExportActionTypes } from 'actions/export-actions'; +import { ExportState } from './interfaces'; + +const defaultState: ExportState = { + tasks: { + datasets: {}, + annotation: {}, + }, + projects: { + datasets: {}, + annotation: {}, + }, + modalVisible: false, +}; + +export default (state: ExportState = defaultState, action: ExportActions): ExportState => { + switch (action.type) { + case ExportActionTypes.TOGGLE_EXPORT_MODAL_VISIBLE: + return { + ...state, + modalVisible: !state.modalVisible, + }; + default: + return state; + } +}; diff --git a/cvat-ui/src/reducers/formats-reducer.ts b/cvat-ui/src/reducers/formats-reducer.ts index 855caaa53df..8260e976ad5 100644 --- a/cvat-ui/src/reducers/formats-reducer.ts +++ b/cvat-ui/src/reducers/formats-reducer.ts @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation // // SPDX-License-Identifier: MIT diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index eaae78a484a..32155547933 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -112,6 +112,26 @@ export interface TasksState { }; } +export interface ExportState { + tasks: { + datasets: { + [tid: number]: string[]; + }; + annotation: { + [tid: number]: string[]; + }; + }; + projects: { + datasets: { + [tid: number]: string[]; + }; + annotation: { + [tid: number]: string[]; + }; + }; + modalVisible: boolean; +} + export interface FormatsState { annotationFormats: any; fetching: boolean; @@ -616,6 +636,7 @@ export interface CombinedState { settings: SettingsState; shortcuts: ShortcutsState; review: ReviewState; + export: ExportState; } export enum DimensionType { diff --git a/cvat-ui/src/reducers/root-reducer.ts b/cvat-ui/src/reducers/root-reducer.ts index 04358b44e63..b1219b7a0b2 100644 --- a/cvat-ui/src/reducers/root-reducer.ts +++ b/cvat-ui/src/reducers/root-reducer.ts @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation // // SPDX-License-Identifier: MIT @@ -17,6 +17,7 @@ import settingsReducer from './settings-reducer'; import shortcutsReducer from './shortcuts-reducer'; import userAgreementsReducer from './useragreements-reducer'; import reviewReducer from './review-reducer'; +import exportReducer from './export-reducer'; export default function createRootReducer(): Reducer { return combineReducers({ @@ -34,5 +35,6 @@ export default function createRootReducer(): Reducer { shortcuts: shortcutsReducer, userAgreements: userAgreementsReducer, review: reviewReducer, + export: exportReducer, }); } From f6b833f5fa8cb148ef8a740bf66ac56afad991ba Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 7 Jul 2021 11:37:03 +0300 Subject: [PATCH 06/63] Another temp --- .vscode/launch.json | 19 +--- .vscode/tasks.json | 28 +++++- cvat-core/src/annotations.js | 23 +++-- cvat-core/src/project.js | 23 +++++ cvat-core/src/server-proxy.js | 51 +++++++---- cvat-ui/src/actions/export-actions.ts | 36 +++++++- .../export-dataset/export-dataset-modal.tsx | 91 ++++++++++++++++--- .../components/project-page/project-page.tsx | 2 +- .../components/projects-page/actions-menu.tsx | 6 +- cvat-ui/src/reducers/export-reducer.ts | 12 ++- cvat-ui/src/reducers/interfaces.ts | 1 + 11 files changed, 230 insertions(+), 62 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 658abd2fe48..d0dbfb8afd7 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,7 +7,7 @@ { "type": "chrome", "request": "launch", - "preLaunchTask": "ui.js: server", + "preLaunchTask": "npm: start - cvat-ui", "name": "ui.js: debug", "url": "http://localhost:3000", "webRoot": "${workspaceFolder}/cvat-ui", @@ -45,7 +45,8 @@ "python": "${command:python.interpreterPath}", "program": "${workspaceRoot}/manage.py", "env": { - "CVAT_SERVERLESS": "1", + // "CVAT_SERVERLESS": "1", + // "DJANGO_CONFIGURATION": "profiled_dev" }, "args": [ "runserver", @@ -57,19 +58,6 @@ "cwd": "${workspaceFolder}", "console": "internalConsole" }, - { - "name": "server: chrome", - "type": "chrome", - "request": "launch", - "url": "http://localhost:7000/", - "disableNetworkCache":true, - "trace": true, - "showAsyncStacks": true, - "pathMapping":{ - "/static/engine/": "${workspaceFolder}/cvat/apps/engine/static/engine/", - "/static/dashboard/": "${workspaceFolder}/cvat/apps/dashboard/static/dashboard/", - } - }, { "name": "server: RQ - default", "type": "python", @@ -207,7 +195,6 @@ { "name": "server: debug", "configurations": [ - "server: chrome", "server: django", "server: RQ - default", "server: RQ - low", diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 10c34d90ea1..901782322d7 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -4,11 +4,35 @@ "version": "2.0.0", "tasks": [ { - "label": "ui.js: server", "type": "npm", "script": "start", "path": "cvat-ui/", - "problemMatcher": [] + "label": "npm: start - cvat-ui", + "detail": "webpack-dev-server --env.API_URL=http://localhost:7000 --config ./webpack.config.js --mode=development", + "promptOnClose": true, + "isBackground": true, + "problemMatcher": { + "owner": "webpack", + "severity": "error", + "fileLocation": "absolute", + "pattern": [ + { + "regexp": "ERROR in (.*)", + "file": 1 + }, + { + "regexp": "\\((\\d+),(\\d+)\\):(.*)", + "line": 1, + "column": 2, + "message": 3 + } + ], + "background": { + "activeOnStart": true, + "beginsPattern": "webpack-dev-server", + "endsPattern": "Compiled" + } + } } ] } \ No newline at end of file diff --git a/cvat-core/src/annotations.js b/cvat-core/src/annotations.js index 991ee33eed7..7390fc0edca 100644 --- a/cvat-core/src/annotations.js +++ b/cvat-core/src/annotations.js @@ -1,14 +1,16 @@ -// Copyright (C) 2019-2020 Intel Corporation +// Copyright (C) 2019-2021 Intel Corporation // // SPDX-License-Identifier: MIT +const { Project } = require('./project'); + (() => { const serverProxy = require('./server-proxy'); const Collection = require('./annotations-collection'); const AnnotationsSaver = require('./annotations-saver'); const AnnotationsHistory = require('./annotations-history'); const { checkObjectType } = require('./common'); - const { Task } = require('./session'); + const { Task, Job } = require('./session'); const { Loader, Dumper } = require('./annotation-formats'); const { ScriptingError, DataError, ArgumentError } = require('./exceptions'); @@ -274,16 +276,25 @@ ); } - async function exportDataset(session, format) { + async function exportDataset(instance, format, name, saveImages = false) { if (!(format instanceof String || typeof format === 'string')) { throw new ArgumentError('Format must be a string'); } - if (!(session instanceof Task)) { - throw new ArgumentError('A dataset can only be created from a task'); + if (!(instance instanceof Task || instance instanceof Project || instance instanceof Job)) { + throw new ArgumentError('A dataset can only be created from a task or project'); + } + if (typeof saveImages !== 'boolean') { + throw new ArgumentError('Save images parameter must be a boolean'); } let result = null; - result = await serverProxy.tasks.exportDataset(session.id, format); + if (instance instanceof Task) { + result = await serverProxy.tasks.exportDataset(instance.id, format, name, saveImages); + } else if (instance instanceof Project) { + result = await serverProxy.projects.exportDataset(instance.id, format, name, saveImages); + } else { + result = await serverProxy.tasks.exportDataset(instance.task.id, format, name, saveImages); + } return result; } diff --git a/cvat-core/src/project.js b/cvat-core/src/project.js index b66eab49652..aab57010073 100644 --- a/cvat-core/src/project.js +++ b/cvat-core/src/project.js @@ -2,7 +2,9 @@ // // SPDX-License-Identifier: MIT + (() => { + const { exportDataset } = require('./annotations'); const PluginRegistry = require('./plugins'); const serverProxy = require('./server-proxy'); const { ArgumentError } = require('./exceptions'); @@ -308,6 +310,22 @@ Project, }; + Object.defineProperties(Project.prototype, Object.freeze({ + annotations: Object.freeze({ + value: { + async exportDataset(format, saveImages) { + const result = await PluginRegistry.apiWrapper.call( + this, + Project.prototype.annotations.exportDataset, + format, + saveImages, + ); + return result; + }, + }, + }), + })); + Project.prototype.save.implementation = async function () { const trainingProjectCopy = this.trainingProject; if (typeof this.id !== 'undefined') { @@ -357,4 +375,9 @@ const frameData = await getPreview(this._internalData.task_ids[0]); return frameData; }; + + Project.prototype.annotations.exportDataset.implementation = async function (format, saveImages) { + const result = exportDataset(this, format, '', saveImages); + return result; + }; })(); diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index 524ceeafd70..67b266dd6fd 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -465,29 +465,39 @@ } } - async function exportDataset(id, format) { - const { backendAPI } = config; - let url = `${backendAPI}/tasks/${id}/dataset?format=${format}`; + function exportDataset(instanceType) { + return async function (id, format, name, saveImages) { + const { backendAPI } = config; + const baseURL = `${backendAPI}/${instanceType}/${id}/${saveImages ? 'annotation' : 'dataset'}`; + let query = `format=${encodeURIComponent(format)}`; + if (name) { + const filename = name.replace(/\//g, '_'); + query += `&filename=${encodeURIComponent(filename)}`; + } + let url = `${baseURL}?${query}`; - return new Promise((resolve, reject) => { - async function request() { - try { - const response = await Axios.get(`${url}`, { + return new Promise((resolve, reject) => { + async function request() { + Axios.get(`${url}`, { proxy: config.proxy, - }); - if (response.status === 202) { - setTimeout(request, 3000); - } else { - url = `${url}&action=download`; - resolve(url); - } - } catch (errorData) { - reject(generateError(errorData)); + }) + .then((response) => { + if (response.status === 202) { + setTimeout(request, 3000); + } else { + query = `${query}&action=download`; + url = `${baseURL}?${query}`; + resolve(url); + } + }) + .catch((errorData) => { + reject(generateError(errorData)); + }); } - } - setTimeout(request); - }); + setTimeout(request); + }); + }; } async function exportTask(id) { @@ -1199,6 +1209,7 @@ save: saveProject, create: createProject, delete: deleteProject, + exportDataset: exportDataset('projects'), }), writable: false, }, @@ -1209,7 +1220,7 @@ saveTask, createTask, deleteTask, - exportDataset, + exportDataset: exportDataset('tasks'), exportTask, importTask, }), diff --git a/cvat-ui/src/actions/export-actions.ts b/cvat-ui/src/actions/export-actions.ts index fc4284108cf..48ae8233935 100644 --- a/cvat-ui/src/actions/export-actions.ts +++ b/cvat-ui/src/actions/export-actions.ts @@ -2,14 +2,44 @@ // // SPDX-License-Identifier: MIT -import { ActionUnion, createAction } from 'utils/redux'; +import { ActionUnion, createAction, ThunkAction } from 'utils/redux'; export enum ExportActionTypes { - TOGGLE_EXPORT_MODAL_VISIBLE = 'TOGGLE_EXPORT_MODAL_VISIBLE', + OPEN_EXPORT_MODAL = 'OPEN_EXPORT_MODAL', + CLOSE_EXPORT_MODAL = 'CLOSE_EXPORT_MODAL', + EXPORT_DATASET = 'EXPORT_DATASET', + EXPORT_DATASET_SUCCESS = 'EXPORT_DATASET_SUCCESS', + EXPORT_DATASET_FAILED = 'EXPORT_DATASET_FAILED', } export const exportActions = { - toggleExportModalVisible: () => createAction(ExportActionTypes.TOGGLE_EXPORT_MODAL_VISIBLE), + openExportModal: (instance: any) => createAction(ExportActionTypes.OPEN_EXPORT_MODAL, { instance }), + closeExportModal: () => createAction(ExportActionTypes.CLOSE_EXPORT_MODAL), + exportDataset: (instance: any, exporter: any) => + createAction(ExportActionTypes.EXPORT_DATASET, { instance, exporter }), + exportDatasetSuccess: (instance: any, exporter: any) => + createAction(ExportActionTypes.EXPORT_DATASET_SUCCESS, { instance, exporter }), + exportDatasetFailed: (instance: any, exporter: any, error: any) => + createAction(ExportActionTypes.EXPORT_DATASET_FAILED, { instance, exporter, error }), +}; + +export const exportDatasetAsync = ( + instance: any, + exporter: any, + name: string, + saveImages: boolean, +): ThunkAction => async (dispatch) => { + dispatch(exportActions.exportDataset(instance, exporter)); + + try { + const url = await instance.annotations.exportDataset(exporter.name, saveImages); + const downloadAnchor = window.document.getElementById('downloadAnchor') as HTMLAnchorElement; + downloadAnchor.href = url; + downloadAnchor.click(); + dispatch(exportActions.exportDatasetSuccess(instance, exporter)); + } catch (error) { + dispatch(exportActions.exportDatasetFailed(instance, exporter, error)); + } }; export type ExportActions = ActionUnion; diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx index b7f7eac3280..e6d58db77e0 100644 --- a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -2,33 +2,102 @@ // // SPDX-License-Identifier: MIT -import React from 'react'; +import React, { useState } from 'react'; import Modal from 'antd/lib/modal'; import { useSelector, useDispatch } from 'react-redux'; +import { DownloadOutlined, LoadingOutlined } from '@ant-design/icons'; import Text from 'antd/lib/typography/Text'; +import Select from 'antd/lib/select'; +import Checkbox from 'antd/lib/checkbox'; +import Input from 'antd/lib/input'; import { CombinedState } from 'reducers/interfaces'; -import { exportActions } from 'actions/export-actions'; +import { exportActions, exportDatasetAsync } from 'actions/export-actions'; import getCore from 'cvat-core-wrapper'; const core = getCore(); -type ExportDatasetModalProps = { - instance: any; -}; - -export default function ExportDatasetModal(props: ExportDatasetModalProps): JSX.Element { - const { instance } = props; +export default function ExportDatasetModal(): JSX.Element { + let instanceName = ''; + let activities: string[] = []; const dispatch = useDispatch(); + const [saveImages, setSaveImages] = useState(false); + const [selectedFormat, setSelectedFormat] = useState(); + const [customName, setCustomName] = useState(); + const instance = useSelector((state: CombinedState) => state.export.instance); const modalVisible = useSelector((state: CombinedState) => state.export.modalVisible); + const dumpers = useSelector((state: CombinedState) => state.formats.annotationFormats.dumpers); + const { + tasks: { + datasets: taskExportActivities, + annotation: taskDumpActivities, + }, + projects: { + datasets: projectExportActivities, + annotation: projectDumpActivities, + }, + } = useSelector((state: CombinedState) => state.export); + + if (instance instanceof core.classes.Project) { + instanceName = 'project'; + activities = (saveImages ? projectExportActivities : projectDumpActivities)[instance.id]; + } else if (instance instanceof core.classes.Task) { + instanceName = 'task'; + activities = (saveImages ? taskExportActivities : taskDumpActivities)[instance.id]; + } + + const handleExport = (): void => { + dispatch(exportDatasetAsync(instance, selectedFormat, customName || '', saveImages)); + dispatch(exportActions.closeExportModal()); + }; return ( dispatch(exportActions.toggleExportModalVisible())} + onCancel={() => dispatch(exportActions.closeExportModal())} + onOk={handleExport} > - Placeholder + + setSaveImages(e.target.checked)} + > + Save images + + setCustomName(e.target.value)} + /> ); } diff --git a/cvat-ui/src/components/project-page/project-page.tsx b/cvat-ui/src/components/project-page/project-page.tsx index 6e83df904ed..199ab975893 100644 --- a/cvat-ui/src/components/project-page/project-page.tsx +++ b/cvat-ui/src/components/project-page/project-page.tsx @@ -110,7 +110,7 @@ export default function ProjectPageComponent(): JSX.Element { ))} - + ); } diff --git a/cvat-ui/src/components/projects-page/actions-menu.tsx b/cvat-ui/src/components/projects-page/actions-menu.tsx index a7c858885f7..ea3059f5123 100644 --- a/cvat-ui/src/components/projects-page/actions-menu.tsx +++ b/cvat-ui/src/components/projects-page/actions-menu.tsx @@ -38,7 +38,11 @@ export default function ProjectActionsMenuComponent(props: Props): JSX.Element { return ( Delete - dispatch(exportActions.toggleExportModalVisible())}>Export dataset + dispatch(exportActions.openExportModal(projectInstance))} + > + Export dataset + ); } diff --git a/cvat-ui/src/reducers/export-reducer.ts b/cvat-ui/src/reducers/export-reducer.ts index ff84f611b42..a0a8eef1362 100644 --- a/cvat-ui/src/reducers/export-reducer.ts +++ b/cvat-ui/src/reducers/export-reducer.ts @@ -14,15 +14,23 @@ const defaultState: ExportState = { datasets: {}, annotation: {}, }, + instance: null, modalVisible: false, }; export default (state: ExportState = defaultState, action: ExportActions): ExportState => { switch (action.type) { - case ExportActionTypes.TOGGLE_EXPORT_MODAL_VISIBLE: + case ExportActionTypes.OPEN_EXPORT_MODAL: return { ...state, - modalVisible: !state.modalVisible, + modalVisible: true, + instance: action.payload.instance, + }; + case ExportActionTypes.CLOSE_EXPORT_MODAL: + return { + ...state, + modalVisible: false, + instance: null, }; default: return state; diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index 32155547933..326187a2002 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -129,6 +129,7 @@ export interface ExportState { [tid: number]: string[]; }; }; + instance: any; modalVisible: boolean; } From d45bec893e68fb5688640c73b4d554ab0bc7d72d Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 7 Jul 2021 12:55:33 +0300 Subject: [PATCH 07/63] WIP on UI --- cvat-core/src/annotations.js | 3 +- cvat-core/src/project.js | 3 +- cvat-ui/src/actions/export-actions.ts | 10 +-- cvat-ui/src/actions/tasks-actions.ts | 78 +++++++++---------- .../export-dataset/export-dataset-modal.tsx | 2 +- cvat-ui/src/reducers/notifications-reducer.ts | 12 ++- 6 files changed, 57 insertions(+), 51 deletions(-) diff --git a/cvat-core/src/annotations.js b/cvat-core/src/annotations.js index 7390fc0edca..edc616da1db 100644 --- a/cvat-core/src/annotations.js +++ b/cvat-core/src/annotations.js @@ -2,8 +2,6 @@ // // SPDX-License-Identifier: MIT -const { Project } = require('./project'); - (() => { const serverProxy = require('./server-proxy'); const Collection = require('./annotations-collection'); @@ -11,6 +9,7 @@ const { Project } = require('./project'); const AnnotationsHistory = require('./annotations-history'); const { checkObjectType } = require('./common'); const { Task, Job } = require('./session'); + const { Project } = require('./project'); const { Loader, Dumper } = require('./annotation-formats'); const { ScriptingError, DataError, ArgumentError } = require('./exceptions'); diff --git a/cvat-core/src/project.js b/cvat-core/src/project.js index aab57010073..d5a2fcd7eb2 100644 --- a/cvat-core/src/project.js +++ b/cvat-core/src/project.js @@ -4,7 +4,6 @@ (() => { - const { exportDataset } = require('./annotations'); const PluginRegistry = require('./plugins'); const serverProxy = require('./server-proxy'); const { ArgumentError } = require('./exceptions'); @@ -310,6 +309,8 @@ Project, }; + const { exportDataset } = require('./annotations'); + Object.defineProperties(Project.prototype, Object.freeze({ annotations: Object.freeze({ value: { diff --git a/cvat-ui/src/actions/export-actions.ts b/cvat-ui/src/actions/export-actions.ts index 48ae8233935..3362e6bf297 100644 --- a/cvat-ui/src/actions/export-actions.ts +++ b/cvat-ui/src/actions/export-actions.ts @@ -25,20 +25,20 @@ export const exportActions = { export const exportDatasetAsync = ( instance: any, - exporter: any, + format: string, name: string, saveImages: boolean, ): ThunkAction => async (dispatch) => { - dispatch(exportActions.exportDataset(instance, exporter)); + dispatch(exportActions.exportDataset(instance, format)); try { - const url = await instance.annotations.exportDataset(exporter.name, saveImages); + const url = await instance.annotations.exportDataset(format, saveImages); const downloadAnchor = window.document.getElementById('downloadAnchor') as HTMLAnchorElement; downloadAnchor.href = url; downloadAnchor.click(); - dispatch(exportActions.exportDatasetSuccess(instance, exporter)); + dispatch(exportActions.exportDatasetSuccess(instance, format)); } catch (error) { - dispatch(exportActions.exportDatasetFailed(instance, exporter, error)); + dispatch(exportActions.exportDatasetFailed(instance, format, error)); } }; diff --git a/cvat-ui/src/actions/tasks-actions.ts b/cvat-ui/src/actions/tasks-actions.ts index c0fabf97ecd..99bcb2c406a 100644 --- a/cvat-ui/src/actions/tasks-actions.ts +++ b/cvat-ui/src/actions/tasks-actions.ts @@ -21,9 +21,9 @@ export enum TasksActionTypes { DUMP_ANNOTATIONS = 'DUMP_ANNOTATIONS', DUMP_ANNOTATIONS_SUCCESS = 'DUMP_ANNOTATIONS_SUCCESS', DUMP_ANNOTATIONS_FAILED = 'DUMP_ANNOTATIONS_FAILED', - EXPORT_DATASET = 'EXPORT_DATASET', - EXPORT_DATASET_SUCCESS = 'EXPORT_DATASET_SUCCESS', - EXPORT_DATASET_FAILED = 'EXPORT_DATASET_FAILED', + // EXPORT_DATASET = 'EXPORT_DATASET', + // EXPORT_DATASET_SUCCESS = 'EXPORT_DATASET_SUCCESS', + // EXPORT_DATASET_FAILED = 'EXPORT_DATASET_FAILED', DELETE_TASK = 'DELETE_TASK', DELETE_TASK_SUCCESS = 'DELETE_TASK_SUCCESS', DELETE_TASK_FAILED = 'DELETE_TASK_FAILED', @@ -263,42 +263,42 @@ export function importTaskAsync(file: File): ThunkAction, {}, {}, }; } -function exportDataset(task: any, exporter: any): AnyAction { - const action = { - type: TasksActionTypes.EXPORT_DATASET, - payload: { - task, - exporter, - }, - }; - - return action; -} - -function exportDatasetSuccess(task: any, exporter: any): AnyAction { - const action = { - type: TasksActionTypes.EXPORT_DATASET_SUCCESS, - payload: { - task, - exporter, - }, - }; - - return action; -} - -function exportDatasetFailed(task: any, exporter: any, error: any): AnyAction { - const action = { - type: TasksActionTypes.EXPORT_DATASET_FAILED, - payload: { - task, - exporter, - error, - }, - }; - - return action; -} +// function exportDataset(task: any, exporter: any): AnyAction { +// const action = { +// type: TasksActionTypes.EXPORT_DATASET, +// payload: { +// task, +// exporter, +// }, +// }; + +// return action; +// } + +// function exportDatasetSuccess(task: any, exporter: any): AnyAction { +// const action = { +// type: TasksActionTypes.EXPORT_DATASET_SUCCESS, +// payload: { +// task, +// exporter, +// }, +// }; + +// return action; +// } + +// function exportDatasetFailed(task: any, exporter: any, error: any): AnyAction { +// const action = { +// type: TasksActionTypes.EXPORT_DATASET_FAILED, +// payload: { +// task, +// exporter, +// error, +// }, +// }; + +// return action; +// } export function exportDatasetAsync(task: any, exporter: any): ThunkAction, {}, {}, AnyAction> { return async (dispatch: ActionCreator): Promise => { diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx index e6d58db77e0..efbbb9969db 100644 --- a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -88,7 +88,7 @@ export default function ExportDatasetModal(): JSX.Element { )} setSaveImages(e.target.checked)} > Save images diff --git a/cvat-ui/src/reducers/notifications-reducer.ts b/cvat-ui/src/reducers/notifications-reducer.ts index e09db0b2cd6..225034333f4 100644 --- a/cvat-ui/src/reducers/notifications-reducer.ts +++ b/cvat-ui/src/reducers/notifications-reducer.ts @@ -16,9 +16,13 @@ import { NotificationsActionType } from 'actions/notification-actions'; import { BoundariesActionTypes } from 'actions/boundaries-actions'; import { UserAgreementsActionTypes } from 'actions/useragreements-actions'; import { ReviewActionTypes } from 'actions/review-actions'; +import { ExportActionTypes } from 'actions/export-actions'; +import getCore from 'cvat-core-wrapper'; import { NotificationsState } from './interfaces'; +const core = getCore(); + const defaultState: NotificationsState = { errors: { auth: { @@ -308,8 +312,9 @@ export default function (state = defaultState, action: AnyAction): Notifications }, }; } - case TasksActionTypes.EXPORT_DATASET_FAILED: { - const taskID = action.payload.task.id; + case ExportActionTypes.EXPORT_DATASET_FAILED: { + const instanceID = action.payload.instance.id; + const instanceType = action.payload.instance instanceof core.classes.Project ? 'project' : 'task'; return { ...state, errors: { @@ -319,7 +324,8 @@ export default function (state = defaultState, action: AnyAction): Notifications exportingAsDataset: { message: 'Could not export dataset for the ' + - `task ${taskID}`, + `` + + `${instanceType} ${instanceID}`, reason: action.payload.error.toString(), }, }, From b23a9ba93d7655c13951df4db781581694c701bd Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 8 Jul 2021 13:22:42 +0300 Subject: [PATCH 08/63] Added working modal --- cvat-core/src/annotations.js | 10 +- cvat-core/src/api.js | 1 + cvat-core/src/project-implementation.js | 70 ++++++++ cvat-core/src/project.js | 101 +++--------- cvat-core/src/server-proxy.js | 2 +- .../export-dataset/export-dataset-modal.tsx | 149 ++++++++++-------- .../src/components/export-dataset/styles.scss | 13 ++ 7 files changed, 202 insertions(+), 144 deletions(-) create mode 100644 cvat-core/src/project-implementation.js create mode 100644 cvat-ui/src/components/export-dataset/styles.scss diff --git a/cvat-core/src/annotations.js b/cvat-core/src/annotations.js index edc616da1db..aacea299d5d 100644 --- a/cvat-core/src/annotations.js +++ b/cvat-core/src/annotations.js @@ -8,8 +8,8 @@ const AnnotationsSaver = require('./annotations-saver'); const AnnotationsHistory = require('./annotations-history'); const { checkObjectType } = require('./common'); - const { Task, Job } = require('./session'); const { Project } = require('./project'); + const { Task, Job } = require('./session'); const { Loader, Dumper } = require('./annotation-formats'); const { ScriptingError, DataError, ArgumentError } = require('./exceptions'); @@ -51,6 +51,7 @@ stopFrame, frameMeta, }); + // eslint-disable-next-line no-unsanitized/method collection.import(rawAnnotations); const saver = new AnnotationsSaver(rawAnnotations.version, collection, session); @@ -254,6 +255,7 @@ const cache = getCache(sessionType); if (cache.has(session)) { + // eslint-disable-next-line no-unsanitized/method return cache.get(session).collection.import(data); } @@ -289,10 +291,10 @@ let result = null; if (instance instanceof Task) { result = await serverProxy.tasks.exportDataset(instance.id, format, name, saveImages); - } else if (instance instanceof Project) { - result = await serverProxy.projects.exportDataset(instance.id, format, name, saveImages); - } else { + } else if (instance instanceof Job) { result = await serverProxy.tasks.exportDataset(instance.task.id, format, name, saveImages); + } else { + result = await serverProxy.projects.exportDataset(instance.id, format, name, saveImages); } return result; diff --git a/cvat-core/src/api.js b/cvat-core/src/api.js index a5d36b6ce83..07fed4ac938 100644 --- a/cvat-core/src/api.js +++ b/cvat-core/src/api.js @@ -18,6 +18,7 @@ function build() { const Review = require('./review'); const { Job, Task } = require('./session'); const { Project } = require('./project'); + require('./project-implementation'); const { Attribute, Label } = require('./labels'); const MLModel = require('./ml-model'); diff --git a/cvat-core/src/project-implementation.js b/cvat-core/src/project-implementation.js new file mode 100644 index 00000000000..24b8f6ff109 --- /dev/null +++ b/cvat-core/src/project-implementation.js @@ -0,0 +1,70 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +// Copyright (C) 2019-2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +(() => { + const serverProxy = require('./server-proxy'); + const { getPreview } = require('./frames'); + + const { Project } = require('./project'); + const { exportDataset } = require('./annotations'); + + Project.prototype.save.implementation = async function () { + const trainingProjectCopy = this.trainingProject; + if (typeof this.id !== 'undefined') { + // project has been already created, need to update some data + const projectData = { + name: this.name, + assignee_id: this.assignee ? this.assignee.id : null, + bug_tracker: this.bugTracker, + labels: [...this._internalData.labels.map((el) => el.toJSON())], + }; + + if (trainingProjectCopy) { + projectData.training_project = trainingProjectCopy; + } + + await serverProxy.projects.save(this.id, projectData); + return this; + } + + // initial creating + const projectSpec = { + name: this.name, + labels: [...this.labels.map((el) => el.toJSON())], + }; + + if (this.bugTracker) { + projectSpec.bug_tracker = this.bugTracker; + } + + if (trainingProjectCopy) { + projectSpec.training_project = trainingProjectCopy; + } + + const project = await serverProxy.projects.create(projectSpec); + return new Project(project); + }; + + Project.prototype.delete.implementation = async function () { + const result = await serverProxy.projects.delete(this.id); + return result; + }; + + Project.prototype.preview.implementation = async function () { + if (!this._internalData.task_ids.length) { + return ''; + } + const frameData = await getPreview(this._internalData.task_ids[0]); + return frameData; + }; + + Project.prototype.annotations.exportDataset.implementation = async function (format, saveImages) { + const result = exportDataset(this, format, '', saveImages); + return result; + }; +})(); diff --git a/cvat-core/src/project.js b/cvat-core/src/project.js index d5a2fcd7eb2..658605252e2 100644 --- a/cvat-core/src/project.js +++ b/cvat-core/src/project.js @@ -2,14 +2,11 @@ // // SPDX-License-Identifier: MIT - (() => { const PluginRegistry = require('./plugins'); - const serverProxy = require('./server-proxy'); const { ArgumentError } = require('./exceptions'); const { Task } = require('./session'); const { Label } = require('./labels'); - const { getPreview } = require('./frames'); const User = require('./user'); /** @@ -255,6 +252,13 @@ }, }), ); + + // When we call a function, for example: task.annotations.get() + // In the method get we lose the task context + // So, we need return it + this.annotations = { + exportDataset: Object.getPrototypeOf(this).annotations.exportDataset.bind(this), + }; } /** @@ -305,80 +309,27 @@ } } - module.exports = { - Project, - }; - - const { exportDataset } = require('./annotations'); - - Object.defineProperties(Project.prototype, Object.freeze({ - annotations: Object.freeze({ - value: { - async exportDataset(format, saveImages) { - const result = await PluginRegistry.apiWrapper.call( - this, - Project.prototype.annotations.exportDataset, - format, - saveImages, - ); - return result; + Object.defineProperties( + Project.prototype, + Object.freeze({ + annotations: Object.freeze({ + value: { + async exportDataset(format, saveImages) { + const result = await PluginRegistry.apiWrapper.call( + this, + Project.prototype.annotations.exportDataset, + format, + saveImages, + ); + return result; + }, }, - }, + writable: true, + }), }), - })); - - Project.prototype.save.implementation = async function () { - const trainingProjectCopy = this.trainingProject; - if (typeof this.id !== 'undefined') { - // project has been already created, need to update some data - const projectData = { - name: this.name, - assignee_id: this.assignee ? this.assignee.id : null, - bug_tracker: this.bugTracker, - labels: [...this._internalData.labels.map((el) => el.toJSON())], - }; - - if (trainingProjectCopy) { - projectData.training_project = trainingProjectCopy; - } - - await serverProxy.projects.save(this.id, projectData); - return this; - } + ); - // initial creating - const projectSpec = { - name: this.name, - labels: [...this.labels.map((el) => el.toJSON())], - }; - - if (this.bugTracker) { - projectSpec.bug_tracker = this.bugTracker; - } - - if (trainingProjectCopy) { - projectSpec.training_project = trainingProjectCopy; - } - - const project = await serverProxy.projects.create(projectSpec); - return new Project(project); - }; - - Project.prototype.delete.implementation = async function () { - const result = await serverProxy.projects.delete(this.id); - return result; - }; - - Project.prototype.preview.implementation = async function () { - if (!this._internalData.task_ids.length) { - return ''; - } - const frameData = await getPreview(this._internalData.task_ids[0]); - return frameData; - }; - - Project.prototype.annotations.exportDataset.implementation = async function (format, saveImages) { - const result = exportDataset(this, format, '', saveImages); - return result; + module.exports = { + Project, }; })(); diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index 67b266dd6fd..bc6ed18ec0e 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -468,7 +468,7 @@ function exportDataset(instanceType) { return async function (id, format, name, saveImages) { const { backendAPI } = config; - const baseURL = `${backendAPI}/${instanceType}/${id}/${saveImages ? 'annotation' : 'dataset'}`; + const baseURL = `${backendAPI}/${instanceType}/${id}/${saveImages ? 'dataset' : 'annotation'}`; let query = `format=${encodeURIComponent(format)}`; if (name) { const filename = name.replace(/\//g, '_'); diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx index efbbb9969db..e84ad03781e 100644 --- a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -2,7 +2,8 @@ // // SPDX-License-Identifier: MIT -import React, { useState } from 'react'; +import './styles.scss'; +import React from 'react'; import Modal from 'antd/lib/modal'; import { useSelector, useDispatch } from 'react-redux'; import { DownloadOutlined, LoadingOutlined } from '@ant-design/icons'; @@ -10,6 +11,7 @@ import Text from 'antd/lib/typography/Text'; import Select from 'antd/lib/select'; import Checkbox from 'antd/lib/checkbox'; import Input from 'antd/lib/input'; +import Form from 'antd/lib/form'; import { CombinedState } from 'reducers/interfaces'; import { exportActions, exportDatasetAsync } from 'actions/export-actions'; @@ -17,87 +19,106 @@ import getCore from 'cvat-core-wrapper'; const core = getCore(); +type FormValues = { + selectedFormat: string | undefined; + saveImages: boolean; + customName: string | undefined; +}; + export default function ExportDatasetModal(): JSX.Element { - let instanceName = ''; + let instanceType = ''; let activities: string[] = []; + const [form] = Form.useForm(); const dispatch = useDispatch(); - const [saveImages, setSaveImages] = useState(false); - const [selectedFormat, setSelectedFormat] = useState(); - const [customName, setCustomName] = useState(); const instance = useSelector((state: CombinedState) => state.export.instance); const modalVisible = useSelector((state: CombinedState) => state.export.modalVisible); const dumpers = useSelector((state: CombinedState) => state.formats.annotationFormats.dumpers); const { - tasks: { - datasets: taskExportActivities, - annotation: taskDumpActivities, - }, - projects: { - datasets: projectExportActivities, - annotation: projectDumpActivities, - }, + tasks: { datasets: taskExportActivities, annotation: taskDumpActivities }, + projects: { datasets: projectExportActivities, annotation: projectDumpActivities }, } = useSelector((state: CombinedState) => state.export); if (instance instanceof core.classes.Project) { - instanceName = 'project'; - activities = (saveImages ? projectExportActivities : projectDumpActivities)[instance.id]; + instanceType = 'project'; + activities = (form.getFieldValue('saveImages') ? projectExportActivities : projectDumpActivities)[instance.id]; } else if (instance instanceof core.classes.Task) { - instanceName = 'task'; - activities = (saveImages ? taskExportActivities : taskDumpActivities)[instance.id]; + instanceType = 'task'; + activities = (form.getFieldValue('saveImages') ? taskExportActivities : taskDumpActivities)[instance.id]; } - const handleExport = (): void => { - dispatch(exportDatasetAsync(instance, selectedFormat, customName || '', saveImages)); + const closeModal = (): void => { + form.resetFields(); dispatch(exportActions.closeExportModal()); }; + const handleExport = (values: FormValues): void => { + // have to validate format before so it would not be undefined + dispatch( + exportDatasetAsync(instance, values.selectedFormat as string, values.customName || '', values.saveImages), + ); + closeModal(); + }; + return ( - dispatch(exportActions.closeExportModal())} - onOk={handleExport} +
- - setSaveImages(e.target.checked)} + form.submit()} > - Save images - - setCustomName(e.target.value)} - /> - + + + + + Save images + + + + + +
); } diff --git a/cvat-ui/src/components/export-dataset/styles.scss b/cvat-ui/src/components/export-dataset/styles.scss new file mode 100644 index 00000000000..26946bd0f8f --- /dev/null +++ b/cvat-ui/src/components/export-dataset/styles.scss @@ -0,0 +1,13 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +@import '../../base.scss'; + +.cvat-modal-export-option-item > .ant-select-item-option-content, +.cvat-modal-export-select .ant-select-selection-item { + > span[role='img'] { + color: $info-icon-color; + margin-right: $grid-unit-size; + } +} From 4c22e114c3b87fc8e93f466667b2b0a9f1031e45 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Fri, 9 Jul 2021 13:28:30 +0300 Subject: [PATCH 09/63] Moved task export to modal --- cvat-core/src/project-implementation.js | 4 +- cvat-core/src/project.js | 3 +- cvat-core/src/server-proxy.js | 3 +- cvat-core/src/session.js | 12 ++-- cvat-ui/src/actions/export-actions.ts | 25 ++++--- .../components/actions-menu/actions-menu.tsx | 21 +----- .../components/actions-menu/dump-submenu.tsx | 54 --------------- .../actions-menu/export-submenu.tsx | 47 ------------- .../top-bar/annotation-menu.tsx | 22 +----- .../export-dataset/export-dataset-modal.tsx | 37 ++++++---- .../src/components/task-page/task-page.tsx | 2 + .../src/components/tasks-page/tasks-page.tsx | 2 + .../containers/actions-menu/actions-menu.tsx | 34 +++------ .../top-bar/annotation-menu.tsx | 36 +++------- cvat-ui/src/reducers/export-reducer.ts | 69 ++++++++++++++++++- cvat-ui/src/reducers/interfaces.ts | 8 +-- cvat-ui/src/reducers/tasks-reducer.ts | 40 ----------- cvat/apps/dataset_manager/bindings.py | 2 +- 18 files changed, 150 insertions(+), 271 deletions(-) delete mode 100644 cvat-ui/src/components/actions-menu/dump-submenu.tsx delete mode 100644 cvat-ui/src/components/actions-menu/export-submenu.tsx diff --git a/cvat-core/src/project-implementation.js b/cvat-core/src/project-implementation.js index 24b8f6ff109..14bfadc3a9a 100644 --- a/cvat-core/src/project-implementation.js +++ b/cvat-core/src/project-implementation.js @@ -63,8 +63,8 @@ return frameData; }; - Project.prototype.annotations.exportDataset.implementation = async function (format, saveImages) { - const result = exportDataset(this, format, '', saveImages); + Project.prototype.annotations.exportDataset.implementation = async function (format, saveImages, customName) { + const result = exportDataset(this, format, customName, saveImages); return result; }; })(); diff --git a/cvat-core/src/project.js b/cvat-core/src/project.js index 658605252e2..23294be9652 100644 --- a/cvat-core/src/project.js +++ b/cvat-core/src/project.js @@ -314,12 +314,13 @@ Object.freeze({ annotations: Object.freeze({ value: { - async exportDataset(format, saveImages) { + async exportDataset(format, saveImages, customName = '') { const result = await PluginRegistry.apiWrapper.call( this, Project.prototype.annotations.exportDataset, format, saveImages, + customName, ); return result; }, diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index bc6ed18ec0e..082a430b937 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -468,7 +468,7 @@ function exportDataset(instanceType) { return async function (id, format, name, saveImages) { const { backendAPI } = config; - const baseURL = `${backendAPI}/${instanceType}/${id}/${saveImages ? 'dataset' : 'annotation'}`; + const baseURL = `${backendAPI}/${instanceType}/${id}/${saveImages ? 'dataset' : 'annotations'}`; let query = `format=${encodeURIComponent(format)}`; if (name) { const filename = name.replace(/\//g, '_'); @@ -1145,6 +1145,7 @@ const closureId = Date.now(); predictAnnotations.latestRequest.id = closureId; + // eslint-disable-next-line max-len const predicate = () => !predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId; if (predictAnnotations.latestRequest.fetching) { waitFor(5, predicate).then(() => { diff --git a/cvat-core/src/session.js b/cvat-core/src/session.js index a1136030fd9..88ff9388a37 100644 --- a/cvat-core/src/session.js +++ b/cvat-core/src/session.js @@ -148,11 +148,13 @@ return result; }, - async exportDataset(format) { + async exportDataset(format, saveImages, customName = '') { const result = await PluginRegistry.apiWrapper.call( this, prototype.annotations.exportDataset, format, + saveImages, + customName, ); return result; }, @@ -1953,8 +1955,8 @@ return result; }; - Job.prototype.annotations.exportDataset.implementation = async function (format) { - const result = await exportDataset(this.task, format); + Job.prototype.annotations.exportDataset.implementation = async function (format, saveImages, customName) { + const result = await exportDataset(this.task, format, customName, saveImages); return result; }; @@ -2267,8 +2269,8 @@ return result; }; - Task.prototype.annotations.exportDataset.implementation = async function (format) { - const result = await exportDataset(this, format); + Task.prototype.annotations.exportDataset.implementation = async function (format, saveImages, customName) { + const result = await exportDataset(this, format, customName, saveImages); return result; }; diff --git a/cvat-ui/src/actions/export-actions.ts b/cvat-ui/src/actions/export-actions.ts index 3362e6bf297..1e1e361c8fe 100644 --- a/cvat-ui/src/actions/export-actions.ts +++ b/cvat-ui/src/actions/export-actions.ts @@ -15,12 +15,17 @@ export enum ExportActionTypes { export const exportActions = { openExportModal: (instance: any) => createAction(ExportActionTypes.OPEN_EXPORT_MODAL, { instance }), closeExportModal: () => createAction(ExportActionTypes.CLOSE_EXPORT_MODAL), - exportDataset: (instance: any, exporter: any) => - createAction(ExportActionTypes.EXPORT_DATASET, { instance, exporter }), - exportDatasetSuccess: (instance: any, exporter: any) => - createAction(ExportActionTypes.EXPORT_DATASET_SUCCESS, { instance, exporter }), - exportDatasetFailed: (instance: any, exporter: any, error: any) => - createAction(ExportActionTypes.EXPORT_DATASET_FAILED, { instance, exporter, error }), + exportDataset: (instance: any, format: any, saveImages: boolean) => + createAction(ExportActionTypes.EXPORT_DATASET, { instance, format, saveImages }), + exportDatasetSuccess: (instance: any, format: any, saveImages: boolean) => + createAction(ExportActionTypes.EXPORT_DATASET_SUCCESS, { instance, format, saveImages }), + exportDatasetFailed: (instance: any, format: any, saveImages: boolean, error: any) => + createAction(ExportActionTypes.EXPORT_DATASET_FAILED, { + instance, + format, + saveImages, + error, + }), }; export const exportDatasetAsync = ( @@ -29,16 +34,16 @@ export const exportDatasetAsync = ( name: string, saveImages: boolean, ): ThunkAction => async (dispatch) => { - dispatch(exportActions.exportDataset(instance, format)); + dispatch(exportActions.exportDataset(instance, format, saveImages)); try { - const url = await instance.annotations.exportDataset(format, saveImages); + const url = await instance.annotations.exportDataset(format, saveImages, name); const downloadAnchor = window.document.getElementById('downloadAnchor') as HTMLAnchorElement; downloadAnchor.href = url; downloadAnchor.click(); - dispatch(exportActions.exportDatasetSuccess(instance, format)); + dispatch(exportActions.exportDatasetSuccess(instance, format, saveImages)); } catch (error) { - dispatch(exportActions.exportDatasetFailed(instance, format, error)); + dispatch(exportActions.exportDatasetFailed(instance, format, saveImages, error)); } }; diff --git a/cvat-ui/src/components/actions-menu/actions-menu.tsx b/cvat-ui/src/components/actions-menu/actions-menu.tsx index aa4d2acfd50..7c6bab8aab4 100644 --- a/cvat-ui/src/components/actions-menu/actions-menu.tsx +++ b/cvat-ui/src/components/actions-menu/actions-menu.tsx @@ -9,9 +9,7 @@ import Modal from 'antd/lib/modal'; import { LoadingOutlined } from '@ant-design/icons'; // eslint-disable-next-line import/no-extraneous-dependencies import { MenuInfo } from 'rc-menu/lib/interface'; -import DumpSubmenu from './dump-submenu'; import LoadSubmenu from './load-submenu'; -import ExportSubmenu from './export-submenu'; import { DimensionType } from '../../reducers/interfaces'; interface Props { @@ -30,7 +28,6 @@ interface Props { } export enum Actions { - DUMP_TASK_ANNO = 'dump_task_anno', LOAD_TASK_ANNO = 'load_task_anno', EXPORT_TASK_DATASET = 'export_task_dataset', DELETE_TASK = 'delete_task', @@ -43,14 +40,10 @@ export enum Actions { export default function ActionsMenuComponent(props: Props): JSX.Element { const { taskID, - taskMode, bugTracker, inferenceIsActive, - dumpers, loaders, onClickMenu, - dumpActivities, - exportActivities, loadActivity, taskDimension, exportIsActive, @@ -106,13 +99,6 @@ export default function ActionsMenuComponent(props: Props): JSX.Element { return ( - {DumpSubmenu({ - taskMode, - dumpers, - dumpActivities, - menuKey: Actions.DUMP_TASK_ANNO, - taskDimension, - })} {LoadSubmenu({ loaders, loadActivity, @@ -122,12 +108,7 @@ export default function ActionsMenuComponent(props: Props): JSX.Element { menuKey: Actions.LOAD_TASK_ANNO, taskDimension, })} - {ExportSubmenu({ - exporters: dumpers, - exportActivities, - menuKey: Actions.EXPORT_TASK_DATASET, - taskDimension, - })} + Export Task dataset {!!bugTracker && Open bug tracker} Automatic annotation diff --git a/cvat-ui/src/components/actions-menu/dump-submenu.tsx b/cvat-ui/src/components/actions-menu/dump-submenu.tsx deleted file mode 100644 index e8d6ca1a28e..00000000000 --- a/cvat-ui/src/components/actions-menu/dump-submenu.tsx +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (C) 2020-2021 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import React from 'react'; -import Menu from 'antd/lib/menu'; -import { DownloadOutlined, LoadingOutlined } from '@ant-design/icons'; -import Text from 'antd/lib/typography/Text'; -import { DimensionType } from '../../reducers/interfaces'; - -function isDefaultFormat(dumperName: string, taskMode: string): boolean { - return ( - (dumperName === 'CVAT for video 1.1' && taskMode === 'interpolation') || - (dumperName === 'CVAT for images 1.1' && taskMode === 'annotation') - ); -} - -interface Props { - taskMode: string; - menuKey: string; - dumpers: any[]; - dumpActivities: string[] | null; - taskDimension: DimensionType; -} - -export default function DumpSubmenu(props: Props): JSX.Element { - const { - taskMode, menuKey, dumpers, dumpActivities, taskDimension, - } = props; - - return ( - - {dumpers - .sort((a: any, b: any) => a.name.localeCompare(b.name)) - .filter((dumper: any): boolean => dumper.dimension === taskDimension) - .map( - (dumper: any): JSX.Element => { - const pending = (dumpActivities || []).includes(dumper.name); - const disabled = !dumper.enabled || pending; - const isDefault = isDefaultFormat(dumper.name, taskMode); - return ( - - - - {dumper.name} - - {pending && } - - ); - }, - )} - - ); -} diff --git a/cvat-ui/src/components/actions-menu/export-submenu.tsx b/cvat-ui/src/components/actions-menu/export-submenu.tsx deleted file mode 100644 index 68356500915..00000000000 --- a/cvat-ui/src/components/actions-menu/export-submenu.tsx +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import React from 'react'; -import Menu from 'antd/lib/menu'; -import Text from 'antd/lib/typography/Text'; -import { ExportOutlined, LoadingOutlined } from '@ant-design/icons'; -import { DimensionType } from '../../reducers/interfaces'; - -interface Props { - menuKey: string; - exporters: any[]; - exportActivities: string[] | null; - taskDimension: DimensionType; -} - -export default function ExportSubmenu(props: Props): JSX.Element { - const { - menuKey, exporters, exportActivities, taskDimension, - } = props; - - return ( - - {exporters - .sort((a: any, b: any) => a.name.localeCompare(b.name)) - .filter((exporter: any): boolean => exporter.dimension === taskDimension) - .map( - (exporter: any): JSX.Element => { - const pending = (exportActivities || []).includes(exporter.name); - const disabled = !exporter.enabled || pending; - return ( - - - {exporter.name} - {pending && } - - ); - }, - )} - - ); -} diff --git a/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx b/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx index 258cf2b2335..d2bc001cd52 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx @@ -8,9 +8,7 @@ import Modal from 'antd/lib/modal'; // eslint-disable-next-line import/no-extraneous-dependencies import { MenuInfo } from 'rc-menu/lib/interface'; -import DumpSubmenu from 'components/actions-menu/dump-submenu'; import LoadSubmenu from 'components/actions-menu/load-submenu'; -import ExportSubmenu from 'components/actions-menu/export-submenu'; import { DimensionType } from '../../../reducers/interfaces'; interface Props { @@ -28,7 +26,6 @@ interface Props { } export enum Actions { - DUMP_TASK_ANNO = 'dump_task_anno', LOAD_JOB_ANNO = 'load_job_anno', EXPORT_TASK_DATASET = 'export_task_dataset', REMOVE_ANNO = 'remove_anno', @@ -41,12 +38,8 @@ export enum Actions { export default function AnnotationMenuComponent(props: Props): JSX.Element { const { - taskMode, loaders, - dumpers, loadActivity, - dumpActivities, - exportActivities, isReviewer, jobInstance, onClickMenu, @@ -163,13 +156,6 @@ export default function AnnotationMenuComponent(props: Props): JSX.Element { return ( - {DumpSubmenu({ - taskMode, - dumpers, - dumpActivities, - menuKey: Actions.DUMP_TASK_ANNO, - taskDimension: jobInstance.task.dimension, - })} {LoadSubmenu({ loaders, loadActivity, @@ -179,13 +165,7 @@ export default function AnnotationMenuComponent(props: Props): JSX.Element { menuKey: Actions.LOAD_JOB_ANNO, taskDimension: jobInstance.task.dimension, })} - {ExportSubmenu({ - exporters: dumpers, - exportActivities, - menuKey: Actions.EXPORT_TASK_DATASET, - taskDimension: jobInstance.task.dimension, - })} - + Export Task dataset Remove annotations e.preventDefault()}> diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx index e84ad03781e..7bb2a2fc706 100644 --- a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -3,7 +3,7 @@ // SPDX-License-Identifier: MIT import './styles.scss'; -import React from 'react'; +import React, { useState, useEffect } from 'react'; import Modal from 'antd/lib/modal'; import { useSelector, useDispatch } from 'react-redux'; import { DownloadOutlined, LoadingOutlined } from '@ant-design/icons'; @@ -26,25 +26,35 @@ type FormValues = { }; export default function ExportDatasetModal(): JSX.Element { - let instanceType = ''; - let activities: string[] = []; + const [instanceType, setInstanceType] = useState(''); + const [activities, setActivities] = useState([]); const [form] = Form.useForm(); const dispatch = useDispatch(); const instance = useSelector((state: CombinedState) => state.export.instance); const modalVisible = useSelector((state: CombinedState) => state.export.modalVisible); const dumpers = useSelector((state: CombinedState) => state.formats.annotationFormats.dumpers); const { - tasks: { datasets: taskExportActivities, annotation: taskDumpActivities }, - projects: { datasets: projectExportActivities, annotation: projectDumpActivities }, + tasks: { datasets: taskExportActivities, annotations: taskDumpActivities }, + projects: { datasets: projectExportActivities, annotations: projectDumpActivities }, } = useSelector((state: CombinedState) => state.export); - if (instance instanceof core.classes.Project) { - instanceType = 'project'; - activities = (form.getFieldValue('saveImages') ? projectExportActivities : projectDumpActivities)[instance.id]; - } else if (instance instanceof core.classes.Task) { - instanceType = 'task'; - activities = (form.getFieldValue('saveImages') ? taskExportActivities : taskDumpActivities)[instance.id]; - } + const initActivities = (): void => { + if (instance instanceof core.classes.Project) { + setInstanceType('project'); + setActivities( + (form.getFieldValue('saveImages') ? projectExportActivities : projectDumpActivities)[instance.id] || [], + ); + } else if (instance instanceof core.classes.Task) { + setInstanceType('task'); + setActivities( + (form.getFieldValue('saveImages') ? taskExportActivities : taskDumpActivities)[instance.id] || [], + ); + } + }; + + useEffect(() => { + initActivities(); + }, [instance?.id, instance instanceof core.classes.Project]); const closeModal = (): void => { form.resetFields(); @@ -65,6 +75,7 @@ export default function ExportDatasetModal(): JSX.Element { form={form} labelCol={{ span: 8 }} wrapperCol={{ span: 16 }} + onValuesChange={initActivities} initialValues={ { selectedFormat: undefined, @@ -75,7 +86,7 @@ export default function ExportDatasetModal(): JSX.Element { onFinish={handleExport} > form.submit()} diff --git a/cvat-ui/src/components/task-page/task-page.tsx b/cvat-ui/src/components/task-page/task-page.tsx index 627dbf9d250..0465fa95d52 100644 --- a/cvat-ui/src/components/task-page/task-page.tsx +++ b/cvat-ui/src/components/task-page/task-page.tsx @@ -14,6 +14,7 @@ import DetailsContainer from 'containers/task-page/details'; import JobListContainer from 'containers/task-page/job-list'; import ModelRunnerModal from 'components/model-runner-modal/model-runner-dialog'; import MoveTaskModal from 'components/move-task-modal/move-task-modal'; +import ExportDatasetModal from 'components/export-dataset/export-dataset-modal'; import { Task } from 'reducers/interfaces'; import TopBarComponent from './top-bar'; @@ -85,6 +86,7 @@ class TaskPageComponent extends React.PureComponent { + {updating && } ); diff --git a/cvat-ui/src/components/tasks-page/tasks-page.tsx b/cvat-ui/src/components/tasks-page/tasks-page.tsx index 6e0df457716..1420822b1e4 100644 --- a/cvat-ui/src/components/tasks-page/tasks-page.tsx +++ b/cvat-ui/src/components/tasks-page/tasks-page.tsx @@ -14,6 +14,7 @@ import Text from 'antd/lib/typography/Text'; import { TasksQuery } from 'reducers/interfaces'; import FeedbackComponent from 'components/feedback/feedback'; import TaskListContainer from 'containers/tasks-page/tasks-list'; +import ExportDatasetModal from 'components/export-dataset/export-dataset-modal'; import TopBar from './top-bar'; import EmptyListComponent from './empty-list'; @@ -221,6 +222,7 @@ class TasksPageComponent extends React.PureComponent )} + ); } diff --git a/cvat-ui/src/containers/actions-menu/actions-menu.tsx b/cvat-ui/src/containers/actions-menu/actions-menu.tsx index 5923928c0a5..10d8c02329b 100644 --- a/cvat-ui/src/containers/actions-menu/actions-menu.tsx +++ b/cvat-ui/src/containers/actions-menu/actions-menu.tsx @@ -12,13 +12,12 @@ import { CombinedState } from 'reducers/interfaces'; import { modelsActions } from 'actions/models-actions'; import { - dumpAnnotationsAsync, loadAnnotationsAsync, - exportDatasetAsync, deleteTaskAsync, exportTaskAsync, switchMoveTaskModalVisible, } from 'actions/tasks-actions'; +import { exportActions } from 'actions/export-actions'; interface OwnProps { taskInstance: any; @@ -35,8 +34,7 @@ interface StateToProps { interface DispatchToProps { loadAnnotations: (taskInstance: any, loader: any, file: File) => void; - dumpAnnotations: (taskInstance: any, dumper: any) => void; - exportDataset: (taskInstance: any, exporter: any) => void; + showExportModal: (taskInstance: any) => void; deleteTask: (taskInstance: any) => void; openRunModelWindow: (taskInstance: any) => void; exportTask: (taskInstance: any) => void; @@ -72,11 +70,8 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { loadAnnotations: (taskInstance: any, loader: any, file: File): void => { dispatch(loadAnnotationsAsync(taskInstance, loader, file)); }, - dumpAnnotations: (taskInstance: any, dumper: any): void => { - dispatch(dumpAnnotationsAsync(taskInstance, dumper)); - }, - exportDataset: (taskInstance: any, exporter: any): void => { - dispatch(exportDatasetAsync(taskInstance, exporter)); + showExportModal: (taskInstance: any): void => { + dispatch(exportActions.openExportModal(taskInstance)); }, deleteTask: (taskInstance: any): void => { dispatch(deleteTaskAsync(taskInstance)); @@ -104,8 +99,7 @@ function ActionsMenuContainer(props: OwnProps & StateToProps & DispatchToProps): exportIsActive, loadAnnotations, - dumpAnnotations, - exportDataset, + showExportModal, deleteTask, openRunModelWindow, exportTask, @@ -115,28 +109,18 @@ function ActionsMenuContainer(props: OwnProps & StateToProps & DispatchToProps): function onClickMenu(params: MenuInfo, file?: File): void { if (params.keyPath.length > 1) { const [additionalKey, action] = params.keyPath; - if (action === Actions.DUMP_TASK_ANNO) { - const format = additionalKey; - const [dumper] = dumpers.filter((_dumper: any): boolean => _dumper.name === format); - if (dumper) { - dumpAnnotations(taskInstance, dumper); - } - } else if (action === Actions.LOAD_TASK_ANNO) { + if (action === Actions.LOAD_TASK_ANNO) { const format = additionalKey; const [loader] = loaders.filter((_loader: any): boolean => _loader.name === format); if (loader && file) { loadAnnotations(taskInstance, loader, file); } - } else if (action === Actions.EXPORT_TASK_DATASET) { - const format = additionalKey; - const [exporter] = dumpers.filter((_exporter: any): boolean => _exporter.name === format); - if (exporter) { - exportDataset(taskInstance, exporter); - } } } else { const [action] = params.keyPath; - if (action === Actions.DELETE_TASK) { + if (action === Actions.EXPORT_TASK_DATASET) { + showExportModal(taskInstance); + } else if (action === Actions.DELETE_TASK) { deleteTask(taskInstance); } else if (action === Actions.OPEN_BUG_TRACKER) { window.open(`${taskInstance.bugTracker}`, '_blank'); diff --git a/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx b/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx index a52d3e11a0f..2f40d414698 100644 --- a/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx +++ b/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation // // SPDX-License-Identifier: MIT @@ -10,7 +10,7 @@ import { MenuInfo } from 'rc-menu/lib/interface'; import { CombinedState, TaskStatus } from 'reducers/interfaces'; import AnnotationMenuComponent, { Actions } from 'components/annotation-page/top-bar/annotation-menu'; -import { dumpAnnotationsAsync, exportDatasetAsync, updateJobAsync } from 'actions/tasks-actions'; +import { updateJobAsync } from 'actions/tasks-actions'; import { uploadJobAnnotationsAsync, removeAnnotationsAsync, @@ -19,6 +19,7 @@ import { switchSubmitReviewDialog as switchSubmitReviewDialogAction, setForceExitAnnotationFlag as setForceExitAnnotationFlagAction, } from 'actions/annotation-actions'; +import { exportActions } from 'actions/export-actions'; interface StateToProps { annotationFormats: any; @@ -31,8 +32,7 @@ interface StateToProps { interface DispatchToProps { loadAnnotations(job: any, loader: any, file: File): void; - dumpAnnotations(task: any, dumper: any): void; - exportDataset(task: any, exporter: any): void; + showExportModal(task: any): void; removeAnnotations(sessionInstance: any): void; switchRequestReviewDialog(visible: boolean): void; switchSubmitReviewDialog(visible: boolean): void; @@ -72,11 +72,8 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { loadAnnotations(job: any, loader: any, file: File): void { dispatch(uploadJobAnnotationsAsync(job, loader, file)); }, - dumpAnnotations(task: any, dumper: any): void { - dispatch(dumpAnnotationsAsync(task, dumper)); - }, - exportDataset(task: any, exporter: any): void { - dispatch(exportDatasetAsync(task, exporter)); + showExportModal(task: any): void { + dispatch(exportActions.openExportModal(task)); }, removeAnnotations(sessionInstance: any): void { dispatch(removeAnnotationsAsync(sessionInstance)); @@ -111,8 +108,7 @@ function AnnotationMenuContainer(props: Props): JSX.Element { dumpActivities, exportActivities, loadAnnotations, - dumpAnnotations, - exportDataset, + showExportModal, removeAnnotations, switchRequestReviewDialog, switchSubmitReviewDialog, @@ -124,28 +120,18 @@ function AnnotationMenuContainer(props: Props): JSX.Element { const onClickMenu = (params: MenuInfo, file?: File): void => { if (params.keyPath.length > 1) { const [additionalKey, action] = params.keyPath; - if (action === Actions.DUMP_TASK_ANNO) { - const format = additionalKey; - const [dumper] = dumpers.filter((_dumper: any): boolean => _dumper.name === format); - if (dumper) { - dumpAnnotations(jobInstance.task, dumper); - } - } else if (action === Actions.LOAD_JOB_ANNO) { + if (action === Actions.LOAD_JOB_ANNO) { const format = additionalKey; const [loader] = loaders.filter((_loader: any): boolean => _loader.name === format); if (loader && file) { loadAnnotations(jobInstance, loader, file); } - } else if (action === Actions.EXPORT_TASK_DATASET) { - const format = additionalKey; - const [exporter] = dumpers.filter((_exporter: any): boolean => _exporter.name === format); - if (exporter) { - exportDataset(jobInstance.task, exporter); - } } } else { const [action] = params.keyPath; - if (action === Actions.REMOVE_ANNO) { + if (action === Actions.EXPORT_TASK_DATASET) { + showExportModal(jobInstance.task); + } else if (action === Actions.REMOVE_ANNO) { removeAnnotations(jobInstance); } else if (action === Actions.REQUEST_REVIEW) { switchRequestReviewDialog(true); diff --git a/cvat-ui/src/reducers/export-reducer.ts b/cvat-ui/src/reducers/export-reducer.ts index a0a8eef1362..b1e06c78c2d 100644 --- a/cvat-ui/src/reducers/export-reducer.ts +++ b/cvat-ui/src/reducers/export-reducer.ts @@ -3,16 +3,19 @@ // SPDX-License-Identifier: MIT import { ExportActions, ExportActionTypes } from 'actions/export-actions'; +import getCore from 'cvat-core-wrapper'; import { ExportState } from './interfaces'; +const core = getCore(); + const defaultState: ExportState = { tasks: { datasets: {}, - annotation: {}, + annotations: {}, }, projects: { datasets: {}, - annotation: {}, + annotations: {}, }, instance: null, modalVisible: false, @@ -32,6 +35,68 @@ export default (state: ExportState = defaultState, action: ExportActions): Expor modalVisible: false, instance: null, }; + case ExportActionTypes.EXPORT_DATASET: { + const { instance, format, saveImages } = action.payload; + let activities; + if (instance instanceof core.classes.Project) { + activities = saveImages ? state.projects.datasets : state.projects.annotations; + } else { + activities = saveImages ? state.tasks.datasets : state.tasks.annotations; + } + + activities[instance.id] = + instance.id in activities && !activities[instance.id].includes(format) ? + [...activities[instance.id], format] : + activities[instance.id] || [format]; + + return { + ...state, + tasks: { + datasets: instance instanceof core.classes.Task && saveImages ? activities : state.tasks.datasets, + annotations: + instance instanceof core.classes.Task && !saveImages ? activities : state.tasks.annotations, + }, + projects: { + datasets: + instance instanceof core.classes.Project && saveImages ? activities : state.projects.datasets, + annotations: + instance instanceof core.classes.Project && !saveImages ? + activities : + state.projects.annotations, + }, + }; + } + case ExportActionTypes.EXPORT_DATASET_FAILED: + case ExportActionTypes.EXPORT_DATASET_SUCCESS: { + const { instance, format, saveImages } = action.payload; + let activities; + if (instance instanceof core.classes.Project) { + activities = saveImages ? state.projects.datasets : state.projects.annotations; + } else { + activities = saveImages ? state.tasks.datasets : state.tasks.annotations; + } + + activities[instance.id] = activities[instance.id].filter( + (exporterName: string): boolean => exporterName !== format, + ); + + return { + ...state, + tasks: { + datasets: instance instanceof core.classes.Task && saveImages ? activities : state.tasks.datasets, + annotations: + instance instanceof core.classes.Task && !saveImages ? activities : state.tasks.annotations, + }, + projects: { + datasets: + instance instanceof core.classes.Project && saveImages ? activities : state.projects.datasets, + annotations: + instance instanceof core.classes.Project && !saveImages ? + activities : + state.projects.annotations, + }, + }; + } default: return state; } diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index 326187a2002..a6894f1180e 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -117,16 +117,16 @@ export interface ExportState { datasets: { [tid: number]: string[]; }; - annotation: { + annotations: { [tid: number]: string[]; }; }; projects: { datasets: { - [tid: number]: string[]; + [pid: number]: string[]; }; - annotation: { - [tid: number]: string[]; + annotations: { + [pid: number]: string[]; }; }; instance: any; diff --git a/cvat-ui/src/reducers/tasks-reducer.ts b/cvat-ui/src/reducers/tasks-reducer.ts index 78236132a9e..48edc997639 100644 --- a/cvat-ui/src/reducers/tasks-reducer.ts +++ b/cvat-ui/src/reducers/tasks-reducer.ts @@ -123,46 +123,6 @@ export default (state: TasksState = defaultState, action: AnyAction): TasksState }, }; } - case TasksActionTypes.EXPORT_DATASET: { - const { task } = action.payload; - const { exporter } = action.payload; - const { exports: activeExports } = state.activities; - - activeExports[task.id] = - task.id in activeExports && !activeExports[task.id].includes(exporter.name) ? - [...activeExports[task.id], exporter.name] : - activeExports[task.id] || [exporter.name]; - - return { - ...state, - activities: { - ...state.activities, - exports: { - ...activeExports, - }, - }, - }; - } - case TasksActionTypes.EXPORT_DATASET_FAILED: - case TasksActionTypes.EXPORT_DATASET_SUCCESS: { - const { task } = action.payload; - const { exporter } = action.payload; - const { exports: activeExports } = state.activities; - - activeExports[task.id] = activeExports[task.id].filter( - (exporterName: string): boolean => exporterName !== exporter.name, - ); - - return { - ...state, - activities: { - ...state.activities, - exports: { - ...activeExports, - }, - }, - }; - } case TasksActionTypes.LOAD_ANNOTATIONS: { const { task } = action.payload; const { loader } = action.payload; diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index af9f0ef149d..32fad137a2a 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -523,7 +523,7 @@ def abs_frame_id(self, task_id: int, relative_id: int) -> int: def rel_frame_id(self, task_id: int, absolute_id: int) -> int: task = self._db_tasks[task_id] d, m = divmod( - absolute_id - self._task_frame_offsets[task_id] - task.data.start_frame, self._frame_step) + absolute_id - task.data.start_frame, task.data.get_frame_step()) if m or d not in range(0, task.data.size): raise ValueError(f"Unknown frame {absolute_id}") return d From a4d609456490ebff467f02622e71ac418cc4e6bb Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Fri, 9 Jul 2021 13:31:36 +0300 Subject: [PATCH 10/63] Fixed vscode files --- .vscode/launch.json | 3 +-- .vscode/tasks.json | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index d0dbfb8afd7..7e1f0267b91 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -45,8 +45,7 @@ "python": "${command:python.interpreterPath}", "program": "${workspaceRoot}/manage.py", "env": { - // "CVAT_SERVERLESS": "1", - // "DJANGO_CONFIGURATION": "profiled_dev" + "CVAT_SERVERLESS": "1" }, "args": [ "runserver", diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 901782322d7..c8d8c11e88f 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -35,4 +35,4 @@ } } ] -} \ No newline at end of file +} From aadc83c7c356dd802c677bd92b25d4bdb0108c04 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Fri, 9 Jul 2021 13:36:49 +0300 Subject: [PATCH 11/63] Removed task actions --- cvat-ui/src/actions/tasks-actions.ts | 114 --------------------------- 1 file changed, 114 deletions(-) diff --git a/cvat-ui/src/actions/tasks-actions.ts b/cvat-ui/src/actions/tasks-actions.ts index 99bcb2c406a..468bfe8c351 100644 --- a/cvat-ui/src/actions/tasks-actions.ts +++ b/cvat-ui/src/actions/tasks-actions.ts @@ -18,12 +18,6 @@ export enum TasksActionTypes { LOAD_ANNOTATIONS = 'LOAD_ANNOTATIONS', LOAD_ANNOTATIONS_SUCCESS = 'LOAD_ANNOTATIONS_SUCCESS', LOAD_ANNOTATIONS_FAILED = 'LOAD_ANNOTATIONS_FAILED', - DUMP_ANNOTATIONS = 'DUMP_ANNOTATIONS', - DUMP_ANNOTATIONS_SUCCESS = 'DUMP_ANNOTATIONS_SUCCESS', - DUMP_ANNOTATIONS_FAILED = 'DUMP_ANNOTATIONS_FAILED', - // EXPORT_DATASET = 'EXPORT_DATASET', - // EXPORT_DATASET_SUCCESS = 'EXPORT_DATASET_SUCCESS', - // EXPORT_DATASET_FAILED = 'EXPORT_DATASET_FAILED', DELETE_TASK = 'DELETE_TASK', DELETE_TASK_SUCCESS = 'DELETE_TASK_SUCCESS', DELETE_TASK_FAILED = 'DELETE_TASK_FAILED', @@ -108,60 +102,6 @@ export function getTasksAsync(query: TasksQuery): ThunkAction, {}, }; } -function dumpAnnotation(task: any, dumper: any): AnyAction { - const action = { - type: TasksActionTypes.DUMP_ANNOTATIONS, - payload: { - task, - dumper, - }, - }; - - return action; -} - -function dumpAnnotationSuccess(task: any, dumper: any): AnyAction { - const action = { - type: TasksActionTypes.DUMP_ANNOTATIONS_SUCCESS, - payload: { - task, - dumper, - }, - }; - - return action; -} - -function dumpAnnotationFailed(task: any, dumper: any, error: any): AnyAction { - const action = { - type: TasksActionTypes.DUMP_ANNOTATIONS_FAILED, - payload: { - task, - dumper, - error, - }, - }; - - return action; -} - -export function dumpAnnotationsAsync(task: any, dumper: any): ThunkAction, {}, {}, AnyAction> { - return async (dispatch: ActionCreator): Promise => { - try { - dispatch(dumpAnnotation(task, dumper)); - const url = await task.annotations.dump(dumper); - const downloadAnchor = window.document.getElementById('downloadAnchor') as HTMLAnchorElement; - downloadAnchor.href = url; - downloadAnchor.click(); - } catch (error) { - dispatch(dumpAnnotationFailed(task, dumper, error)); - return; - } - - dispatch(dumpAnnotationSuccess(task, dumper)); - }; -} - function loadAnnotations(task: any, loader: any): AnyAction { const action = { type: TasksActionTypes.LOAD_ANNOTATIONS, @@ -263,60 +203,6 @@ export function importTaskAsync(file: File): ThunkAction, {}, {}, }; } -// function exportDataset(task: any, exporter: any): AnyAction { -// const action = { -// type: TasksActionTypes.EXPORT_DATASET, -// payload: { -// task, -// exporter, -// }, -// }; - -// return action; -// } - -// function exportDatasetSuccess(task: any, exporter: any): AnyAction { -// const action = { -// type: TasksActionTypes.EXPORT_DATASET_SUCCESS, -// payload: { -// task, -// exporter, -// }, -// }; - -// return action; -// } - -// function exportDatasetFailed(task: any, exporter: any, error: any): AnyAction { -// const action = { -// type: TasksActionTypes.EXPORT_DATASET_FAILED, -// payload: { -// task, -// exporter, -// error, -// }, -// }; - -// return action; -// } - -export function exportDatasetAsync(task: any, exporter: any): ThunkAction, {}, {}, AnyAction> { - return async (dispatch: ActionCreator): Promise => { - dispatch(exportDataset(task, exporter)); - - try { - const url = await task.annotations.exportDataset(exporter.name); - const downloadAnchor = window.document.getElementById('downloadAnchor') as HTMLAnchorElement; - downloadAnchor.href = url; - downloadAnchor.click(); - } catch (error) { - dispatch(exportDatasetFailed(task, exporter, error)); - } - - dispatch(exportDatasetSuccess(task, exporter)); - }; -} - function exportTask(taskID: number): AnyAction { const action = { type: TasksActionTypes.EXPORT_TASK, From bcd20343912f408ad69982b3ac7a830fbb33be90 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Mon, 12 Jul 2021 12:17:34 +0300 Subject: [PATCH 12/63] Fixed image_maker calling --- cvat/apps/dataset_manager/bindings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 32fad137a2a..7eaaa038b78 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -855,7 +855,7 @@ def _make_image(i, **kwargs): 'size': (frame_data.height, frame_data.width), } if include_images: - dm_image = _make_image(frame_data.idx, **image_args) + dm_image = image_maker_per_task[frame_data.task_id](frame_data.idx, **image_args) else: dm_image = Image(**image_args) dm_anno = self._read_cvat_anno(frame_data, project_data) From 3aa16a1c1b80c55df71e82949928ea6c4eade0e5 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Mon, 12 Jul 2021 13:07:11 +0300 Subject: [PATCH 13/63] Separated DataExtractor class --- cvat/apps/dataset_manager/bindings.py | 129 +++++++----------- cvat/apps/dataset_manager/formats/camvid.py | 4 +- cvat/apps/dataset_manager/formats/coco.py | 4 +- cvat/apps/dataset_manager/formats/icdar.py | 8 +- cvat/apps/dataset_manager/formats/imagenet.py | 4 +- cvat/apps/dataset_manager/formats/labelme.py | 4 +- .../dataset_manager/formats/market1501.py | 4 +- cvat/apps/dataset_manager/formats/mask.py | 4 +- cvat/apps/dataset_manager/formats/mot.py | 4 +- cvat/apps/dataset_manager/formats/mots.py | 4 +- .../dataset_manager/formats/pascal_voc.py | 4 +- cvat/apps/dataset_manager/formats/tfrecord.py | 4 +- cvat/apps/dataset_manager/formats/vggface2.py | 4 +- .../apps/dataset_manager/formats/widerface.py | 4 +- cvat/apps/dataset_manager/formats/yolo.py | 4 +- 15 files changed, 81 insertions(+), 108 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 7eaaa038b78..c37333c5038 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -735,10 +735,55 @@ def _get_filename(path): return osp.splitext(path)[0] -class CvatTaskDataExtractor(datumaro.SourceExtractor): +class CVATDataExtractor(datumaro.SourceExtractor): + dm_items: List[datumaro.DatasetItem] = [] + _categories: Dict[datumaro.AnnotationType, datumaro.LabelCategories] = {} + + def __init__(self): + super().__init__() + + def __iter__(self): + for item in self._items: + yield item + + def __len__(self): + return len(self._items) + + def categories(self): + return self._categories + + @staticmethod + def _load_categories(labels: list): + categories: Dict[datumaro.AnnotationType, datumaro.LabelCategories] = {} + + label_categories = datumaro.LabelCategories(attributes=['occluded']) + + for _, label in labels: + label_categories.add(label['name']) + for _, attr in label['attributes']: + label_categories.attributes.add(attr['name']) + + categories[datumaro.AnnotationType.label] = label_categories + + return categories + + + def _read_cvat_anno(self, cvat_frame_anno: Union[ProjectData.Frame, TaskData.Frame], labels: list): + categories = self.categories() + label_cat = categories[datumaro.AnnotationType.label] + def map_label(name): return label_cat.find(name)[0] + label_attrs = { + label['name']: label['attributes'] + for _, label in labels + } + + return convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label) + + +class CvatTaskDataExtractor(CVATDataExtractor): def __init__(self, task_data, include_images=False): super().__init__() - self._categories = self._load_categories(task_data) + self._categories = self._load_categories(task_data.meta['task']['labels']) dm_items = [] @@ -773,7 +818,7 @@ def _make_image(i, **kwargs): dm_image = _make_image(frame_data.idx, **image_args) else: dm_image = Image(**image_args) - dm_anno = self._read_cvat_anno(frame_data, task_data) + dm_anno = self._read_cvat_anno(frame_data, task_data.meta['task']['labels']) dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0], annotations=dm_anno, image=dm_image, attributes={'frame': frame_data.frame}) @@ -781,46 +826,10 @@ def _make_image(i, **kwargs): self._items = dm_items - def __iter__(self): - for item in self._items: - yield item - - def __len__(self): - return len(self._items) - - def categories(self): - return self._categories - - @staticmethod - def _load_categories(cvat_anno): - categories = {} - - label_categories = datumaro.LabelCategories(attributes=['occluded']) - - for _, label in cvat_anno.meta['task']['labels']: - label_categories.add(label['name']) - for _, attr in label['attributes']: - label_categories.attributes.add(attr['name']) - - categories[datumaro.AnnotationType.label] = label_categories - - return categories - - def _read_cvat_anno(self, cvat_frame_anno, task_data): - categories = self.categories() - label_cat = categories[datumaro.AnnotationType.label] - def map_label(name): return label_cat.find(name)[0] - label_attrs = { - label['name']: label['attributes'] - for _, label in task_data.meta['task']['labels'] - } - - return convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label) - class CVATProjectDataExtractor(datumaro.SourceExtractor): def __init__(self, project_data: ProjectData, include_images=False): super().__init__() - self._categories = self._load_categories(project_data) + self._categories = self._load_categories(project_data.meta['project']['labels']) dm_items: List[datumaro.DatasetItem] = [] @@ -858,7 +867,7 @@ def _make_image(i, **kwargs): dm_image = image_maker_per_task[frame_data.task_id](frame_data.idx, **image_args) else: dm_image = Image(**image_args) - dm_anno = self._read_cvat_anno(frame_data, project_data) + dm_anno = self._read_cvat_anno(frame_data, project_data.meta['project']['labels']) dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0], annotations=dm_anno, image=dm_image, subset=get_defaulted_subset(frame_data.subset, project_data.subsets), @@ -868,44 +877,8 @@ def _make_image(i, **kwargs): self._items = dm_items - def __iter__(self): - for item in self._items: - yield item - - def __len__(self): - return len(self._items) - - def categories(self): - return self._categories - - - @staticmethod - def _load_categories(project_data: ProjectData): - categories: Dict[datumaro.AnnotationType, datumaro.LabelCategories] = {} - - label_categories = datumaro.LabelCategories(attributes=['occluded']) - - for _, label in project_data.meta['project']['labels']: - label_categories.add(label['name']) - for _, attr in label['attributes']: - label_categories.attributes.add(attr['name']) - - categories[datumaro.AnnotationType.label] = label_categories - - return categories - - def _read_cvat_anno(self, cvat_frame_anno: ProjectData.Frame, project_data: ProjectData): - categories = self.categories() - label_cat = categories[datumaro.AnnotationType.label] - def map_label(name: str): return label_cat.find(name)[0] - label_attrs = { - label['name']: label['attributes'] - for _, label in project_data.meta['project']['labels'] - } - - return convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label) -def CVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_images: bool=False): +def GetCVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_images: bool=False): if isinstance(instance_data, ProjectData): return CVATProjectDataExtractor(instance_data, include_images) else: diff --git a/cvat/apps/dataset_manager/formats/camvid.py b/cvat/apps/dataset_manager/formats/camvid.py index d8d8c4d9041..2522f3fb226 100644 --- a/cvat/apps/dataset_manager/formats/camvid.py +++ b/cvat/apps/dataset_manager/formats/camvid.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive @@ -17,7 +17,7 @@ @exporter(name='CamVid', ext='ZIP', version='1.0') def _export(dst_file, instance_data, save_images=False): - dataset = Dataset.from_extractors(CVATDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) dataset.transform('polygons_to_masks') dataset.transform('boxes_to_masks') diff --git a/cvat/apps/dataset_manager/formats/coco.py b/cvat/apps/dataset_manager/formats/coco.py index dbd4a4946f0..927df2de567 100644 --- a/cvat/apps/dataset_manager/formats/coco.py +++ b/cvat/apps/dataset_manager/formats/coco.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import CVATDataExtractor, \ +from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, \ import_dm_annotations from cvat.apps.dataset_manager.util import make_zip_archive @@ -16,7 +16,7 @@ @exporter(name='COCO', ext='ZIP', version='1.0') def _export(dst_file, instance_data, save_images=False): - dataset = Dataset.from_extractors(CVATDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'coco_instances', save_images=save_images, diff --git a/cvat/apps/dataset_manager/formats/icdar.py b/cvat/apps/dataset_manager/formats/icdar.py index 8845ab411db..6bf099ec691 100644 --- a/cvat/apps/dataset_manager/formats/icdar.py +++ b/cvat/apps/dataset_manager/formats/icdar.py @@ -9,7 +9,7 @@ from datumaro.components.extractor import (AnnotationType, Caption, Label, LabelCategories, Transform) -from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive @@ -76,7 +76,7 @@ def transform_item(self, item): @exporter(name='ICDAR Recognition', ext='ZIP', version='1.0') def _export_recognition(dst_file, instance_data, save_images=False): - dataset = Dataset.from_extractors(CVATDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) dataset.transform(LabelToCaption) with TemporaryDirectory() as temp_dir: @@ -94,7 +94,7 @@ def _import(src_file, instance_data): @exporter(name='ICDAR Localization', ext='ZIP', version='1.0') def _export_localization(dst_file, instance_data, save_images=False): - dataset = Dataset.from_extractors(CVATDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'icdar_text_localization', save_images=save_images) @@ -112,7 +112,7 @@ def _import(src_file, instance_data): @exporter(name='ICDAR Segmentation', ext='ZIP', version='1.0') def _export_segmentation(dst_file, instance_data, save_images=False): - dataset = Dataset.from_extractors(CVATDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.transform('polygons_to_masks') diff --git a/cvat/apps/dataset_manager/formats/imagenet.py b/cvat/apps/dataset_manager/formats/imagenet.py index 8720a6cdf64..1085ef74535 100644 --- a/cvat/apps/dataset_manager/formats/imagenet.py +++ b/cvat/apps/dataset_manager/formats/imagenet.py @@ -9,7 +9,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import CVATDataExtractor, \ +from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, \ import_dm_annotations from cvat.apps.dataset_manager.util import make_zip_archive @@ -18,7 +18,7 @@ @exporter(name='ImageNet', ext='ZIP', version='1.0') def _export(dst_file, instance_data, save_images=False): - dataset = Dataset.from_extractors(CVATDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: if save_images: diff --git a/cvat/apps/dataset_manager/formats/labelme.py b/cvat/apps/dataset_manager/formats/labelme.py index 5a65b407942..2fc1f7f73c4 100644 --- a/cvat/apps/dataset_manager/formats/labelme.py +++ b/cvat/apps/dataset_manager/formats/labelme.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive @@ -16,7 +16,7 @@ @exporter(name='LabelMe', ext='ZIP', version='3.0') def _export(dst_file, instance_data, save_images=False): - dataset = Dataset.from_extractors(CVATDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'label_me', save_images=save_images) diff --git a/cvat/apps/dataset_manager/formats/market1501.py b/cvat/apps/dataset_manager/formats/market1501.py index ce170db719a..b4ea2999c8b 100644 --- a/cvat/apps/dataset_manager/formats/market1501.py +++ b/cvat/apps/dataset_manager/formats/market1501.py @@ -9,7 +9,7 @@ from datumaro.components.extractor import (AnnotationType, Label, LabelCategories, Transform) -from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive @@ -60,7 +60,7 @@ def transform_item(self, item): @exporter(name='Market-1501', ext='ZIP', version='1.0') def _export(dst_file, instance_data, save_images=False): - dataset = Dataset.from_extractors(CVATDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.transform(LabelAttrToAttr, 'market-1501') diff --git a/cvat/apps/dataset_manager/formats/mask.py b/cvat/apps/dataset_manager/formats/mask.py index 97ec836ef8b..67d61eed359 100644 --- a/cvat/apps/dataset_manager/formats/mask.py +++ b/cvat/apps/dataset_manager/formats/mask.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive @@ -17,7 +17,7 @@ @exporter(name='Segmentation mask', ext='ZIP', version='1.1') def _export(dst_file, instance_data, save_images=False): - dataset = Dataset.from_extractors(CVATDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) dataset.transform('polygons_to_masks') dataset.transform('boxes_to_masks') diff --git a/cvat/apps/dataset_manager/formats/mot.py b/cvat/apps/dataset_manager/formats/mot.py index 53b5ccda010..26fc7b0db61 100644 --- a/cvat/apps/dataset_manager/formats/mot.py +++ b/cvat/apps/dataset_manager/formats/mot.py @@ -8,7 +8,7 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import CVATDataExtractor +from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor from cvat.apps.dataset_manager.util import make_zip_archive from .registry import dm_env, exporter, importer @@ -16,7 +16,7 @@ @exporter(name='MOT', ext='ZIP', version='1.1') def _export(dst_file, instance_data, save_images=False): - dataset = Dataset.from_extractors(CVATDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'mot_seq_gt', save_images=save_images) diff --git a/cvat/apps/dataset_manager/formats/mots.py b/cvat/apps/dataset_manager/formats/mots.py index 0ed0467fa05..50bd4317e5d 100644 --- a/cvat/apps/dataset_manager/formats/mots.py +++ b/cvat/apps/dataset_manager/formats/mots.py @@ -8,7 +8,7 @@ from datumaro.components.extractor import AnnotationType, Transform from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, find_dataset_root, match_dm_item) from cvat.apps.dataset_manager.util import make_zip_archive @@ -22,7 +22,7 @@ def transform_item(self, item): @exporter(name='MOTS PNG', ext='ZIP', version='1.0') def _export(dst_file, instance_data, save_images=False): - dataset = Dataset.from_extractors(CVATDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) dataset.transform(KeepTracks) # can only export tracks dataset.transform('polygons_to_masks') diff --git a/cvat/apps/dataset_manager/formats/pascal_voc.py b/cvat/apps/dataset_manager/formats/pascal_voc.py index 3e040200836..93504628eb3 100644 --- a/cvat/apps/dataset_manager/formats/pascal_voc.py +++ b/cvat/apps/dataset_manager/formats/pascal_voc.py @@ -11,7 +11,7 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, ProjectData, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive @@ -20,7 +20,7 @@ @exporter(name='PASCAL VOC', ext='ZIP', version='1.1') def _export(dst_file, instance_data, save_images=False): - dataset = Dataset.from_extractors(CVATDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'voc', save_images=save_images, diff --git a/cvat/apps/dataset_manager/formats/tfrecord.py b/cvat/apps/dataset_manager/formats/tfrecord.py index 29270f09b23..d9c705a7542 100644 --- a/cvat/apps/dataset_manager/formats/tfrecord.py +++ b/cvat/apps/dataset_manager/formats/tfrecord.py @@ -6,7 +6,7 @@ from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive from datumaro.components.project import Dataset @@ -24,7 +24,7 @@ @exporter(name='TFRecord', ext='ZIP', version='1.0', enabled=tf_available) def _export(dst_file, instance_data, save_images=False): - dataset = Dataset.from_extractors(CVATDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'tf_detection_api', save_images=save_images) diff --git a/cvat/apps/dataset_manager/formats/vggface2.py b/cvat/apps/dataset_manager/formats/vggface2.py index 5f618739c1c..3e3ead13333 100644 --- a/cvat/apps/dataset_manager/formats/vggface2.py +++ b/cvat/apps/dataset_manager/formats/vggface2.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import CVATDataExtractor, \ +from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, \ import_dm_annotations from cvat.apps.dataset_manager.util import make_zip_archive @@ -16,7 +16,7 @@ @exporter(name='VGGFace2', ext='ZIP', version='1.0') def _export(dst_file, instance_data, save_images=False): - dataset = Dataset.from_extractors(CVATDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'vgg_face2', save_images=save_images) diff --git a/cvat/apps/dataset_manager/formats/widerface.py b/cvat/apps/dataset_manager/formats/widerface.py index 4dc3b9ab355..b578c14c7ce 100644 --- a/cvat/apps/dataset_manager/formats/widerface.py +++ b/cvat/apps/dataset_manager/formats/widerface.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import CVATDataExtractor, \ +from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, \ import_dm_annotations from cvat.apps.dataset_manager.util import make_zip_archive @@ -16,7 +16,7 @@ @exporter(name='WiderFace', ext='ZIP', version='1.0') def _export(dst_file, instance_data, save_images=False): - dataset = Dataset.from_extractors(CVATDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'wider_face', save_images=save_images) diff --git a/cvat/apps/dataset_manager/formats/yolo.py b/cvat/apps/dataset_manager/formats/yolo.py index 76251b29ba7..6327f3c04fb 100644 --- a/cvat/apps/dataset_manager/formats/yolo.py +++ b/cvat/apps/dataset_manager/formats/yolo.py @@ -8,7 +8,7 @@ from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CVATDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, import_dm_annotations, match_dm_item, find_dataset_root) from cvat.apps.dataset_manager.util import make_zip_archive from datumaro.components.extractor import DatasetItem @@ -20,7 +20,7 @@ @exporter(name='YOLO', ext='ZIP', version='1.1') def _export(dst_file, instance_data, save_images=False): - dataset = Dataset.from_extractors(CVATDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'yolo', save_images=save_images) From b59ab03d4b0f6b8f7fb200ce0a4b86f6a2f3110f Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 14 Jul 2021 14:08:39 +0300 Subject: [PATCH 14/63] wip --- cvat/apps/dataset_manager/formats/cvat.py | 19 ++++++++---- cvat/settings/base.py | 36 +++++++++++------------ 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/cvat/apps/dataset_manager/formats/cvat.py b/cvat/apps/dataset_manager/formats/cvat.py index 786a5025e7c..b66c8aab309 100644 --- a/cvat/apps/dataset_manager/formats/cvat.py +++ b/cvat/apps/dataset_manager/formats/cvat.py @@ -11,7 +11,7 @@ from datumaro.components.extractor import DatasetItem -from cvat.apps.dataset_manager.bindings import match_dm_item +from cvat.apps.dataset_manager.bindings import match_dm_item, ProjectData from cvat.apps.dataset_manager.util import make_zip_archive from cvat.apps.engine.frame_provider import FrameProvider @@ -42,8 +42,10 @@ def _add_version(self): self.xmlgen.characters(self.version) self.xmlgen.endElement("version") - def open_root(self): + def open_document(self): self.xmlgen.startDocument() + + def open_root(self): self.xmlgen.startElement("annotations", {}) self._level += 1 self._add_version() @@ -168,12 +170,14 @@ def close_root(self): self._level -= 1 self._indent() self.xmlgen.endElement("annotations") + + def close_document(self): self.xmlgen.endDocument() + return XmlAnnotationWriter(file_object) -def dump_as_cvat_annotation(file_object, annotations): - dumper = create_xml_dumper(file_object) +def dump_as_cvat_annotation(dumper, annotations): dumper.open_root() dumper.add_meta(annotations.meta) @@ -286,8 +290,7 @@ def dump_as_cvat_annotation(file_object, annotations): dumper.close_image() dumper.close_root() -def dump_as_cvat_interpolation(file_object, annotations): - dumper = create_xml_dumper(file_object) +def dump_as_cvat_interpolation(dumper, annotations): dumper.open_root() dumper.add_meta(annotations.meta) def dump_track(idx, track): @@ -527,6 +530,10 @@ def load(file_object, annotations): tag = None el.clear() +def dump_task_anno(dst_file, task_data, callback): + dumper = create_xml_dumper(dst_file) + callback(dumper, task_data) + def _export(dst_file, task_data, anno_callback, save_images=False): with TemporaryDirectory() as temp_dir: with open(osp.join(temp_dir, 'annotations.xml'), 'wb') as f: diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 3da6c06854e..13d12290926 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -376,11 +376,11 @@ def add_ssh_keys(): 'version': 1, 'disable_existing_loggers': False, 'formatters': { - 'logstash': { - '()': 'logstash_async.formatter.DjangoLogstashFormatter', - 'message_type': 'python-logstash', - 'fqdn': False, # Fully qualified domain name. Default value: false. - }, + # 'logstash': { + # '()': 'logstash_async.formatter.DjangoLogstashFormatter', + # 'message_type': 'python-logstash', + # 'fqdn': False, # Fully qualified domain name. Default value: false. + # }, 'standard': { 'format': '[%(asctime)s] %(levelname)s %(name)s: %(message)s' } @@ -399,19 +399,19 @@ def add_ssh_keys(): 'maxBytes': 1024*1024*50, # 50 MB 'backupCount': 5, }, - 'logstash': { - 'level': 'INFO', - 'class': 'logstash_async.handler.AsynchronousLogstashHandler', - 'formatter': 'logstash', - 'transport': 'logstash_async.transport.HttpTransport', - 'ssl_enable': False, - 'ssl_verify': False, - 'host': os.getenv('DJANGO_LOG_SERVER_HOST', 'localhost'), - 'port': os.getenv('DJANGO_LOG_SERVER_PORT', 8080), - 'version': 1, - 'message_type': 'django', - 'database_path': LOGSTASH_DB, - } + # 'logstash': { + # 'level': 'INFO', + # 'class': 'logstash_async.handler.AsynchronousLogstashHandler', + # 'formatter': 'logstash', + # 'transport': 'logstash_async.transport.HttpTransport', + # 'ssl_enable': False, + # 'ssl_verify': False, + # 'host': os.getenv('DJANGO_LOG_SERVER_HOST', 'localhost'), + # 'port': os.getenv('DJANGO_LOG_SERVER_PORT', 8080), + # 'version': 1, + # 'message_type': 'django', + # 'database_path': LOGSTASH_DB, + # } }, 'loggers': { 'cvat.server': { From 5d9b512659e35e3584846ea0ef6baff41badde83 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 15 Jul 2021 13:07:18 +0300 Subject: [PATCH 15/63] Added export with CVAT format --- cvat/apps/dataset_manager/bindings.py | 11 ++- cvat/apps/dataset_manager/formats/cvat.py | 85 ++++++++++++++++------- 2 files changed, 70 insertions(+), 26 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index c37333c5038..ff7a433cdc8 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -694,7 +694,9 @@ def shapes(self): @property def tracks(self): - raise NotImplementedError() + for task in self._db_tasks.values(): + for track in self._annotation_irs[task.id].tracks: + yield self._export_labeled_shape(task.id, track) @property def tags(self): @@ -730,6 +732,11 @@ def subsets(self) -> List[str]: def tasks(self): return list(self._db_tasks.values()) + @property + def task_data(self): + for task_id, task in self._db_tasks.items(): + yield TaskData(self._annotation_irs[task_id], task, self._host) + @staticmethod def _get_filename(path): return osp.splitext(path)[0] @@ -826,7 +833,7 @@ def _make_image(i, **kwargs): self._items = dm_items -class CVATProjectDataExtractor(datumaro.SourceExtractor): +class CVATProjectDataExtractor(CVATDataExtractor): def __init__(self, project_data: ProjectData, include_images=False): super().__init__() self._categories = self._load_categories(project_data.meta['project']['labels']) diff --git a/cvat/apps/dataset_manager/formats/cvat.py b/cvat/apps/dataset_manager/formats/cvat.py index b66c8aab309..2340373f483 100644 --- a/cvat/apps/dataset_manager/formats/cvat.py +++ b/cvat/apps/dataset_manager/formats/cvat.py @@ -2,8 +2,10 @@ # # SPDX-License-Identifier: MIT +from io import BufferedWriter import os import os.path as osp +from typing import Callable import zipfile from collections import OrderedDict from glob import glob @@ -11,7 +13,7 @@ from datumaro.components.extractor import DatasetItem -from cvat.apps.dataset_manager.bindings import match_dm_item, ProjectData +from cvat.apps.dataset_manager.bindings import TaskData, match_dm_item, ProjectData, get_defaulted_subset from cvat.apps.dataset_manager.util import make_zip_archive from cvat.apps.engine.frame_provider import FrameProvider @@ -170,6 +172,7 @@ def close_root(self): self._level -= 1 self._indent() self.xmlgen.endElement("annotations") + self._indent() def close_document(self): self.xmlgen.endDocument() @@ -532,41 +535,75 @@ def load(file_object, annotations): def dump_task_anno(dst_file, task_data, callback): dumper = create_xml_dumper(dst_file) + dumper.open_document() callback(dumper, task_data) + dumper.close_document() -def _export(dst_file, task_data, anno_callback, save_images=False): +def dump_project_anno(dst_file: BufferedWriter, project_data: ProjectData, callback: Callable): + dumper = create_xml_dumper(dst_file) + dumper.open_document() + for task_data in project_data.task_data: + callback(dumper, task_data) + dumper.close_document() + +def dump_media_files(task_data: TaskData, img_dir: str): + ext = '' + if task_data.meta['task']['mode'] == 'interpolation': + ext = FrameProvider.VIDEO_FRAME_EXT + + frame_provider = FrameProvider(task_data.db_task.data) + frames = frame_provider.get_frames( + frame_provider.Quality.ORIGINAL, + frame_provider.Type.BUFFER) + for frame_id, (frame_data, _) in enumerate(frames): + frame_name = task_data.frame_info[frame_id]['path'] + img_path = osp.join(img_dir, frame_name + ext) + os.makedirs(osp.dirname(img_path), exist_ok=True) + with open(img_path, 'wb') as f: + f.write(frame_data.getvalue()) + +def _export_task(dst_file, task_data, anno_callback, save_images=False): with TemporaryDirectory() as temp_dir: with open(osp.join(temp_dir, 'annotations.xml'), 'wb') as f: - anno_callback(f, task_data) + dump_task_anno(f, task_data, anno_callback) if save_images: - ext = '' - if task_data.meta['task']['mode'] == 'interpolation': - ext = FrameProvider.VIDEO_FRAME_EXT - - img_dir = osp.join(temp_dir, 'images') - frame_provider = FrameProvider(task_data.db_task.data) - frames = frame_provider.get_frames( - frame_provider.Quality.ORIGINAL, - frame_provider.Type.BUFFER) - for frame_id, (frame_data, _) in enumerate(frames): - frame_name = task_data.frame_info[frame_id]['path'] - img_path = osp.join(img_dir, frame_name + ext) - os.makedirs(osp.dirname(img_path), exist_ok=True) - with open(img_path, 'wb') as f: - f.write(frame_data.getvalue()) + dump_media_files(task_data, osp.join(temp_dir, 'images')) + + make_zip_archive(temp_dir, dst_file) + +def _export_project(dst_file: str, project_data: ProjectData, anno_callback: Callable, save_images: bool=False): + with TemporaryDirectory() as temp_dir: + with open(osp.join(temp_dir, 'annotations.xml'), 'wb') as f: + dump_project_anno(f, project_data, anno_callback) + + if save_images: + for task_data in project_data.task_data: + subset = get_defaulted_subset(task_data.db_task.subset, project_data.subsets) + subset_dir = osp.join(temp_dir, 'images', subset) + if not osp.exists(subset_dir): + os.mkdir(subset_dir) + dump_media_files(task_data, subset_dir) make_zip_archive(temp_dir, dst_file) @exporter(name='CVAT for video', ext='ZIP', version='1.1') -def _export_video(dst_file, task_data, save_images=False): - _export(dst_file, task_data, - anno_callback=dump_as_cvat_interpolation, save_images=save_images) +def _export_video(dst_file, instance_data, save_images=False): + if isinstance(instance_data, ProjectData): + _export_project(dst_file, instance_data, + anno_callback=dump_as_cvat_interpolation, save_images=save_images) + else: + _export_task(dst_file, instance_data, + anno_callback=dump_as_cvat_interpolation, save_images=save_images) @exporter(name='CVAT for images', ext='ZIP', version='1.1') -def _export_images(dst_file, task_data, save_images=False): - _export(dst_file, task_data, - anno_callback=dump_as_cvat_annotation, save_images=save_images) +def _export_images(dst_file, instance_data, save_images=False): + if isinstance(instance_data, ProjectData): + _export_project(dst_file, instance_data, + anno_callback=dump_as_cvat_annotation, save_images=save_images) + else: + _export_task(dst_file, instance_data, + anno_callback=dump_as_cvat_annotation, save_images=save_images) @importer(name='CVAT', ext='XML, ZIP', version='1.1') def _import(src_file, task_data): From a914d05b5615be9865f5a46dcd665ba1fbc04dff Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 15 Jul 2021 13:39:23 +0300 Subject: [PATCH 16/63] Fixed format resetting --- .../components/export-dataset/export-dataset-modal.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx index 7bb2a2fc706..bf1db4a5bd8 100644 --- a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -61,6 +61,13 @@ export default function ExportDatasetModal(): JSX.Element { dispatch(exportActions.closeExportModal()); }; + const handleValuesChange = (changedValues: any): void => { + if ('saveImages' in changedValues) { + form.setFieldsValue({ selectedFormat: undefined }); + } + initActivities(); + }; + const handleExport = (values: FormValues): void => { // have to validate format before so it would not be undefined dispatch( @@ -75,7 +82,7 @@ export default function ExportDatasetModal(): JSX.Element { form={form} labelCol={{ span: 8 }} wrapperCol={{ span: 16 }} - onValuesChange={initActivities} + onValuesChange={handleValuesChange} initialValues={ { selectedFormat: undefined, From 3b480fa1d6b6cb54f6ad3ab51b0263409284dc0e Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 15 Jul 2021 13:40:37 +0300 Subject: [PATCH 17/63] Fixed dir creating --- cvat/apps/dataset_manager/formats/cvat.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cvat/apps/dataset_manager/formats/cvat.py b/cvat/apps/dataset_manager/formats/cvat.py index 2340373f483..94a76bbb956 100644 --- a/cvat/apps/dataset_manager/formats/cvat.py +++ b/cvat/apps/dataset_manager/formats/cvat.py @@ -581,8 +581,7 @@ def _export_project(dst_file: str, project_data: ProjectData, anno_callback: Cal for task_data in project_data.task_data: subset = get_defaulted_subset(task_data.db_task.subset, project_data.subsets) subset_dir = osp.join(temp_dir, 'images', subset) - if not osp.exists(subset_dir): - os.mkdir(subset_dir) + os.makedirs(subset_dir, exist_ok=True) dump_media_files(task_data, subset_dir) make_zip_archive(temp_dir, dst_file) From 2647eb9722a42f8bb5d047253c20046779920189 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Fri, 16 Jul 2021 13:36:41 +0300 Subject: [PATCH 18/63] Fixed export in CVAT formats --- cvat/apps/dataset_manager/bindings.py | 71 +++++++++++++++++------ cvat/apps/dataset_manager/formats/cvat.py | 53 ++++++++++++----- 2 files changed, 90 insertions(+), 34 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index ff7a433cdc8..3ab2ee219c0 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -6,7 +6,7 @@ import sys import os.path as osp from collections import namedtuple -from typing import Any, Callable, Dict, List, Mapping, NamedTuple, OrderedDict, Tuple, Union +from typing import Any, Callable, Dict, List, Literal, Mapping, NamedTuple, OrderedDict, Tuple, Union from pathlib import Path from django.utils import timezone @@ -479,14 +479,14 @@ def match_frame_fuzzy(self, path): class ProjectData(InstanceLabelData): # TODO: strictify - LabeledShape = NamedTuple('LabledShape', [('type',Any), ('frame',Any), ('label',Any), ('points',Any), ('occluded',Any), ('attributes',Any), ('source',Any), ('group',Any), ('z_order',Any)]) + LabeledShape = NamedTuple('LabledShape', [('type',Any), ('frame',Any), ('label',Any), ('points',Any), ('occluded',Any), ('attributes',Any), ('source',Any), ('group',Any), ('z_order',Any), ('task_id', int)]) LabeledShape.__new__.__defaults__ = (0,0) TrackedShape = NamedTuple('TrackedShape', [('type',Any), ('frame',Any), ('points',Any), ('occluded',Any), ('outside',Any), ('keyframe',Any), ('attributes',Any), ('source',Any), ('group',Any), ('z_order',Any), ('label',Any), ('track_id',Any)], ) TrackedShape.__new__.__defaults__ = ('manual', 0, 0, None, 0) - Track = NamedTuple('Track', [('label',Any), ('attributes',Any), ('source',Any), ('group',Any)]) - Tag = NamedTuple('Tag', [('frame',Any), ('label',Any), ('attributes',Any), ('source',Any), ('group',Any)]) + Track = NamedTuple('Track', [('label',Any), ('group',Any), ('source',Any), ('shapes', Any), ('task_id', int)]) + Tag = NamedTuple('Tag', [('frame',Any), ('label',Any), ('attributes',Any), ('source',Any), ('group',Any), ('task_id', int)]) Tag.__new__.__defaults__ = (0, ) Frame = NamedTuple('Frame', [('task_id', int), ('subset', str), ('idx', int), ('frame', int), ('name', str), ('width', int), ('height', int), ('labeled_shapes', List[Union[LabeledShape, TrackedShape]]), ('tags', List[Tag])]) @@ -500,9 +500,9 @@ def __init__(self, annotation_irs: Mapping[str, AnnotationIR], db_project: Proje self._host = host self._create_callback = create_callback self._MAX_ANNO_SIZE = 30000 - self._frame_info: Dict[Tuple[int, int], dict] = dict() + self._frame_info: Dict[Tuple[int, int], Literal["path", "width", "height", "subset"]] = dict() self._frame_mapping: Dict[Tuple[str, str], Tuple[str, str]] = dict() - self._frame_step_mapping = None + self._frame_steps: Dict[int, int] = {task.id: task.data.get_frame_step() for task in self._db_tasks.values()} for task in self._db_tasks.values(): self._subsets.add(task.subset) @@ -549,14 +549,16 @@ def _init_frame_info(self): "path": "frame_{:06d}".format(self.abs_frame_id(task.id, frame)), "width": task.data.video.width, "height": task.data.video.height, - "subset": task.subset, + "subset": get_defaulted_subset(task.subset, self._subsets), } for frame in range(task.data.size)}) else: self._frame_info.update({(task.id, self.rel_frame_id(task.id, db_image.frame)): { - "path": db_image.path, + "path": mung_image_name(db_image.path, + filter(lambda frame: frame["subset"] == task.subset ,self._frame_info.values()) + ), "width": db_image.width, "height": db_image.height, - "subset": task.subset + "subset": get_defaulted_subset(task.subset, self._subsets) } for db_image in task.data.images.all()}) self._frame_mapping = { @@ -575,7 +577,7 @@ def _init_meta(self): ("tasks", [ ('task', TaskData.meta_for_task(db_task, self._host) - ) for db_task in self._db_project.tasks.all() + ) for db_task in self._db_tasks.values() ]), ("labels", [ @@ -633,6 +635,7 @@ def _export_labeled_shape(self, shape: dict, task_id: int): group=shape.get("group", 0), source=shape["source"], attributes=self._export_attributes(shape["attributes"]), + task_id=task_id, ) def _export_tag(self, tag: dict, task_id: int): @@ -642,6 +645,7 @@ def _export_tag(self, tag: dict, task_id: int): group=tag.get("group", 0), source=tag["source"], attributes=self._export_attributes(tag["attributes"]), + task_id=task_id ) def group_by_frame(self, include_empty=False): @@ -690,19 +694,37 @@ def get_frame(task_id: int, idx: int) -> ProjectData.Frame: def shapes(self): for task in self._db_tasks.values(): for shape in self._annotation_irs[task.id].shapes: - yield self._export_labeled_shape(task.id, shape) + yield self._export_labeled_shape(shape, task.id) @property def tracks(self): + idx = 0 for task in self._db_tasks.values(): for track in self._annotation_irs[task.id].tracks: - yield self._export_labeled_shape(task.id, track) + tracked_shapes = TrackManager.get_interpolated_shapes( + track, 0, task.data.size + ) + for tracked_shape in tracked_shapes: + tracked_shape["attributes"] += track["attributes"] + tracked_shape["track_id"] = idx + tracked_shape["group"] = track["group"] + tracked_shape["source"] = track["source"] + tracked_shape["label_id"] = track["label_id"] + yield ProjectData.Track( + label=self._get_label_name(track["label_id"]), + group=track["group"], + source=track["source"], + shapes=[self._export_tracked_shape(shape, task.id) + for shape in tracked_shapes], + task_id=task.id + ) + idx+=1 @property def tags(self): for task in self._db_tasks.values(): for tag in self._annotation_irs[task.id].tags: - yield self._export_tag(task.id, tag) + yield self._export_tag(tag, task.id) @property def meta(self): @@ -714,15 +736,15 @@ def data(self): @property def frame_info(self): - raise NotImplementedError() + return self._frame_info @property def frame_step(self): - raise NotImplementedError() + return self._frame_steps @property - def db_project(self): - raise NotImplementedError() + def project(self): + return self._db_project @property def subsets(self) -> List[str]: @@ -877,7 +899,7 @@ def _make_image(i, **kwargs): dm_anno = self._read_cvat_anno(frame_data, project_data.meta['project']['labels']) dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0], annotations=dm_anno, image=dm_image, - subset=get_defaulted_subset(frame_data.subset, project_data.subsets), + subset=frame_data.subset, attributes={'frame': frame_data.frame} ) dm_items.append(dm_item) @@ -894,6 +916,17 @@ def GetCVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_im class CvatImportError(Exception): pass +def mung_image_name(name: str, images: List[Literal["path", "width", "height", "subset"]]) -> str: + name, ext = name.split(osp.extsep) + if not any(map(lambda image: image["path"] == name, images)): + return name + osp.extsep + ext + else: + i = 1 + while i < sys.maxsize: + if not any(map(lambda image: image["path"] == f"{name}_{i}{osp.extsep}{ext}", images)): + return f"{name}_{i}{osp.extsep}{ext}" + i += 1 + raise Exception('Cannot mung image name') def get_defaulted_subset(subset: str, subsets: List[str]) -> str: if subset: @@ -907,7 +940,7 @@ def get_defaulted_subset(subset: str, subsets: List[str]) -> str: if f'{datumaro.DEFAULT_SUBSET_NAME}_{i}' not in subsets: return f'{datumaro.DEFAULT_SUBSET_NAME}_{i}' i += 1 - raise Exception('Cannot find deafoult name for subset') + raise Exception('Cannot find default name for subset') def convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label): diff --git a/cvat/apps/dataset_manager/formats/cvat.py b/cvat/apps/dataset_manager/formats/cvat.py index 94a76bbb956..9e7fa514fb6 100644 --- a/cvat/apps/dataset_manager/formats/cvat.py +++ b/cvat/apps/dataset_manager/formats/cvat.py @@ -186,12 +186,20 @@ def dump_as_cvat_annotation(dumper, annotations): for frame_annotation in annotations.group_by_frame(include_empty=True): frame_id = frame_annotation.frame - dumper.open_image(OrderedDict([ + image_attrs = OrderedDict([ ("id", str(frame_id)), ("name", frame_annotation.name), + ]) + if isinstance(annotations, ProjectData): + image_attrs.update(OrderedDict([ + ("subset", frame_annotation.subset), + ("task_id", str(frame_annotation.task_id)), + ])) + image_attrs.update(OrderedDict([ ("width", str(frame_annotation.width)), ("height", str(frame_annotation.height)) ])) + dumper.open_image(image_attrs) for shape in frame_annotation.labeled_shapes: dump_data = OrderedDict([ @@ -304,6 +312,13 @@ def dump_track(idx, track): ("source", track.source), ]) + if hasattr(track, 'task_id'): + task, = filter(lambda task: task.id == track.task_id, annotations.tasks) + dump_data.update(OrderedDict([ + ('task_id', str(track.task_id)), + ('subset', get_defaulted_subset(task.subset, annotations.subsets)), + ])) + if track.group: dump_data['group_id'] = str(track.group) dumper.open_track(dump_data) @@ -389,11 +404,17 @@ def dump_track(idx, track): counter += 1 for shape in annotations.shapes: - dump_track(counter, annotations.Track( - label=shape.label, - group=shape.group, - source=shape.source, - shapes=[annotations.TrackedShape( + frame_step = annotations.frame_step if isinstance(annotations, TaskData) else annotations.frame_step[shape.task_id] + if isinstance(annotations, TaskData): + stop_frame = int(annotations.meta['task']['stop_frame']) + else: + task_meta = list(filter(lambda task: int(task[1]['id']) == shape.task_id, annotations.meta['project']['tasks']))[0][1] + stop_frame = int(task_meta['stop_frame']) + track = { + 'label': shape.label, + 'group': shape.group, + 'source': shape.source, + 'shapes': [annotations.TrackedShape( type=shape.type, points=shape.points, occluded=shape.occluded, @@ -411,13 +432,15 @@ def dump_track(idx, track): outside=True, keyframe=True, z_order=shape.z_order, - frame=shape.frame + annotations.frame_step, + frame=shape.frame + frame_step, attributes=shape.attributes, - )] if shape.frame + annotations.frame_step < \ - int(annotations.meta['task']['stop_frame']) \ + )] if shape.frame + frame_step < \ + stop_frame \ else [] ), - )) + } + if isinstance(annotations, ProjectData): track['task_id'] = shape.task_id + dump_track(counter, annotations.Track(**track)) counter += 1 dumper.close_root() @@ -542,11 +565,10 @@ def dump_task_anno(dst_file, task_data, callback): def dump_project_anno(dst_file: BufferedWriter, project_data: ProjectData, callback: Callable): dumper = create_xml_dumper(dst_file) dumper.open_document() - for task_data in project_data.task_data: - callback(dumper, task_data) + callback(dumper, project_data) dumper.close_document() -def dump_media_files(task_data: TaskData, img_dir: str): +def dump_media_files(task_data: TaskData, img_dir: str, project_data: ProjectData = None): ext = '' if task_data.meta['task']['mode'] == 'interpolation': ext = FrameProvider.VIDEO_FRAME_EXT @@ -556,7 +578,8 @@ def dump_media_files(task_data: TaskData, img_dir: str): frame_provider.Quality.ORIGINAL, frame_provider.Type.BUFFER) for frame_id, (frame_data, _) in enumerate(frames): - frame_name = task_data.frame_info[frame_id]['path'] + frame_name = task_data.frame_info[frame_id]['path'] if project_data is None \ + else project_data.frame_info[(task_data.db_task.id, frame_id)]['path'] img_path = osp.join(img_dir, frame_name + ext) os.makedirs(osp.dirname(img_path), exist_ok=True) with open(img_path, 'wb') as f: @@ -582,7 +605,7 @@ def _export_project(dst_file: str, project_data: ProjectData, anno_callback: Cal subset = get_defaulted_subset(task_data.db_task.subset, project_data.subsets) subset_dir = osp.join(temp_dir, 'images', subset) os.makedirs(subset_dir, exist_ok=True) - dump_media_files(task_data, subset_dir) + dump_media_files(task_data, subset_dir, project_data) make_zip_archive(temp_dir, dst_file) From 15d226aae6e48829ac576fe4b9670d654c218cac Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 20 Jul 2021 03:56:30 +0300 Subject: [PATCH 19/63] Added simple server tests, fixed datumaro format export --- cvat/apps/dataset_manager/bindings.py | 2 +- .../formats/datumaro/__init__.py | 39 +++-- cvat/apps/dataset_manager/formats/utils.py | 5 +- .../tests/assets/projects.json | 55 +++++++ .../dataset_manager/tests/assets/tasks.json | 23 +-- .../tests/test_rest_api_formats.py | 137 +++++++++++++++++- cvat/apps/engine/views.py | 5 +- 7 files changed, 232 insertions(+), 34 deletions(-) create mode 100644 cvat/apps/dataset_manager/tests/assets/projects.json diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 3ab2ee219c0..c7e48690ca4 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -743,7 +743,7 @@ def frame_step(self): return self._frame_steps @property - def project(self): + def db_project(self): return self._db_project @property diff --git a/cvat/apps/dataset_manager/formats/datumaro/__init__.py b/cvat/apps/dataset_manager/formats/datumaro/__init__.py index 3e5b1e6cb56..7d7605c9d7e 100644 --- a/cvat/apps/dataset_manager/formats/datumaro/__init__.py +++ b/cvat/apps/dataset_manager/formats/datumaro/__init__.py @@ -8,8 +8,8 @@ import shutil from tempfile import TemporaryDirectory -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, - import_dm_annotations) +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, + import_dm_annotations, ProjectData) from cvat.apps.dataset_manager.util import make_zip_archive from cvat.settings.base import BASE_DIR from datumaro.components.project import Project @@ -23,34 +23,40 @@ class DatumaroProjectExporter: _TEMPLATES_DIR = osp.join(osp.dirname(__file__), 'export_templates') @staticmethod - def _save_image_info(save_dir, task_data): + def _save_image_info(save_dir, instance_data): os.makedirs(save_dir, exist_ok=True) config = { - 'server_url': task_data._host or 'localhost', - 'task_id': task_data.db_task.id, + 'server_url': instance_data._host or 'localhost' } + if isinstance(instance_data, ProjectData): + config['project_id'] = instance_data.db_project.id + else: + config['task_id'] = instance_data.db_task.id images = [] images_meta = { 'images': images, } - for frame_id, frame in task_data.frame_info.items(): - images.append({ + for frame_id, frame in enumerate(instance_data.frame_info.values()): + image_info = { 'id': frame_id, 'name': osp.basename(frame['path']), 'width': frame['width'], 'height': frame['height'], - }) + } + if isinstance(instance_data, ProjectData): + image_info['subset'] = frame['subset'] with open(osp.join(save_dir, 'config.json'), 'w') as config_file: json.dump(config, config_file) with open(osp.join(save_dir, 'images_meta.json'), 'w') as images_file: json.dump(images_meta, images_file) - def _export(self, task_data, save_dir, save_images=False): - dataset = CvatTaskDataExtractor(task_data, include_images=save_images) + def _export(self, instance_data, save_dir, save_images=False): + dataset = GetCVATDataExtractor(instance_data, include_images=save_images) + db_instance = instance_data.db_project if isinstance(instance_data, ProjectData) else instance_data.db_task dm_env.converters.get('datumaro_project').convert(dataset, save_dir=save_dir, save_images=save_images, - project_config={ 'project_name': task_data.db_task.name, } + project_config={ 'project_name': db_instance.name, } ) project = Project.load(save_dir) @@ -62,13 +68,16 @@ def _export(self, task_data, save_dir, save_images=False): if not save_images: # add remote links to images - source_name = 'task_%s_images' % task_data.db_task.id + source_name = '{}_{}_images'.format( + 'project' if isinstance(instance_data, ProjectData) else 'task', + db_instance.id, + ) project.add_source(source_name, { 'format': self._REMOTE_IMAGES_EXTRACTOR, }) self._save_image_info( osp.join(save_dir, project.local_source_dir(source_name)), - task_data) + instance_data) project.save() templates_dir = osp.join(self._TEMPLATES_DIR, 'plugins') @@ -85,7 +94,7 @@ def _export(self, task_data, save_dir, save_images=False): shutil.copytree(osp.join(BASE_DIR, 'utils', 'cli'), osp.join(cvat_utils_dst_dir, 'cli')) - def __call__(self, dst_file, task_data, save_images=False): + def __call__(self, dst_file, instance_data, save_images=False): with TemporaryDirectory() as temp_dir: - self._export(task_data, save_dir=temp_dir, save_images=save_images) + self._export(instance_data, save_dir=temp_dir, save_images=save_images) make_zip_archive(temp_dir, dst_file) diff --git a/cvat/apps/dataset_manager/formats/utils.py b/cvat/apps/dataset_manager/formats/utils.py index 184a133161b..0d545e46583 100644 --- a/cvat/apps/dataset_manager/formats/utils.py +++ b/cvat/apps/dataset_manager/formats/utils.py @@ -48,8 +48,9 @@ def rgb2hex(color): def hex2rgb(color): return tuple(int(color.lstrip('#')[i:i+2], 16) for i in (0, 2, 4)) -def make_colormap(task_data): - labels = [label for _, label in task_data.meta['task']['labels']] +def make_colormap(instance_data): + instance_name = 'project' if 'project' in instance_data.meta.keys() else 'task' + labels = [label for _, label in instance_data.meta[instance_name]['labels']] label_names = [label['name'] for label in labels] if 'background' not in label_names: diff --git a/cvat/apps/dataset_manager/tests/assets/projects.json b/cvat/apps/dataset_manager/tests/assets/projects.json new file mode 100644 index 00000000000..cef9a4ba891 --- /dev/null +++ b/cvat/apps/dataset_manager/tests/assets/projects.json @@ -0,0 +1,55 @@ +{ + "main": { + "name": "Main project", + "owner_id": 1, + "assignee_id": 2, + "labels": [ + { + "name": "car", + "color": "#2080c0", + "attributes": [ + { + "name": "select_name", + "mutable": false, + "input_type": "select", + "default_value": "bmw", + "values": ["bmw", "mazda", "renault"] + }, + { + "name": "radio_name", + "mutable": false, + "input_type": "radio", + "default_value": "x1", + "values": ["x1", "x2", "x3"] + }, + { + "name": "check_name", + "mutable": true, + "input_type": "checkbox", + "default_value": "false", + "values": ["false"] + }, + { + "name": "text_name", + "mutable": false, + "input_type": "text", + "default_value": "qwerty", + "values": ["qwerty"] + }, + { + "name": "number_name", + "mutable": false, + "input_type": "number", + "default_value": "-4", + "values": ["-4", "4", "1"] + } + ] + }, + { + "name": "person", + "color": "#c06060", + "attributes": [] + } + ] + } +} diff --git a/cvat/apps/dataset_manager/tests/assets/tasks.json b/cvat/apps/dataset_manager/tests/assets/tasks.json index 09e2d866287..23ea55e2ae7 100644 --- a/cvat/apps/dataset_manager/tests/assets/tasks.json +++ b/cvat/apps/dataset_manager/tests/assets/tasks.json @@ -282,17 +282,20 @@ } ] }, - "many jobs": { - "name": "many jobs", + "task in project #1": { + "name": "First task in project", + "project_id": 1, "overlap": 0, - "segment_size": 5, + "segment_size": 100, "owner_id": 1, - "labels": [ - { - "name": "car", - "color": "#2080c0", - "attributes": [] - } - ] + "assignee_id": 2 + }, + "task in project #2": { + "name": "Second task in project", + "project_id": 1, + "overlap": 0, + "segment_size": 100, + "owner_id": 1, + "assignee_id": 2 } } diff --git a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py index 880585b25f2..e6d664986b4 100644 --- a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py +++ b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py @@ -7,6 +7,7 @@ import os.path as osp import os import av +from django.http import response import numpy as np import random import xml.etree.ElementTree as ET @@ -26,6 +27,10 @@ from cvat.apps.dataset_manager.task import TaskAnnotation from cvat.apps.engine.models import Task +projects_path = osp.join(osp.dirname(__file__), 'assets', 'projects.json') +with open(projects_path) as file: + projects = json.load(file) + tasks_path = osp.join(osp.dirname(__file__), 'assets', 'tasks.json') with open(tasks_path) as file: tasks = json.load(file) @@ -133,8 +138,8 @@ def _put_api_v1_job_id_annotations(self, jid, data): return response @staticmethod - def _generate_task_images(count): # pylint: disable=no-self-use - images = {"client_files[%d]" % i: generate_image_file("image_%d.jpg" % i) for i in range(count)} + def _generate_task_images(count, name_offsets = 0): # pylint: disable=no-self-use + images = {"client_files[%d]" % i: generate_image_file("image_%d.jpg" % (i + name_offsets)) for i in range(count)} images["image_quality"] = 75 return images @@ -159,6 +164,14 @@ def _create_task(self, data, image_data): return task + def _create_project(self, data): + with ForceLogin(self.user, self.client): + response = self.client.post('/api/v1/projects', data=data, format="json") + assert response.status_code == status.HTTP_201_CREATED, response.status_code + project = response.data + + return project + def _get_jobs(self, task_id): with ForceLogin(self.admin, self.client): response = self.client.get("/api/v1/tasks/{}/jobs".format(task_id)) @@ -297,14 +310,25 @@ def _generate_url_dump_job_annotations(self, job_id): def _generate_url_upload_job_annotations(self, job_id, upload_format_name): return f"/api/v1/jobs/{job_id}/annotations?format={upload_format_name}" - def _generate_url_dump_dataset(self, task_id): + def _generate_url_dump_task_dataset(self, task_id): return f"/api/v1/tasks/{task_id}/dataset" + def _generate_url_dump_project_annotations(self, project_id, format_name): + return f"/api/v1/projects/{project_id}/annotations?format={format_name}" + + def _generate_url_dump_project_dataset(self, project_id, format_name): + return f"/api/v1/projects/{project_id}/dataset?format={format_name}" + def _remove_annotations(self, url, user): response = self._delete_request(url, user) self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) return response + def _delete_project(self, project_id, user): + response = self._delete_request(f'/api/v1/projects/{project_id}', user) + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + return response + class TaskDumpUploadTest(_DbTestBase): def test_api_v1_dump_and_upload_annotations_with_objects_type_is_shape(self): @@ -783,7 +807,7 @@ def test_api_v1_export_dataset(self): task = self._create_task(tasks["main"], images) task_id = task["id"] # dump annotations - url = self._generate_url_dump_dataset(task_id) + url = self._generate_url_dump_task_dataset(task_id) for user, edata in list(expected.items()): user_name = edata['name'] file_zip_name = osp.join(test_dir, f'{test_name}_{user_name}_{dump_format_name}.zip') @@ -1136,3 +1160,108 @@ def test_api_v1_check_attribute_import_in_tracks(self): # equals annotations data_from_task_after_upload = self._get_data_from_task(task_id, include_images) compare_datasets(self, data_from_task_before_upload, data_from_task_after_upload) + +class ProjectDump(_DbTestBase): + def test_api_v1_export_dataset(self): + test_name = self._testMethodName + dump_formats = dm.views.get_export_formats() + + expected = { + self.admin: {'name': 'admin', 'code': status.HTTP_200_OK, 'create code': status.HTTP_201_CREATED, + 'accept code': status.HTTP_202_ACCEPTED, 'file_exists': True}, + self.user: {'name': 'user', 'code': status.HTTP_200_OK, 'create code': status.HTTP_201_CREATED, + 'accept code': status.HTTP_202_ACCEPTED, 'file_exists': True}, + None: {'name': 'none', 'code': status.HTTP_401_UNAUTHORIZED, 'create code': status.HTTP_401_UNAUTHORIZED, + 'accept code': status.HTTP_401_UNAUTHORIZED, 'file_exists': False}, + } + + with TestDir() as test_dir: + for dump_format in dump_formats: + if not dump_format.ENABLED: + continue + dump_format_name = dump_format.DISPLAY_NAME + with self.subTest(format=dump_format_name): + project = self._create_project(projects['main']) + pid = project['id'] + images = self._generate_task_images(3) + tasks['task in project #1']['project_id'] = pid + self._create_task(tasks['task in project #1'], images) + images = self._generate_task_images(3, 3) + tasks['task in project #2']['project_id'] = pid + self._create_task(tasks['task in project #2'], images) + url = self._generate_url_dump_project_dataset(project['id'], dump_format_name) + + for user, edata in list(expected.items()): + user_name = edata['name'] + file_zip_name = osp.join(test_dir, f'{test_name}_{user_name}_{dump_format_name}.zip') + data = { + "format": dump_format_name, + } + response = self._get_request_with_data(url, data, user) + self.assertEqual(response.status_code, edata["accept code"]) + response = self._get_request_with_data(url, data, user) + self.assertEqual(response.status_code, edata["create code"]) + data = { + "format": dump_format_name, + "action": "download", + } + response = self._get_request_with_data(url, data, user) + self.assertEqual(response.status_code, edata["code"]) + if response.status_code == status.HTTP_200_OK: + content = BytesIO(b"".join(response.streaming_content)) + with open(file_zip_name, "wb") as f: + f.write(content.getvalue()) + self.assertEqual(response.status_code, edata['code']) + self.assertEqual(osp.exists(file_zip_name), edata['file_exists']) + + def test_api_v1_export_annotatios(self): + test_name = self._testMethodName + dump_formats = dm.views.get_export_formats() + + expected = { + self.admin: {'name': 'admin', 'code': status.HTTP_200_OK, 'create code': status.HTTP_201_CREATED, + 'accept code': status.HTTP_202_ACCEPTED, 'file_exists': True}, + self.user: {'name': 'user', 'code': status.HTTP_200_OK, 'create code': status.HTTP_201_CREATED, + 'accept code': status.HTTP_202_ACCEPTED, 'file_exists': True}, + None: {'name': 'none', 'code': status.HTTP_401_UNAUTHORIZED, 'create code': status.HTTP_401_UNAUTHORIZED, + 'accept code': status.HTTP_401_UNAUTHORIZED, 'file_exists': False}, + } + + with TestDir() as test_dir: + for dump_format in dump_formats: + if not dump_format.ENABLED: + continue + dump_format_name = dump_format.DISPLAY_NAME + with self.subTest(format=dump_format_name): + project = self._create_project(projects['main']) + pid = project['id'] + images = self._generate_task_images(3) + tasks['task in project #1']['project_id'] = pid + self._create_task(tasks['task in project #1'], images) + images = self._generate_task_images(3, 3) + tasks['task in project #2']['project_id'] = pid + self._create_task(tasks['task in project #2'], images) + url = self._generate_url_dump_project_annotations(project['id'], dump_format_name) + + for user, edata in list(expected.items()): + user_name = edata['name'] + file_zip_name = osp.join(test_dir, f'{test_name}_{user_name}_{dump_format_name}.zip') + data = { + "format": dump_format_name, + } + response = self._get_request_with_data(url, data, user) + self.assertEqual(response.status_code, edata["accept code"]) + response = self._get_request_with_data(url, data, user) + self.assertEqual(response.status_code, edata["create code"]) + data = { + "format": dump_format_name, + "action": "download", + } + response = self._get_request_with_data(url, data, user) + self.assertEqual(response.status_code, edata["code"]) + if response.status_code == status.HTTP_200_OK: + content = BytesIO(b"".join(response.streaming_content)) + with open(file_zip_name, "wb") as f: + f.write(content.getvalue()) + self.assertEqual(response.status_code, edata['code']) + self.assertEqual(osp.exists(file_zip_name), edata['file_exists']) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index d57b105afe9..85ade1734eb 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -335,7 +335,7 @@ def dataset_export(self, request, pk): manual_parameters=[ openapi.Parameter('format', openapi.IN_QUERY, description="Desired output format name\nYou can get the list of supported formats at:\n/server/annotation/formats", - type=openapi.TYPE_STRING, required=False), + type=openapi.TYPE_STRING, required=True), openapi.Parameter('filename', openapi.IN_QUERY, description="Desired output file name", type=openapi.TYPE_STRING, required=False), @@ -348,6 +348,7 @@ def dataset_export(self, request, pk): '201': openapi.Response(description='Annotations file is ready to download'), '200': openapi.Response(description='Download of file started'), '405': openapi.Response(description='Format is not available'), + '401': openapi.Response(description='Format is not specified'), } ) @action(detail=True, methods=['GET'], @@ -365,7 +366,7 @@ def annotations(self, request, pk): filename=request.query_params.get("filename", "").lower(), ) else: - return Response(status=500) + return Response("Format is not specified",status=status.HTTP_400_BAD_REQUEST) class TaskFilter(filters.FilterSet): project = filters.CharFilter(field_name="project__name", lookup_expr="icontains") From 3da98c430601a6ca00c89097c969ae8ce027988f Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 20 Jul 2021 03:59:23 +0300 Subject: [PATCH 20/63] Reverted settings change --- cvat/settings/base.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 13d12290926..3da6c06854e 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -376,11 +376,11 @@ def add_ssh_keys(): 'version': 1, 'disable_existing_loggers': False, 'formatters': { - # 'logstash': { - # '()': 'logstash_async.formatter.DjangoLogstashFormatter', - # 'message_type': 'python-logstash', - # 'fqdn': False, # Fully qualified domain name. Default value: false. - # }, + 'logstash': { + '()': 'logstash_async.formatter.DjangoLogstashFormatter', + 'message_type': 'python-logstash', + 'fqdn': False, # Fully qualified domain name. Default value: false. + }, 'standard': { 'format': '[%(asctime)s] %(levelname)s %(name)s: %(message)s' } @@ -399,19 +399,19 @@ def add_ssh_keys(): 'maxBytes': 1024*1024*50, # 50 MB 'backupCount': 5, }, - # 'logstash': { - # 'level': 'INFO', - # 'class': 'logstash_async.handler.AsynchronousLogstashHandler', - # 'formatter': 'logstash', - # 'transport': 'logstash_async.transport.HttpTransport', - # 'ssl_enable': False, - # 'ssl_verify': False, - # 'host': os.getenv('DJANGO_LOG_SERVER_HOST', 'localhost'), - # 'port': os.getenv('DJANGO_LOG_SERVER_PORT', 8080), - # 'version': 1, - # 'message_type': 'django', - # 'database_path': LOGSTASH_DB, - # } + 'logstash': { + 'level': 'INFO', + 'class': 'logstash_async.handler.AsynchronousLogstashHandler', + 'formatter': 'logstash', + 'transport': 'logstash_async.transport.HttpTransport', + 'ssl_enable': False, + 'ssl_verify': False, + 'host': os.getenv('DJANGO_LOG_SERVER_HOST', 'localhost'), + 'port': os.getenv('DJANGO_LOG_SERVER_PORT', 8080), + 'version': 1, + 'message_type': 'django', + 'database_path': LOGSTASH_DB, + } }, 'loggers': { 'cvat.server': { From b8b3d9c70fd60043bfabc67f28fcbf9e91698ece Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 20 Jul 2021 14:59:21 +0300 Subject: [PATCH 21/63] Fixed merge --- cvat/apps/dataset_manager/bindings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index a3b4811ccac..f7b95e09fc8 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -1070,7 +1070,8 @@ def convert_attrs(label, cvat_attrs): attributes=anno_attr, group=anno_group) item_anno.append(anno) - shapes = [] + shapes = [] + if hasattr(cvat_frame_anno, 'shapes'): for shape in cvat_frame_anno.shapes: shapes.append({"id": shape.id, "label_id": shape.label_id}) From e1f8db60eb38530b826975667c6375e805a77045 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 21 Jul 2021 13:35:39 +0300 Subject: [PATCH 22/63] Fixed some comments --- .vscode/launch.json | 18 ++++++++++-- .vscode/settings.json | 12 ++------ .vscode/tasks.json | 30 ++------------------ cvat/apps/dataset_manager/bindings.py | 41 ++++++++++----------------- 4 files changed, 36 insertions(+), 65 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 7e1f0267b91..658abd2fe48 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,7 +7,7 @@ { "type": "chrome", "request": "launch", - "preLaunchTask": "npm: start - cvat-ui", + "preLaunchTask": "ui.js: server", "name": "ui.js: debug", "url": "http://localhost:3000", "webRoot": "${workspaceFolder}/cvat-ui", @@ -45,7 +45,7 @@ "python": "${command:python.interpreterPath}", "program": "${workspaceRoot}/manage.py", "env": { - "CVAT_SERVERLESS": "1" + "CVAT_SERVERLESS": "1", }, "args": [ "runserver", @@ -57,6 +57,19 @@ "cwd": "${workspaceFolder}", "console": "internalConsole" }, + { + "name": "server: chrome", + "type": "chrome", + "request": "launch", + "url": "http://localhost:7000/", + "disableNetworkCache":true, + "trace": true, + "showAsyncStacks": true, + "pathMapping":{ + "/static/engine/": "${workspaceFolder}/cvat/apps/engine/static/engine/", + "/static/dashboard/": "${workspaceFolder}/cvat/apps/dashboard/static/dashboard/", + } + }, { "name": "server: RQ - default", "type": "python", @@ -194,6 +207,7 @@ { "name": "server: debug", "configurations": [ + "server: chrome", "server: django", "server: RQ - default", "server: RQ - low", diff --git a/.vscode/settings.json b/.vscode/settings.json index ce5a350e8f3..cb78ca045e6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,5 @@ { + "eslint.enable": true, "eslint.probe": [ "javascript", "typescript", @@ -23,14 +24,5 @@ "python.linting.pycodestyleEnabled": false, "licenser.license": "Custom", "licenser.customHeader": "Copyright (C) @YEAR@ Intel Corporation\n\nSPDX-License-Identifier: MIT", - "files.trimTrailingWhitespace": true, - "python.pythonPath": ".env/bin/python", - "sqltools.connections": [ - { - "previewLimit": 50, - "driver": "SQLite", - "name": "cvat", - "database": "${workspaceFolder:cvat}/db.sqlite3" - } - ] + "files.trimTrailingWhitespace": true } diff --git a/.vscode/tasks.json b/.vscode/tasks.json index c8d8c11e88f..10c34d90ea1 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -4,35 +4,11 @@ "version": "2.0.0", "tasks": [ { + "label": "ui.js: server", "type": "npm", "script": "start", "path": "cvat-ui/", - "label": "npm: start - cvat-ui", - "detail": "webpack-dev-server --env.API_URL=http://localhost:7000 --config ./webpack.config.js --mode=development", - "promptOnClose": true, - "isBackground": true, - "problemMatcher": { - "owner": "webpack", - "severity": "error", - "fileLocation": "absolute", - "pattern": [ - { - "regexp": "ERROR in (.*)", - "file": 1 - }, - { - "regexp": "\\((\\d+),(\\d+)\\):(.*)", - "line": 1, - "column": 2, - "message": 3 - } - ], - "background": { - "activeOnStart": true, - "beginsPattern": "webpack-dev-server", - "endsPattern": "Compiled" - } - } + "problemMatcher": [] } ] -} +} \ No newline at end of file diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index f7b95e09fc8..bf1e648104d 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -579,9 +579,7 @@ def _init_frame_info(self): } for frame in range(task.data.size)}) else: self._frame_info.update({(task.id, self.rel_frame_id(task.id, db_image.frame)): { - "path": mung_image_name(db_image.path, - filter(lambda frame: frame["subset"] == task.subset ,self._frame_info.values()) - ), + "path": mangle_image_name(db_image.path, self._frame_info.values(), task.subset), "width": db_image.width, "height": db_image.height, "subset": get_defaulted_subset(task.subset, self._subsets) @@ -791,21 +789,10 @@ def _get_filename(path): class CVATDataExtractor(datumaro.SourceExtractor): - dm_items: List[datumaro.DatasetItem] = [] - _categories: Dict[datumaro.AnnotationType, datumaro.LabelCategories] = {} - def __init__(self): super().__init__() - - def __iter__(self): - for item in self._items: - yield item - - def __len__(self): - return len(self._items) - - def categories(self): - return self._categories + self.dm_items: List[datumaro.DatasetItem] = [] + self._categories: Dict[datumaro.AnnotationType, datumaro.LabelCategories] = {} @staticmethod def _load_categories(labels: list): @@ -1011,17 +998,22 @@ def GetCVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_im class CvatImportError(Exception): pass -def mung_image_name(name: str, images: List[Literal["path", "width", "height", "subset"]]) -> str: - name, ext = name.split(osp.extsep) - if not any(map(lambda image: image["path"] == name, images)): - return name + osp.extsep + ext +def mangle_image_name(name: str, images: List[Literal["path", "width", "height", "subset"]], subset: str) -> str: + name, ext = name.rsplit(osp.extsep, maxsplit=1) + paths: Dict[str, int] = {} + for image in images: + if image['subset'] == subset: + paths[image['path']] = 1 + if not paths[name]: + return osp.extsep.join([name, ext]) else: i = 1 while i < sys.maxsize: - if not any(map(lambda image: image["path"] == f"{name}_{i}{osp.extsep}{ext}", images)): - return f"{name}_{i}{osp.extsep}{ext}" + image_name = f"{name}_{i}{osp.extsep}{ext}" + if not paths[image_name]: + return image_name i += 1 - raise Exception('Cannot mung image name') + raise Exception('Cannot mangle image name') def get_defaulted_subset(subset: str, subsets: List[str]) -> str: if subset: @@ -1057,8 +1049,6 @@ def convert_attrs(label, cvat_attrs): raise Exception( "Failed to convert attribute '%s'='%s': %s" % (a_name, a_value, e)) - if format_name == "sly_pointcloud" and (a_desc.get('input_type') == 'select' or a_desc.get('input_type') == 'radio'): - dm_attr[f"{a_name}__values"] = a_desc["values"] return dm_attr for tag_obj in cvat_frame_anno.tags: @@ -1107,7 +1097,6 @@ def convert_attrs(label, cvat_attrs): if dimension == DimensionType.DIM_3D: if format_name == "sly_pointcloud": anno_id = shapes[index]["id"] - anno_attr["label_id"] = shapes[index]["label_id"] else: anno_id = index position, rotation, scale = anno_points[0:3], anno_points[3:6], anno_points[6:9] From 98c908364704e9416d4de9982f3673da5626a10b Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 21 Jul 2021 13:38:07 +0300 Subject: [PATCH 23/63] Update cvat/apps/dataset_manager/bindings.py Co-authored-by: Maxim Zhiltsov --- cvat/apps/dataset_manager/bindings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index bf1e648104d..de899b43283 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -796,7 +796,7 @@ def __init__(self): @staticmethod def _load_categories(labels: list): - categories: Dict[datumaro.AnnotationType, datumaro.LabelCategories] = {} + categories: Dict[datumaro.AnnotationType, datumaro.Categories] = {} label_categories = datumaro.LabelCategories(attributes=['occluded']) From 84f8b7a78f47bb766272ba4eca68460de9207ef9 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 21 Jul 2021 13:56:09 +0300 Subject: [PATCH 24/63] Fixed comment --- cvat/apps/dataset_manager/formats/velodynepoint.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cvat/apps/dataset_manager/formats/velodynepoint.py b/cvat/apps/dataset_manager/formats/velodynepoint.py index daa7d360939..404988f1599 100644 --- a/cvat/apps/dataset_manager/formats/velodynepoint.py +++ b/cvat/apps/dataset_manager/formats/velodynepoint.py @@ -20,9 +20,6 @@ @exporter(name='Kitti Raw Format', ext='ZIP', version='1.0', dimension=DimensionType.DIM_3D) def _export_images(dst_file, task_data, save_images=False): - if not isinstance(task_data, TaskData): - raise Exception("Export to kitti raw format is working only with tasks") - dataset = Dataset.from_extractors(CvatTaskDataExtractor( task_data, include_images=save_images, format_type="kitti_raw", dimension=DimensionType.DIM_3D), env=dm_env) From 29ffac1596f4946a8e7913b243886a688a2c191f Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 22 Jul 2021 13:16:58 +0300 Subject: [PATCH 25/63] Added restrictions for 3d tasks --- cvat/apps/dataset_manager/bindings.py | 51 ++++++++++--------- .../dataset_manager/formats/pointcloud.py | 5 +- .../dataset_manager/formats/velodynepoint.py | 3 ++ cvat/apps/engine/serializers.py | 10 +++- 4 files changed, 43 insertions(+), 26 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index de899b43283..0a293221985 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -5,9 +5,10 @@ import sys import os.path as osp -from collections import namedtuple -from typing import Any, Callable, Dict, List, Literal, Mapping, NamedTuple, OrderedDict, Tuple, Union +from collections import defaultdict, namedtuple +from typing import Any, Callable, DefaultDict, Dict, List, Literal, Mapping, NamedTuple, OrderedDict, Tuple, Union from pathlib import Path +from PIL.Image import new from django.utils import timezone @@ -21,7 +22,7 @@ class InstanceLabelData: - Attribute = namedtuple('Attribute', 'name, value') + Attribute = NamedTuple('Attribute', [('name', str), ('value', Any)]) def __init__(self, instance: Union[Task, Project]) -> None: instance = instance.project if isinstance(instance, Task) and instance.project_id is not None else instance @@ -110,7 +111,7 @@ def _export_attributes(self, attributes): exported_attributes = [] for attr in attributes: attribute_name = self._get_attribute_name(attr["spec_id"]) - exported_attributes.append(TaskData.Attribute( + exported_attributes.append(InstanceLabelData.Attribute( name=attribute_name, value=attr["value"], )) @@ -504,15 +505,14 @@ def match_frame_fuzzy(self, path): return None class ProjectData(InstanceLabelData): - # TODO: strictify - LabeledShape = NamedTuple('LabledShape', [('type',Any), ('frame',Any), ('label',Any), ('points',Any), ('occluded',Any), ('attributes',Any), ('source',Any), ('group',Any), ('z_order',Any), ('task_id', int)]) + LabeledShape = NamedTuple('LabledShape', [('type', str), ('frame', int), ('label', str), ('points', List[float]), ('occluded', bool), ('attributes', List[InstanceLabelData.Attribute]), ('source', str), ('group', int), ('z_order', int), ('task_id', int)]) LabeledShape.__new__.__defaults__ = (0,0) TrackedShape = NamedTuple('TrackedShape', - [('type',Any), ('frame',Any), ('points',Any), ('occluded',Any), ('outside',Any), ('keyframe',Any), ('attributes',Any), ('source',Any), ('group',Any), ('z_order',Any), ('label',Any), ('track_id',Any)], + [('type', str), ('frame', int), ('points', List[float]), ('occluded', bool), ('outside', bool), ('keyframe', bool), ('attributes', List[InstanceLabelData.Attribute]), ('source', str), ('group', int), ('z_order', int), ('label', str), ('track_id', int)], ) TrackedShape.__new__.__defaults__ = ('manual', 0, 0, None, 0) - Track = NamedTuple('Track', [('label',Any), ('group',Any), ('source',Any), ('shapes', Any), ('task_id', int)]) - Tag = NamedTuple('Tag', [('frame',Any), ('label',Any), ('attributes',Any), ('source',Any), ('group',Any), ('task_id', int)]) + Track = NamedTuple('Track', [('label', str), ('group', int), ('source', str), ('shapes', List[TrackedShape]), ('task_id', int)]) + Tag = NamedTuple('Tag', [('frame', int), ('label', str), ('attributes', List[InstanceLabelData.Attribute]), ('source', str), ('group', int), ('task_id', int)]) Tag.__new__.__defaults__ = (0, ) Frame = NamedTuple('Frame', [('task_id', int), ('subset', str), ('idx', int), ('frame', int), ('name', str), ('width', int), ('height', int), ('labeled_shapes', List[Union[LabeledShape, TrackedShape]]), ('tags', List[Tag])]) @@ -569,6 +569,7 @@ def _init_task_frame_offsets(self): def _init_frame_info(self): self._frame_info = dict() + original_names = DefaultDict[Tuple[str, str], int](int) for task in self._db_tasks.values(): if hasattr(task.data, 'video'): self._frame_info.update({(task.id, frame): { @@ -579,7 +580,7 @@ def _init_frame_info(self): } for frame in range(task.data.size)}) else: self._frame_info.update({(task.id, self.rel_frame_id(task.id, db_image.frame)): { - "path": mangle_image_name(db_image.path, self._frame_info.values(), task.subset), + "path": mangle_image_name(db_image.path, task.subset, original_names), "width": db_image.width, "height": db_image.height, "subset": get_defaulted_subset(task.subset, self._subsets) @@ -998,22 +999,26 @@ def GetCVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_im class CvatImportError(Exception): pass -def mangle_image_name(name: str, images: List[Literal["path", "width", "height", "subset"]], subset: str) -> str: +def mangle_image_name(name: str, subset: str, names: DefaultDict[Tuple[str, str], int]) -> str: name, ext = name.rsplit(osp.extsep, maxsplit=1) - paths: Dict[str, int] = {} - for image in images: - if image['subset'] == subset: - paths[image['path']] = 1 - if not paths[name]: + + if not names[(subset, name)]: + names[(subset, name)] += 1 return osp.extsep.join([name, ext]) else: - i = 1 - while i < sys.maxsize: - image_name = f"{name}_{i}{osp.extsep}{ext}" - if not paths[image_name]: - return image_name - i += 1 - raise Exception('Cannot mangle image name') + image_name = f"{name}_{names[(subset, name)]}" + if not names[(subset, image_name)]: + names[(subset, name)] += 1 + return osp.extsep.join([image_name, ext]) + else: + i = 1 + while i < sys.maxsize: + new_image_name = f"{image_name}_{i}" + if not names[(subset, new_image_name)]: + names[(subset, name)] += 1 + return osp.extsep.join([new_image_name, ext]) + i += 1 + raise Exception('Cannot mangle image name') def get_defaulted_subset(subset: str, subsets: List[str]) -> str: if subset: diff --git a/cvat/apps/dataset_manager/formats/pointcloud.py b/cvat/apps/dataset_manager/formats/pointcloud.py index 1fc31e4a6ef..458029a132d 100644 --- a/cvat/apps/dataset_manager/formats/pointcloud.py +++ b/cvat/apps/dataset_manager/formats/pointcloud.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, TaskData, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive from cvat.apps.engine.models import DimensionType @@ -18,6 +18,9 @@ @exporter(name='Sly Point Cloud Format', ext='ZIP', version='1.0', dimension=DimensionType.DIM_3D) def _export_images(dst_file, task_data, save_images=False): + if not isinstance(task_data, TaskData): + raise Exception("Export to \"Sly Point Cloud\" format is working only with tasks temporarily") + dataset = Dataset.from_extractors(CvatTaskDataExtractor( task_data, include_images=save_images, format_type='sly_pointcloud', dimension=DimensionType.DIM_3D), env=dm_env) diff --git a/cvat/apps/dataset_manager/formats/velodynepoint.py b/cvat/apps/dataset_manager/formats/velodynepoint.py index 404988f1599..7384f7beabe 100644 --- a/cvat/apps/dataset_manager/formats/velodynepoint.py +++ b/cvat/apps/dataset_manager/formats/velodynepoint.py @@ -20,6 +20,9 @@ @exporter(name='Kitti Raw Format', ext='ZIP', version='1.0', dimension=DimensionType.DIM_3D) def _export_images(dst_file, task_data, save_images=False): + if not isinstance(task_data, TaskData): + raise Exception("Export to \"Kitti raw\" format is working only with tasks temporarily") + dataset = Dataset.from_extractors(CvatTaskDataExtractor( task_data, include_images=save_images, format_type="kitti_raw", dimension=DimensionType.DIM_3D), env=dm_env) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 0c0b1130e11..842bec7cfdf 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -411,6 +411,8 @@ def update(self, instance, validated_data): validated_project_id = validated_data.get('project_id', None) if validated_project_id is not None and validated_project_id != instance.project_id: project = models.Project.objects.get(id=validated_data.get('project_id', None)) + if len(project.tasks) and project.tasks.first().dimension != instance.dimension: + raise serializers.ValidationError(f'Dimension ({instance.dimension}) of the task must be the same as other tasks in project ({project.tasks.first().dimension})') if instance.project_id is None: for old_label in instance.label_set.all(): try: @@ -453,8 +455,12 @@ def validate(self, attrs): # When moving task labels can be mapped to one, but when not names must be unique if 'project_id' in attrs.keys() and self.instance is not None: project_id = attrs.get('project_id') - if project_id is not None and not models.Project.objects.filter(id=project_id).count(): - raise serializers.ValidationError(f'Cannot find project with ID {project_id}') + if project_id is not None: + project = models.Project.objects.filter(id=project_id).first() + if project is None: + raise serializers.ValidationError(f'Cannot find project with ID {project_id}') + if len(project.tasks) and project.tasks.first().dimension != attrs.get('dimension'): + raise serializers.ValidationError(f'Dimension ({attrs.get("dimension")}) of the task must be the same as other tasks in project ({project.tasks.first().dimension})') # Check that all labels can be mapped new_label_names = set() old_labels = self.instance.project.label_set.all() if self.instance.project_id else self.instance.label_set.all() From 4e2aac1835689f5087050670c48d5f6fff058b23 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 22 Jul 2021 14:09:05 +0300 Subject: [PATCH 26/63] Fixed validator --- cvat/apps/engine/serializers.py | 2 -- cvat/apps/engine/task.py | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 842bec7cfdf..3b0120e6ac0 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -459,8 +459,6 @@ def validate(self, attrs): project = models.Project.objects.filter(id=project_id).first() if project is None: raise serializers.ValidationError(f'Cannot find project with ID {project_id}') - if len(project.tasks) and project.tasks.first().dimension != attrs.get('dimension'): - raise serializers.ValidationError(f'Dimension ({attrs.get("dimension")}) of the task must be the same as other tasks in project ({project.tasks.first().dimension})') # Check that all labels can be mapped new_label_names = set() old_labels = self.instance.project.label_set.all() if self.instance.project_id else self.instance.label_set.all() diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index cc6a5ffa9af..62d39568c19 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -311,6 +311,9 @@ def _create_thread(tid, data, isImport=False): validate_dimension.set_path(upload_dir) validate_dimension.validate() + if db_task.project is not None and db_task.project.tasks.count() > 1 and db_task.project.tasks.first().dimension != validate_dimension.dimension: + raise Exception(f'Dimension ({validate_dimension.dimension}) of the task must be the same as other tasks in project ({db_task.project.tasks.first().dimension})') + if validate_dimension.dimension == models.DimensionType.DIM_3D: db_task.dimension = models.DimensionType.DIM_3D From ff19e71c0f1a56774b339c88b927afc8ad83125b Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Fri, 23 Jul 2021 12:23:32 +0300 Subject: [PATCH 27/63] Fixed linter and tests --- cvat/apps/dataset_manager/bindings.py | 5 ++--- cvat/apps/engine/serializers.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 0a293221985..494a9874b61 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -5,10 +5,9 @@ import sys import os.path as osp -from collections import defaultdict, namedtuple +from collections import namedtuple from typing import Any, Callable, DefaultDict, Dict, List, Literal, Mapping, NamedTuple, OrderedDict, Tuple, Union from pathlib import Path -from PIL.Image import new from django.utils import timezone @@ -907,7 +906,7 @@ def _make_image(i, **kwargs): self._items = dm_items @staticmethod - def _load_categories(cvat_anno, dimension): + def _load_categories(cvat_anno, dimension): # pylint: disable=arguments-differ categories = {} label_categories = datumaro.LabelCategories(attributes=['occluded']) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 3b0120e6ac0..767b393e1f3 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -411,7 +411,7 @@ def update(self, instance, validated_data): validated_project_id = validated_data.get('project_id', None) if validated_project_id is not None and validated_project_id != instance.project_id: project = models.Project.objects.get(id=validated_data.get('project_id', None)) - if len(project.tasks) and project.tasks.first().dimension != instance.dimension: + if project.tasks.count() and project.tasks.first().dimension != instance.dimension: raise serializers.ValidationError(f'Dimension ({instance.dimension}) of the task must be the same as other tasks in project ({project.tasks.first().dimension})') if instance.project_id is None: for old_label in instance.label_set.all(): From dc45b9f73b3a25b4366017252b0814b31080e6d3 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 27 Jul 2021 13:29:40 +0300 Subject: [PATCH 28/63] Fixed comments --- cvat-core/src/annotations.js | 21 +--- cvat-core/src/api.js | 4 +- cvat-core/src/project-implementation.js | 96 ++++++++++--------- cvat-core/src/project.js | 10 +- cvat-core/src/server-proxy.js | 5 +- cvat-core/src/session.js | 38 -------- cvat-ui/src/actions/export-actions.ts | 17 ++-- .../components/actions-menu/actions-menu.tsx | 6 +- .../top-bar/annotation-menu.tsx | 6 +- .../export-dataset/export-dataset-modal.tsx | 68 +++++++------ .../components/projects-page/actions-menu.tsx | 2 +- .../containers/actions-menu/actions-menu.tsx | 10 +- .../top-bar/annotation-menu.tsx | 10 +- cvat-ui/src/reducers/export-reducer.ts | 60 +++--------- cvat-ui/src/reducers/interfaces.ts | 22 +---- cvat-ui/src/reducers/notifications-reducer.ts | 18 ---- cvat-ui/src/reducers/tasks-reducer.ts | 40 -------- cvat-ui/src/utils/deep-copy.ts | 21 ++++ 18 files changed, 146 insertions(+), 308 deletions(-) create mode 100644 cvat-ui/src/utils/deep-copy.ts diff --git a/cvat-core/src/annotations.js b/cvat-core/src/annotations.js index aacea299d5d..07ce90f9e2f 100644 --- a/cvat-core/src/annotations.js +++ b/cvat-core/src/annotations.js @@ -10,7 +10,7 @@ const { checkObjectType } = require('./common'); const { Project } = require('./project'); const { Task, Job } = require('./session'); - const { Loader, Dumper } = require('./annotation-formats'); + const { Loader } = require('./annotation-formats'); const { ScriptingError, DataError, ArgumentError } = require('./exceptions'); const jobCache = new WeakMap(); @@ -234,22 +234,6 @@ await serverProxy.annotations.uploadAnnotations(sessionType, session.id, file, loader.name); } - async function dumpAnnotations(session, name, dumper) { - if (!(dumper instanceof Dumper)) { - throw new ArgumentError('A dumper must be instance of Dumper class'); - } - - let result = null; - const sessionType = session instanceof Task ? 'task' : 'job'; - if (sessionType === 'job') { - result = await serverProxy.annotations.dumpAnnotations(session.task.id, name, dumper.name); - } else { - result = await serverProxy.annotations.dumpAnnotations(session.id, name, dumper.name); - } - - return result; - } - function importAnnotations(session, data) { const sessionType = session instanceof Task ? 'task' : 'job'; const cache = getCache(sessionType); @@ -282,7 +266,7 @@ throw new ArgumentError('Format must be a string'); } if (!(instance instanceof Task || instance instanceof Project || instance instanceof Job)) { - throw new ArgumentError('A dataset can only be created from a task or project'); + throw new ArgumentError('A dataset can only be created from a job, task or project'); } if (typeof saveImages !== 'boolean') { throw new ArgumentError('Save images parameter must be a boolean'); @@ -379,7 +363,6 @@ annotationsStatistics, selectObject, uploadAnnotations, - dumpAnnotations, importAnnotations, exportAnnotations, exportDataset, diff --git a/cvat-core/src/api.js b/cvat-core/src/api.js index 07fed4ac938..ac2e7312208 100644 --- a/cvat-core/src/api.js +++ b/cvat-core/src/api.js @@ -18,7 +18,7 @@ function build() { const Review = require('./review'); const { Job, Task } = require('./session'); const { Project } = require('./project'); - require('./project-implementation'); + const implementProject = require('./project-implementation'); const { Attribute, Label } = require('./labels'); const MLModel = require('./ml-model'); @@ -754,7 +754,7 @@ function build() { */ classes: { User, - Project, + Project: implementProject(Project), Task, Job, Log, diff --git a/cvat-core/src/project-implementation.js b/cvat-core/src/project-implementation.js index 14bfadc3a9a..05425120ef4 100644 --- a/cvat-core/src/project-implementation.js +++ b/cvat-core/src/project-implementation.js @@ -1,7 +1,3 @@ -// Copyright (C) 2021 Intel Corporation -// -// SPDX-License-Identifier: MIT - // Copyright (C) 2019-2021 Intel Corporation // // SPDX-License-Identifier: MIT @@ -13,58 +9,66 @@ const { Project } = require('./project'); const { exportDataset } = require('./annotations'); - Project.prototype.save.implementation = async function () { - const trainingProjectCopy = this.trainingProject; - if (typeof this.id !== 'undefined') { - // project has been already created, need to update some data - const projectData = { + function implementProject(projectClass) { + projectClass.prototype.save.implementation = async function () { + const trainingProjectCopy = this.trainingProject; + if (typeof this.id !== 'undefined') { + // project has been already created, need to update some data + const projectData = { + name: this.name, + assignee_id: this.assignee ? this.assignee.id : null, + bug_tracker: this.bugTracker, + labels: [...this._internalData.labels.map((el) => el.toJSON())], + }; + + if (trainingProjectCopy) { + projectData.training_project = trainingProjectCopy; + } + + await serverProxy.projects.save(this.id, projectData); + return this; + } + + // initial creating + const projectSpec = { name: this.name, - assignee_id: this.assignee ? this.assignee.id : null, - bug_tracker: this.bugTracker, - labels: [...this._internalData.labels.map((el) => el.toJSON())], + labels: [...this.labels.map((el) => el.toJSON())], }; - if (trainingProjectCopy) { - projectData.training_project = trainingProjectCopy; + if (this.bugTracker) { + projectSpec.bug_tracker = this.bugTracker; } - await serverProxy.projects.save(this.id, projectData); - return this; - } + if (trainingProjectCopy) { + projectSpec.training_project = trainingProjectCopy; + } - // initial creating - const projectSpec = { - name: this.name, - labels: [...this.labels.map((el) => el.toJSON())], + const project = await serverProxy.projects.create(projectSpec); + return new Project(project); }; - if (this.bugTracker) { - projectSpec.bug_tracker = this.bugTracker; - } - - if (trainingProjectCopy) { - projectSpec.training_project = trainingProjectCopy; - } + projectClass.prototype.delete.implementation = async function () { + const result = await serverProxy.projects.delete(this.id); + return result; + }; - const project = await serverProxy.projects.create(projectSpec); - return new Project(project); - }; + projectClass.prototype.preview.implementation = async function () { + if (!this._internalData.task_ids.length) { + return ''; + } + const frameData = await getPreview(this._internalData.task_ids[0]); + return frameData; + }; - Project.prototype.delete.implementation = async function () { - const result = await serverProxy.projects.delete(this.id); - return result; - }; + projectClass.prototype.annotations.exportDataset.implementation = async function ( + format, saveImages, customName, + ) { + const result = exportDataset(this, format, customName, saveImages); + return result; + }; - Project.prototype.preview.implementation = async function () { - if (!this._internalData.task_ids.length) { - return ''; - } - const frameData = await getPreview(this._internalData.task_ids[0]); - return frameData; - }; + return projectClass; + } - Project.prototype.annotations.exportDataset.implementation = async function (format, saveImages, customName) { - const result = exportDataset(this, format, customName, saveImages); - return result; - }; + module.exports = implementProject; })(); diff --git a/cvat-core/src/project.js b/cvat-core/src/project.js index 23294be9652..bd25fc0f1b1 100644 --- a/cvat-core/src/project.js +++ b/cvat-core/src/project.js @@ -201,7 +201,7 @@ }, }, /** - * Tasks linked with the project + * Tasks related with the project * @name tasks * @type {module:API.cvat.classes.Task[]} * @memberof module:API.cvat.classes.Project @@ -212,7 +212,7 @@ get: () => [...data.tasks], }, /** - * Subsets array for linked tasks + * Subsets array for related tasks * @name subsets * @type {string[]} * @memberof module:API.cvat.classes.Project @@ -253,8 +253,8 @@ }), ); - // When we call a function, for example: task.annotations.get() - // In the method get we lose the task context + // When we call a function, for example: project.annotations.get() + // In the method get we lose the project context // So, we need return it this.annotations = { exportDataset: Object.getPrototypeOf(this).annotations.exportDataset.bind(this), @@ -294,7 +294,7 @@ } /** - * Method deletes a task from a server + * Method deletes a project from a server * @method delete * @memberof module:API.cvat.classes.Project * @readonly diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index 082a430b937..3d914ce9db7 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -1145,8 +1145,9 @@ const closureId = Date.now(); predictAnnotations.latestRequest.id = closureId; - // eslint-disable-next-line max-len - const predicate = () => !predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId; + const predicate = () => ( + !predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId + ); if (predictAnnotations.latestRequest.fetching) { waitFor(5, predicate).then(() => { if (predictAnnotations.latestRequest.id !== closureId) { diff --git a/cvat-core/src/session.js b/cvat-core/src/session.js index 88ff9388a37..c5366ea1d5f 100644 --- a/cvat-core/src/session.js +++ b/cvat-core/src/session.js @@ -42,16 +42,6 @@ return result; }, - async dump(dumper, name = null) { - const result = await PluginRegistry.apiWrapper.call( - this, - prototype.annotations.dump, - dumper, - name, - ); - return result; - }, - async statistics() { const result = await PluginRegistry.apiWrapper.call(this, prototype.annotations.statistics); return result; @@ -331,21 +321,6 @@ * @instance * @async */ - /** - * Dump of annotations to a file. - * Method always dumps annotations for a whole task. - * @method dump - * @memberof Session.annotations - * @param {module:API.cvat.classes.Dumper} dumper - a dumper - * @param {string} [name = null] - a name of a file with annotations - * which will be used to dump - * @returns {string} URL which can be used in order to get a dump file - * @throws {module:API.cvat.exceptions.PluginError} - * @throws {module:API.cvat.exceptions.ServerError} - * @throws {module:API.cvat.exceptions.ArgumentError} - * @instance - * @async - */ /** * Collect short statistics about a task or a job. * @method statistics @@ -879,7 +854,6 @@ get: Object.getPrototypeOf(this).annotations.get.bind(this), put: Object.getPrototypeOf(this).annotations.put.bind(this), save: Object.getPrototypeOf(this).annotations.save.bind(this), - dump: Object.getPrototypeOf(this).annotations.dump.bind(this), merge: Object.getPrototypeOf(this).annotations.merge.bind(this), split: Object.getPrototypeOf(this).annotations.split.bind(this), group: Object.getPrototypeOf(this).annotations.group.bind(this), @@ -1577,7 +1551,6 @@ get: Object.getPrototypeOf(this).annotations.get.bind(this), put: Object.getPrototypeOf(this).annotations.put.bind(this), save: Object.getPrototypeOf(this).annotations.save.bind(this), - dump: Object.getPrototypeOf(this).annotations.dump.bind(this), merge: Object.getPrototypeOf(this).annotations.merge.bind(this), split: Object.getPrototypeOf(this).annotations.split.bind(this), group: Object.getPrototypeOf(this).annotations.group.bind(this), @@ -1717,7 +1690,6 @@ selectObject, annotationsStatistics, uploadAnnotations, - dumpAnnotations, importAnnotations, exportAnnotations, exportDataset, @@ -1950,11 +1922,6 @@ return result; }; - Job.prototype.annotations.dump.implementation = async function (dumper, name) { - const result = await dumpAnnotations(this, name, dumper); - return result; - }; - Job.prototype.annotations.exportDataset.implementation = async function (format, saveImages, customName) { const result = await exportDataset(this.task, format, customName, saveImages); return result; @@ -2254,11 +2221,6 @@ return result; }; - Task.prototype.annotations.dump.implementation = async function (dumper, name) { - const result = await dumpAnnotations(this, name, dumper); - return result; - }; - Task.prototype.annotations.import.implementation = function (data) { const result = importAnnotations(this, data); return result; diff --git a/cvat-ui/src/actions/export-actions.ts b/cvat-ui/src/actions/export-actions.ts index 1e1e361c8fe..d5a2b801b90 100644 --- a/cvat-ui/src/actions/export-actions.ts +++ b/cvat-ui/src/actions/export-actions.ts @@ -15,15 +15,14 @@ export enum ExportActionTypes { export const exportActions = { openExportModal: (instance: any) => createAction(ExportActionTypes.OPEN_EXPORT_MODAL, { instance }), closeExportModal: () => createAction(ExportActionTypes.CLOSE_EXPORT_MODAL), - exportDataset: (instance: any, format: any, saveImages: boolean) => - createAction(ExportActionTypes.EXPORT_DATASET, { instance, format, saveImages }), - exportDatasetSuccess: (instance: any, format: any, saveImages: boolean) => - createAction(ExportActionTypes.EXPORT_DATASET_SUCCESS, { instance, format, saveImages }), - exportDatasetFailed: (instance: any, format: any, saveImages: boolean, error: any) => + exportDataset: (instance: any, format: string) => + createAction(ExportActionTypes.EXPORT_DATASET, { instance, format }), + exportDatasetSuccess: (instance: any, format: string) => + createAction(ExportActionTypes.EXPORT_DATASET_SUCCESS, { instance, format }), + exportDatasetFailed: (instance: any, format: string, error: any) => createAction(ExportActionTypes.EXPORT_DATASET_FAILED, { instance, format, - saveImages, error, }), }; @@ -34,16 +33,16 @@ export const exportDatasetAsync = ( name: string, saveImages: boolean, ): ThunkAction => async (dispatch) => { - dispatch(exportActions.exportDataset(instance, format, saveImages)); + dispatch(exportActions.exportDataset(instance, format)); try { const url = await instance.annotations.exportDataset(format, saveImages, name); const downloadAnchor = window.document.getElementById('downloadAnchor') as HTMLAnchorElement; downloadAnchor.href = url; downloadAnchor.click(); - dispatch(exportActions.exportDatasetSuccess(instance, format, saveImages)); + dispatch(exportActions.exportDatasetSuccess(instance, format)); } catch (error) { - dispatch(exportActions.exportDatasetFailed(instance, format, saveImages, error)); + dispatch(exportActions.exportDatasetFailed(instance, format, error)); } }; diff --git a/cvat-ui/src/components/actions-menu/actions-menu.tsx b/cvat-ui/src/components/actions-menu/actions-menu.tsx index 7c6bab8aab4..d208ef682d0 100644 --- a/cvat-ui/src/components/actions-menu/actions-menu.tsx +++ b/cvat-ui/src/components/actions-menu/actions-menu.tsx @@ -19,8 +19,6 @@ interface Props { loaders: any[]; dumpers: any[]; loadActivity: string | null; - dumpActivities: string[] | null; - exportActivities: string[] | null; inferenceIsActive: boolean; taskDimension: DimensionType; onClickMenu: (params: MenuInfo, file?: File) => void; @@ -108,14 +106,14 @@ export default function ActionsMenuComponent(props: Props): JSX.Element { menuKey: Actions.LOAD_TASK_ANNO, taskDimension, })} - Export Task dataset + Export task dataset {!!bugTracker && Open bug tracker} Automatic annotation {exportIsActive && } - Export Task + Export task
Move to project diff --git a/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx b/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx index d2bc001cd52..1a9a10164eb 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx @@ -8,6 +8,7 @@ import Modal from 'antd/lib/modal'; // eslint-disable-next-line import/no-extraneous-dependencies import { MenuInfo } from 'rc-menu/lib/interface'; +import ExportDatasetModal from 'components/export-dataset/export-dataset-modal'; import LoadSubmenu from 'components/actions-menu/load-submenu'; import { DimensionType } from '../../../reducers/interfaces'; @@ -16,8 +17,6 @@ interface Props { loaders: any[]; dumpers: any[]; loadActivity: string | null; - dumpActivities: string[] | null; - exportActivities: string[] | null; isReviewer: boolean; jobInstance: any; onClickMenu(params: MenuInfo, file?: File): void; @@ -165,7 +164,7 @@ export default function AnnotationMenuComponent(props: Props): JSX.Element { menuKey: Actions.LOAD_JOB_ANNO, taskDimension: jobInstance.task.dimension, })} - Export Task dataset + Export task dataset Remove annotations e.preventDefault()}> @@ -178,6 +177,7 @@ export default function AnnotationMenuComponent(props: Props): JSX.Element { Submit the review )} {jobStatus === 'completed' && Renew the job} + ); } diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx index bf1db4a5bd8..e48220557f1 100644 --- a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -3,7 +3,7 @@ // SPDX-License-Identifier: MIT import './styles.scss'; -import React, { useState, useEffect } from 'react'; +import React, { useState, useEffect, useCallback } from 'react'; import Modal from 'antd/lib/modal'; import { useSelector, useDispatch } from 'react-redux'; import { DownloadOutlined, LoadingOutlined } from '@ant-design/icons'; @@ -25,7 +25,7 @@ type FormValues = { customName: string | undefined; }; -export default function ExportDatasetModal(): JSX.Element { +function ExportDatasetModal(): JSX.Element { const [instanceType, setInstanceType] = useState(''); const [activities, setActivities] = useState([]); const [form] = Form.useForm(); @@ -34,21 +34,16 @@ export default function ExportDatasetModal(): JSX.Element { const modalVisible = useSelector((state: CombinedState) => state.export.modalVisible); const dumpers = useSelector((state: CombinedState) => state.formats.annotationFormats.dumpers); const { - tasks: { datasets: taskExportActivities, annotations: taskDumpActivities }, - projects: { datasets: projectExportActivities, annotations: projectDumpActivities }, + tasks: taskExportActivities, projects: projectExportActivities, } = useSelector((state: CombinedState) => state.export); const initActivities = (): void => { if (instance instanceof core.classes.Project) { setInstanceType('project'); - setActivities( - (form.getFieldValue('saveImages') ? projectExportActivities : projectDumpActivities)[instance.id] || [], - ); + setActivities(projectExportActivities[instance.id] || []); } else if (instance instanceof core.classes.Task) { setInstanceType('task'); - setActivities( - (form.getFieldValue('saveImages') ? taskExportActivities : taskDumpActivities)[instance.id] || [], - ); + setActivities(taskExportActivities[instance.id] || []); } }; @@ -68,35 +63,35 @@ export default function ExportDatasetModal(): JSX.Element { initActivities(); }; - const handleExport = (values: FormValues): void => { + const handleExport = useCallback((values: FormValues): void => { // have to validate format before so it would not be undefined dispatch( exportDatasetAsync(instance, values.selectedFormat as string, values.customName || '', values.saveImages), ); closeModal(); - }; + }, [instance?.id, instance instanceof core.classes.Project]); return ( -
form.submit()} > - form.submit()} + a.name.localeCompare(b.name)) .filter( (dumper: any): boolean => - !(instance instanceof core.classes.Task && dumper.dimension !== instance.dimension), + !(instance instanceof core.classes.Task) || + dumper.dimension === instance?.dimension, ) .map( (dumper: any): JSX.Element => { @@ -134,9 +130,11 @@ export default function ExportDatasetModal(): JSX.Element { Save images - + - -
+ + ); } + +export default React.memo(ExportDatasetModal); diff --git a/cvat-ui/src/components/projects-page/actions-menu.tsx b/cvat-ui/src/components/projects-page/actions-menu.tsx index ea3059f5123..75d71508652 100644 --- a/cvat-ui/src/components/projects-page/actions-menu.tsx +++ b/cvat-ui/src/components/projects-page/actions-menu.tsx @@ -41,7 +41,7 @@ export default function ProjectActionsMenuComponent(props: Props): JSX.Element { dispatch(exportActions.openExportModal(projectInstance))} > - Export dataset + Export project dataset ); diff --git a/cvat-ui/src/containers/actions-menu/actions-menu.tsx b/cvat-ui/src/containers/actions-menu/actions-menu.tsx index 10d8c02329b..5c38825cb10 100644 --- a/cvat-ui/src/containers/actions-menu/actions-menu.tsx +++ b/cvat-ui/src/containers/actions-menu/actions-menu.tsx @@ -26,8 +26,6 @@ interface OwnProps { interface StateToProps { annotationFormats: any; loadActivity: string | null; - dumpActivities: string[] | null; - exportActivities: string[] | null; inferenceIsActive: boolean; exportIsActive: boolean; } @@ -50,14 +48,12 @@ function mapStateToProps(state: CombinedState, own: OwnProps): StateToProps { formats: { annotationFormats }, tasks: { activities: { - dumps, loads, exports: activeExports, backups, + loads, backups, }, }, } = state; return { - dumpActivities: tid in dumps ? dumps[tid] : null, - exportActivities: tid in activeExports ? activeExports[tid] : null, loadActivity: tid in loads ? loads[tid] : null, annotationFormats, inferenceIsActive: tid in state.models.inferences, @@ -93,8 +89,6 @@ function ActionsMenuContainer(props: OwnProps & StateToProps & DispatchToProps): taskInstance, annotationFormats: { loaders, dumpers }, loadActivity, - dumpActivities, - exportActivities, inferenceIsActive, exportIsActive, @@ -142,8 +136,6 @@ function ActionsMenuContainer(props: OwnProps & StateToProps & DispatchToProps): loaders={loaders} dumpers={dumpers} loadActivity={loadActivity} - dumpActivities={dumpActivities} - exportActivities={exportActivities} inferenceIsActive={inferenceIsActive} onClickMenu={onClickMenu} taskDimension={taskInstance.dimension} diff --git a/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx b/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx index 2f40d414698..60d5fef6fa6 100644 --- a/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx +++ b/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx @@ -25,8 +25,6 @@ interface StateToProps { annotationFormats: any; jobInstance: any; loadActivity: string | null; - dumpActivities: string[] | null; - exportActivities: string[] | null; user: any; } @@ -49,7 +47,7 @@ function mapStateToProps(state: CombinedState): StateToProps { }, formats: { annotationFormats }, tasks: { - activities: { dumps, loads, exports: activeExports }, + activities: { loads }, }, auth: { user }, } = state; @@ -58,8 +56,6 @@ function mapStateToProps(state: CombinedState): StateToProps { const jobID = jobInstance.id; return { - dumpActivities: taskID in dumps ? dumps[taskID] : null, - exportActivities: taskID in activeExports ? activeExports[taskID] : null, loadActivity: taskID in loads || jobID in jobLoads ? loads[taskID] || jobLoads[jobID] : null, jobInstance, annotationFormats, @@ -105,8 +101,6 @@ function AnnotationMenuContainer(props: Props): JSX.Element { annotationFormats: { loaders, dumpers }, history, loadActivity, - dumpActivities, - exportActivities, loadAnnotations, showExportModal, removeAnnotations, @@ -159,8 +153,6 @@ function AnnotationMenuContainer(props: Props): JSX.Element { loaders={loaders} dumpers={dumpers} loadActivity={loadActivity} - dumpActivities={dumpActivities} - exportActivities={exportActivities} onClickMenu={onClickMenu} setForceExitAnnotationFlag={setForceExitAnnotationFlag} saveAnnotations={saveAnnotations} diff --git a/cvat-ui/src/reducers/export-reducer.ts b/cvat-ui/src/reducers/export-reducer.ts index b1e06c78c2d..a7a023524f0 100644 --- a/cvat-ui/src/reducers/export-reducer.ts +++ b/cvat-ui/src/reducers/export-reducer.ts @@ -4,19 +4,15 @@ import { ExportActions, ExportActionTypes } from 'actions/export-actions'; import getCore from 'cvat-core-wrapper'; +import deepCopy from 'utils/deep-copy'; + import { ExportState } from './interfaces'; const core = getCore(); const defaultState: ExportState = { - tasks: { - datasets: {}, - annotations: {}, - }, - projects: { - datasets: {}, - annotations: {}, - }, + tasks: {}, + projects: {}, instance: null, modalVisible: false, }; @@ -36,13 +32,8 @@ export default (state: ExportState = defaultState, action: ExportActions): Expor instance: null, }; case ExportActionTypes.EXPORT_DATASET: { - const { instance, format, saveImages } = action.payload; - let activities; - if (instance instanceof core.classes.Project) { - activities = saveImages ? state.projects.datasets : state.projects.annotations; - } else { - activities = saveImages ? state.tasks.datasets : state.tasks.annotations; - } + const { instance, format } = action.payload; + const activities = deepCopy(instance instanceof core.classes.Project ? state.projects : state.tasks); activities[instance.id] = instance.id in activities && !activities[instance.id].includes(format) ? @@ -51,30 +42,14 @@ export default (state: ExportState = defaultState, action: ExportActions): Expor return { ...state, - tasks: { - datasets: instance instanceof core.classes.Task && saveImages ? activities : state.tasks.datasets, - annotations: - instance instanceof core.classes.Task && !saveImages ? activities : state.tasks.annotations, - }, - projects: { - datasets: - instance instanceof core.classes.Project && saveImages ? activities : state.projects.datasets, - annotations: - instance instanceof core.classes.Project && !saveImages ? - activities : - state.projects.annotations, - }, + tasks: instance instanceof core.classes.Task ? activities : state.tasks, + projects: instance instanceof core.classes.Project ? activities : state.projects, }; } case ExportActionTypes.EXPORT_DATASET_FAILED: case ExportActionTypes.EXPORT_DATASET_SUCCESS: { - const { instance, format, saveImages } = action.payload; - let activities; - if (instance instanceof core.classes.Project) { - activities = saveImages ? state.projects.datasets : state.projects.annotations; - } else { - activities = saveImages ? state.tasks.datasets : state.tasks.annotations; - } + const { instance, format } = action.payload; + const activities = deepCopy(instance instanceof core.classes.Project ? state.projects : state.tasks); activities[instance.id] = activities[instance.id].filter( (exporterName: string): boolean => exporterName !== format, @@ -82,19 +57,8 @@ export default (state: ExportState = defaultState, action: ExportActions): Expor return { ...state, - tasks: { - datasets: instance instanceof core.classes.Task && saveImages ? activities : state.tasks.datasets, - annotations: - instance instanceof core.classes.Task && !saveImages ? activities : state.tasks.annotations, - }, - projects: { - datasets: - instance instanceof core.classes.Project && saveImages ? activities : state.projects.datasets, - annotations: - instance instanceof core.classes.Project && !saveImages ? - activities : - state.projects.annotations, - }, + tasks: instance instanceof core.classes.Task ? activities : state.tasks, + projects: instance instanceof core.classes.Project ? activities : state.projects, }; } default: diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index a6894f1180e..50a051b071e 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -86,14 +86,6 @@ export interface TasksState { count: number; current: Task[]; activities: { - dumps: { - // dumps in different formats at the same time - [tid: number]: string[]; // dumper names - }; - exports: { - // exports in different formats at the same time - [tid: number]: string[]; // dumper names - }; loads: { // only one loading simultaneously [tid: number]: string; // loader name @@ -114,20 +106,10 @@ export interface TasksState { export interface ExportState { tasks: { - datasets: { - [tid: number]: string[]; - }; - annotations: { - [tid: number]: string[]; - }; + [tid: number]: string[]; }; projects: { - datasets: { - [pid: number]: string[]; - }; - annotations: { - [pid: number]: string[]; - }; + [pid: number]: string[]; }; instance: any; modalVisible: boolean; diff --git a/cvat-ui/src/reducers/notifications-reducer.ts b/cvat-ui/src/reducers/notifications-reducer.ts index 225034333f4..34907060ed7 100644 --- a/cvat-ui/src/reducers/notifications-reducer.ts +++ b/cvat-ui/src/reducers/notifications-reducer.ts @@ -398,24 +398,6 @@ export default function (state = defaultState, action: AnyAction): Notifications }, }; } - case TasksActionTypes.DUMP_ANNOTATIONS_FAILED: { - const taskID = action.payload.task.id; - return { - ...state, - errors: { - ...state.errors, - tasks: { - ...state.errors.tasks, - dumping: { - message: - 'Could not dump annotations for the ' + - `task ${taskID}`, - reason: action.payload.error.toString(), - }, - }, - }, - }; - } case TasksActionTypes.DELETE_TASK_FAILED: { const { taskID } = action.payload; return { diff --git a/cvat-ui/src/reducers/tasks-reducer.ts b/cvat-ui/src/reducers/tasks-reducer.ts index 48edc997639..592286d1650 100644 --- a/cvat-ui/src/reducers/tasks-reducer.ts +++ b/cvat-ui/src/reducers/tasks-reducer.ts @@ -32,8 +32,6 @@ const defaultState: TasksState = { mode: null, }, activities: { - dumps: {}, - exports: {}, loads: {}, deletes: {}, creates: { @@ -85,44 +83,6 @@ export default (state: TasksState = defaultState, action: AnyAction): TasksState initialized: true, fetching: false, }; - case TasksActionTypes.DUMP_ANNOTATIONS: { - const { task } = action.payload; - const { dumper } = action.payload; - const { dumps } = state.activities; - - dumps[task.id] = - task.id in dumps && !dumps[task.id].includes(dumper.name) ? - [...dumps[task.id], dumper.name] : - dumps[task.id] || [dumper.name]; - - return { - ...state, - activities: { - ...state.activities, - dumps: { - ...dumps, - }, - }, - }; - } - case TasksActionTypes.DUMP_ANNOTATIONS_FAILED: - case TasksActionTypes.DUMP_ANNOTATIONS_SUCCESS: { - const { task } = action.payload; - const { dumper } = action.payload; - const { dumps } = state.activities; - - dumps[task.id] = dumps[task.id].filter((dumperName: string): boolean => dumperName !== dumper.name); - - return { - ...state, - activities: { - ...state.activities, - dumps: { - ...dumps, - }, - }, - }; - } case TasksActionTypes.LOAD_ANNOTATIONS: { const { task } = action.payload; const { loader } = action.payload; diff --git a/cvat-ui/src/utils/deep-copy.ts b/cvat-ui/src/utils/deep-copy.ts new file mode 100644 index 00000000000..986c4dbea5b --- /dev/null +++ b/cvat-ui/src/utils/deep-copy.ts @@ -0,0 +1,21 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +function deepCopy(obj: T): T { + if (typeof obj !== 'object') { + return obj; + } + if (!obj) { + return obj; + } + const container: any = (obj instanceof Array) ? [] : {}; + for (const i in obj) { + if (Object.prototype.hasOwnProperty.call(obj, i)) { + container[i] = deepCopy(obj[i]); + } + } + return container; +} + +export default deepCopy; From 6c683acfef3c31f0c22ac7438b3706546bb60e0c Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 27 Jul 2021 16:33:41 +0300 Subject: [PATCH 29/63] Added file extesion to the form --- .../src/components/export-dataset/export-dataset-modal.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx index e48220557f1..7e5a6782bb3 100644 --- a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -66,7 +66,7 @@ function ExportDatasetModal(): JSX.Element { const handleExport = useCallback((values: FormValues): void => { // have to validate format before so it would not be undefined dispatch( - exportDatasetAsync(instance, values.selectedFormat as string, values.customName || '', values.saveImages), + exportDatasetAsync(instance, values.selectedFormat as string, values.customName ? `${values.customName}.zip` : '', values.saveImages), ); closeModal(); }, [instance?.id, instance instanceof core.classes.Project]); @@ -130,7 +130,7 @@ function ExportDatasetModal(): JSX.Element { Save images - + From abdb59f9fd2fdfda4b69c17eb1a809d9e481c449 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 27 Jul 2021 17:11:19 +0300 Subject: [PATCH 30/63] Added notification --- .../export-dataset/export-dataset-modal.tsx | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx index 7e5a6782bb3..9efbd8afb0e 100644 --- a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -5,6 +5,7 @@ import './styles.scss'; import React, { useState, useEffect, useCallback } from 'react'; import Modal from 'antd/lib/modal'; +import Notification from 'antd/lib/notification'; import { useSelector, useDispatch } from 'react-redux'; import { DownloadOutlined, LoadingOutlined } from '@ant-design/icons'; import Text from 'antd/lib/typography/Text'; @@ -56,19 +57,17 @@ function ExportDatasetModal(): JSX.Element { dispatch(exportActions.closeExportModal()); }; - const handleValuesChange = (changedValues: any): void => { - if ('saveImages' in changedValues) { - form.setFieldsValue({ selectedFormat: undefined }); - } - initActivities(); - }; - const handleExport = useCallback((values: FormValues): void => { // have to validate format before so it would not be undefined dispatch( exportDatasetAsync(instance, values.selectedFormat as string, values.customName ? `${values.customName}.zip` : '', values.saveImages), ); closeModal(); + Notification.info({ + message: 'Dataset export started', + description: `Dataset export was started for ${instanceType} #${instance?.id}. ` + + 'Download will start automaticly as soon as the dataset is ready.', + }); }, [instance?.id, instance instanceof core.classes.Project]); return ( @@ -83,7 +82,6 @@ function ExportDatasetModal(): JSX.Element { form={form} labelCol={{ span: 8 }} wrapperCol={{ span: 16 }} - onValuesChange={handleValuesChange} initialValues={ { selectedFormat: undefined, From f4c4e87be18d2c473edde929eede9a56b2fd6715 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 27 Jul 2021 17:12:48 +0300 Subject: [PATCH 31/63] Fixed header --- cvat-core/src/project-implementation.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-core/src/project-implementation.js b/cvat-core/src/project-implementation.js index 05425120ef4..c5bb2387099 100644 --- a/cvat-core/src/project-implementation.js +++ b/cvat-core/src/project-implementation.js @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2021 Intel Corporation +// Copyright (C) 2021 Intel Corporation // // SPDX-License-Identifier: MIT From 76c073ce830bd5e09b44468b397287c87d16017f Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 27 Jul 2021 17:26:43 +0300 Subject: [PATCH 32/63] Added default format for tasks --- .../src/components/export-dataset/export-dataset-modal.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx index 9efbd8afb0e..b2af6ae6d5f 100644 --- a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -45,6 +45,11 @@ function ExportDatasetModal(): JSX.Element { } else if (instance instanceof core.classes.Task) { setInstanceType('task'); setActivities(taskExportActivities[instance.id] || []); + if (instance.mode === 'interpolation') { + form.setFieldsValue({ selectedFormat: 'CVAT for video 1.1' }); + } else if (instance.mode === 'annotation') { + form.setFieldsValue({ selectedFormat: 'CVAT for images 1.1' }); + } } }; From 370f3375ea3f5518e3e12f473b712eaf1e7e5510 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 28 Jul 2021 15:41:38 +0300 Subject: [PATCH 33/63] Fixed test --- .../integration/actions_tasks2/case_97_export_import_task.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cypress/integration/actions_tasks2/case_97_export_import_task.js b/tests/cypress/integration/actions_tasks2/case_97_export_import_task.js index d3f54d02e2b..f23ac5d9607 100644 --- a/tests/cypress/integration/actions_tasks2/case_97_export_import_task.js +++ b/tests/cypress/integration/actions_tasks2/case_97_export_import_task.js @@ -70,7 +70,7 @@ context('Export, import an annotation task.', { browser: '!firefox' }, () => { cy.get('.ant-dropdown') .not('.ant-dropdown-hidden') .within(() => { - cy.contains('[role="menuitem"]', 'Export Task').click().trigger('mouseout'); + cy.contains('[role="menuitem"]', 'Export task').click().trigger('mouseout'); }); cy.wait('@exportTask', { timeout: 5000 }).its('response.statusCode').should('equal', 202); cy.wait('@exportTask').its('response.statusCode').should('equal', 201); From c0765841490f5107769b783032875afc347dd3b2 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 29 Jul 2021 13:30:53 +0300 Subject: [PATCH 34/63] Fixed notification --- cvat-ui/src/components/export-dataset/export-dataset-modal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx index b2af6ae6d5f..49f09578f5a 100644 --- a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -73,7 +73,7 @@ function ExportDatasetModal(): JSX.Element { description: `Dataset export was started for ${instanceType} #${instance?.id}. ` + 'Download will start automaticly as soon as the dataset is ready.', }); - }, [instance?.id, instance instanceof core.classes.Project]); + }, [instance?.id, instance instanceof core.classes.Project, instanceType]); return ( Date: Thu, 29 Jul 2021 13:42:45 +0300 Subject: [PATCH 35/63] Added some classes --- cvat-ui/src/components/export-dataset/export-dataset-modal.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx index b2af6ae6d5f..afe9d610b82 100644 --- a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -72,6 +72,7 @@ function ExportDatasetModal(): JSX.Element { message: 'Dataset export started', description: `Dataset export was started for ${instanceType} #${instance?.id}. ` + 'Download will start automaticly as soon as the dataset is ready.', + className: `cvat-notification-notice-export-${instanceType}-start`, }); }, [instance?.id, instance instanceof core.classes.Project]); @@ -81,6 +82,7 @@ function ExportDatasetModal(): JSX.Element { visible={modalVisible} onCancel={closeModal} onOk={() => form.submit()} + className={`cvat-modal-export-${instanceType}-dataset`} >
Date: Thu, 29 Jul 2021 13:43:10 +0300 Subject: [PATCH 36/63] Tests adaptation --- .../case_97_export_import_task.js | 2 +- .../actions_tasks3/case_47_export_dataset.js | 22 ++++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/cypress/integration/actions_tasks2/case_97_export_import_task.js b/tests/cypress/integration/actions_tasks2/case_97_export_import_task.js index f23ac5d9607..c558a621cd9 100644 --- a/tests/cypress/integration/actions_tasks2/case_97_export_import_task.js +++ b/tests/cypress/integration/actions_tasks2/case_97_export_import_task.js @@ -70,7 +70,7 @@ context('Export, import an annotation task.', { browser: '!firefox' }, () => { cy.get('.ant-dropdown') .not('.ant-dropdown-hidden') .within(() => { - cy.contains('[role="menuitem"]', 'Export task').click().trigger('mouseout'); + cy.contains('[role="menuitem"]', new RegExp('^Export task$')).click().trigger('mouseout'); }); cy.wait('@exportTask', { timeout: 5000 }).its('response.statusCode').should('equal', 202); cy.wait('@exportTask').its('response.statusCode').should('equal', 201); diff --git a/tests/cypress/integration/actions_tasks3/case_47_export_dataset.js b/tests/cypress/integration/actions_tasks3/case_47_export_dataset.js index 97fcc1d3f46..d23c818a19c 100644 --- a/tests/cypress/integration/actions_tasks3/case_47_export_dataset.js +++ b/tests/cypress/integration/actions_tasks3/case_47_export_dataset.js @@ -6,8 +6,9 @@ import { taskName, labelName } from '../../support/const'; -context('Export as a dataset.', () => { +context('Export task dataset.', () => { const caseId = '47'; + const exportFormat = 'CVAT for images'; const rectangleShape2Points = { points: 'By 2 Points', type: 'Shape', @@ -17,22 +18,31 @@ context('Export as a dataset.', () => { secondX: 500, secondY: 200, }; + let datasetArchiveName = ''; before(() => { cy.openTaskJob(taskName); cy.createRectangle(rectangleShape2Points); - cy.saveJob(); + cy.saveJob('PATCH', 200, 'saveJobExportDataset'); }); describe(`Testing case "${caseId}"`, () => { - it('Go to Menu. Press "Export as a dataset" -> "CVAT for images".', () => { + it(`Go to Menu. Press "Export task dataset" with the "${exportFormat}" format.`, () => { cy.intercept('GET', '/api/v1/tasks/**/dataset**').as('exportDataset'); - cy.interactMenu('Export as a dataset'); - cy.get('.cvat-menu-export-submenu-item').within(() => { - cy.contains('CVAT for images').click(); + cy.interactMenu('Export task dataset'); + cy.get('.cvat-modal-export-task-dataset').within(() => { + cy.get('.cvat-modal-export-select').should('contain.text', exportFormat); + cy.get('[type="checkbox"]').should('not.be.checked').check(); + cy.contains('button', 'OK').click(); }); cy.wait('@exportDataset', { timeout: 5000 }).its('response.statusCode').should('equal', 202); cy.wait('@exportDataset').its('response.statusCode').should('equal', 201); + cy.task('listFiles', 'cypress/fixtures').each((fileName) => { + if (fileName.includes(exportFormat.toLowerCase())) { + datasetArchiveName = fileName; + cy.readZipArchive(datasetArchiveName) + } + }); }); }); }); From f129f762310abbe8e2ca1eb1333fb1c299ee59ff Mon Sep 17 00:00:00 2001 From: dvkruchinin Date: Thu, 29 Jul 2021 15:24:48 +0300 Subject: [PATCH 37/63] Rename classes --- cvat-ui/src/components/export-dataset/export-dataset-modal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx index 509ebc39b0b..f71d15dfd5b 100644 --- a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -82,7 +82,7 @@ function ExportDatasetModal(): JSX.Element { visible={modalVisible} onCancel={closeModal} onOk={() => form.submit()} - className={`cvat-modal-export-${instanceType}-dataset`} + className={`cvat-modal-export-${instanceType}`} > Date: Thu, 29 Jul 2021 15:25:06 +0300 Subject: [PATCH 38/63] The next tests adapdation --- .../actions_tasks/case_52_dump_upload_annotation.js | 7 ++++--- ...e_2473_import_annotations_frames_dots_in_name.js | 13 +++++++++---- .../actions_tasks3/case_47_export_dataset.js | 11 +++-------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/cypress/integration/actions_tasks/case_52_dump_upload_annotation.js b/tests/cypress/integration/actions_tasks/case_52_dump_upload_annotation.js index b46a0c55758..ec131b67e13 100644 --- a/tests/cypress/integration/actions_tasks/case_52_dump_upload_annotation.js +++ b/tests/cypress/integration/actions_tasks/case_52_dump_upload_annotation.js @@ -68,9 +68,10 @@ context('Dump/Upload annotation.', { browser: '!firefox' }, () => { it('Save job. Dump annotation. Remove annotation. Save job.', () => { cy.saveJob('PATCH', 200, 'saveJobDump'); cy.intercept('GET', '/api/v1/tasks/**/annotations**').as('dumpAnnotations'); - cy.interactMenu('Dump annotations'); - cy.get('.cvat-menu-dump-submenu-item').within(() => { - cy.contains(dumpType).click(); + cy.interactMenu('Export task dataset'); + cy.get('.cvat-modal-export-task').within(() => { + cy.get('.cvat-modal-export-select').should('contain.text', dumpType); + cy.contains('button', 'OK').click(); }); cy.wait('@dumpAnnotations', { timeout: 5000 }).its('response.statusCode').should('equal', 202); cy.wait('@dumpAnnotations').its('response.statusCode').should('equal', 201); diff --git a/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js b/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js index 3d43f3d3a13..9cb52acc602 100644 --- a/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js +++ b/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js @@ -66,10 +66,15 @@ context('Import annotations for frames with dots in name.', { browser: '!firefox it('Save job. Dump annotation to YOLO format. Remove annotation. Save job.', () => { cy.saveJob('PATCH', 200, 'saveJobDump'); cy.intercept('GET', '/api/v1/tasks/**/annotations**').as('dumpAnnotations'); - cy.interactMenu('Dump annotations'); - cy.get('.cvat-menu-dump-submenu-item').within(() => { - cy.contains(dumpType).click(); - }); + cy.interactMenu('Export task dataset'); + cy.get('.cvat-modal-export-task').find('.cvat-modal-export-select').click(); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .trigger('wheel', {deltaY: 700}) + .contains('.cvat-modal-export-option-item', dumpType) + .click(); + cy.get('.cvat-modal-export-select').should('contain.text', dumpType); + cy.get('.cvat-modal-export-task').contains('button', 'OK').click(); cy.wait('@dumpAnnotations', { timeout: 5000 }).its('response.statusCode').should('equal', 202); cy.wait('@dumpAnnotations').its('response.statusCode').should('equal', 201); cy.removeAnnotations(); diff --git a/tests/cypress/integration/actions_tasks3/case_47_export_dataset.js b/tests/cypress/integration/actions_tasks3/case_47_export_dataset.js index d23c818a19c..6de7a7cd3cb 100644 --- a/tests/cypress/integration/actions_tasks3/case_47_export_dataset.js +++ b/tests/cypress/integration/actions_tasks3/case_47_export_dataset.js @@ -18,7 +18,6 @@ context('Export task dataset.', () => { secondX: 500, secondY: 200, }; - let datasetArchiveName = ''; before(() => { cy.openTaskJob(taskName); @@ -30,19 +29,15 @@ context('Export task dataset.', () => { it(`Go to Menu. Press "Export task dataset" with the "${exportFormat}" format.`, () => { cy.intercept('GET', '/api/v1/tasks/**/dataset**').as('exportDataset'); cy.interactMenu('Export task dataset'); - cy.get('.cvat-modal-export-task-dataset').within(() => { + cy.get('.cvat-modal-export-task').within(() => { cy.get('.cvat-modal-export-select').should('contain.text', exportFormat); cy.get('[type="checkbox"]').should('not.be.checked').check(); cy.contains('button', 'OK').click(); }); + cy.get('.cvat-notification-notice-export-task-start').should('exist'); + cy.closeNotification('.cvat-notification-notice-export-task-start'); cy.wait('@exportDataset', { timeout: 5000 }).its('response.statusCode').should('equal', 202); cy.wait('@exportDataset').its('response.statusCode').should('equal', 201); - cy.task('listFiles', 'cypress/fixtures').each((fileName) => { - if (fileName.includes(exportFormat.toLowerCase())) { - datasetArchiveName = fileName; - cy.readZipArchive(datasetArchiveName) - } - }); }); }); }); From cd67f5b641abf91a1dc3d869bc253e62a20e0cca Mon Sep 17 00:00:00 2001 From: dvkruchinin Date: Thu, 29 Jul 2021 16:54:46 +0300 Subject: [PATCH 39/63] Additional tests adaptations. --- ...mp_upload_annotation_point_cloud_format.js | 15 ++++---- ...pload_annotation_velodyne_points_format.js | 12 ++++--- ...3_canvas3d_functionality_export_dataset.js | 31 ++++++++-------- .../issue_1568_cuboid_dump_annotation.js | 35 ++++++++++--------- 4 files changed, 52 insertions(+), 41 deletions(-) diff --git a/tests/cypress/integration/canvas3d_functionality/case_91_canvas3d_functionality_dump_upload_annotation_point_cloud_format.js b/tests/cypress/integration/canvas3d_functionality/case_91_canvas3d_functionality_dump_upload_annotation_point_cloud_format.js index b68e1c87ac4..a87a6a2cced 100644 --- a/tests/cypress/integration/canvas3d_functionality/case_91_canvas3d_functionality_dump_upload_annotation_point_cloud_format.js +++ b/tests/cypress/integration/canvas3d_functionality/case_91_canvas3d_functionality_dump_upload_annotation_point_cloud_format.js @@ -51,13 +51,14 @@ context('Canvas 3D functionality. Dump/upload annotation. "Point Cloud" format', it('Save a job. Dump with "Point Cloud" format.', () => { cy.saveJob('PATCH', 200, 'saveJob'); cy.intercept('GET', '/api/v1/tasks/**/annotations**').as('dumpAnnotations'); - cy.interactMenu('Dump annotations'); - cy.get('.cvat-menu-dump-submenu-item').then((subMenu) => { - expect(subMenu.length).to.be.equal(2); - }); - cy.get('.cvat-menu-dump-submenu-item').within(() => { - cy.contains(dumpTypePC).click(); - }); + cy.interactMenu('Export task dataset'); + cy.get('.cvat-modal-export-task').find('.cvat-modal-export-select').click(); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .contains('.cvat-modal-export-option-item', dumpTypePC) + .click(); + cy.get('.cvat-modal-export-select').should('contain.text', dumpTypePC); + cy.get('.cvat-modal-export-task').contains('button', 'OK').click(); cy.wait('@dumpAnnotations', { timeout: 5000 }).its('response.statusCode').should('equal', 202); cy.wait('@dumpAnnotations').its('response.statusCode').should('equal', 201); cy.removeAnnotations(); diff --git a/tests/cypress/integration/canvas3d_functionality/case_92_canvas3d_functionality_dump_upload_annotation_velodyne_points_format.js b/tests/cypress/integration/canvas3d_functionality/case_92_canvas3d_functionality_dump_upload_annotation_velodyne_points_format.js index d8b9676a370..e2ef8c41e35 100644 --- a/tests/cypress/integration/canvas3d_functionality/case_92_canvas3d_functionality_dump_upload_annotation_velodyne_points_format.js +++ b/tests/cypress/integration/canvas3d_functionality/case_92_canvas3d_functionality_dump_upload_annotation_velodyne_points_format.js @@ -51,10 +51,14 @@ context('Canvas 3D functionality. Dump/upload annotation. "Velodyne Points" form it('Save a job. Dump with "Velodyne Points" format.', () => { cy.saveJob('PATCH', 200, 'saveJob'); cy.intercept('GET', '/api/v1/tasks/**/annotations**').as('dumpAnnotations'); - cy.interactMenu('Dump annotations'); - cy.get('.cvat-menu-dump-submenu-item').within(() => { - cy.contains(dumpTypeVC).click(); - }); + cy.interactMenu('Export task dataset'); + cy.get('.cvat-modal-export-task').find('.cvat-modal-export-select').click(); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .contains('.cvat-modal-export-option-item', dumpTypeVC) + .click(); + cy.get('.cvat-modal-export-select').should('contain.text', dumpTypeVC); + cy.get('.cvat-modal-export-task').contains('button', 'OK').click(); cy.wait('@dumpAnnotations', { timeout: 5000 }).its('response.statusCode').should('equal', 202); cy.wait('@dumpAnnotations').its('response.statusCode').should('equal', 201); cy.removeAnnotations(); diff --git a/tests/cypress/integration/canvas3d_functionality/case_93_canvas3d_functionality_export_dataset.js b/tests/cypress/integration/canvas3d_functionality/case_93_canvas3d_functionality_export_dataset.js index 04e1f63817f..1625feef8d8 100644 --- a/tests/cypress/integration/canvas3d_functionality/case_93_canvas3d_functionality_export_dataset.js +++ b/tests/cypress/integration/canvas3d_functionality/case_93_canvas3d_functionality_export_dataset.js @@ -15,6 +15,21 @@ context('Canvas 3D functionality. Export as a dataset.', () => { const dumpTypePC = 'Sly Point Cloud Format'; const dumpTypeVC = 'Kitti Raw Format'; + function exportDataset (format, as) { + cy.intercept('GET', '/api/v1/tasks/**/dataset**').as(as); + cy.interactMenu('Export task dataset'); + cy.get('.cvat-modal-export-task').find('.cvat-modal-export-select').click(); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .contains('.cvat-modal-export-option-item', format) + .click(); + cy.get('.cvat-modal-export-select').should('contain.text', format); + cy.get('.cvat-modal-export-task').find('[type="checkbox"]').should('not.be.checked').check(); + cy.get('.cvat-modal-export-task').contains('button', 'OK').click(); + cy.wait(`@${as}`, { timeout: 5000 }).its('response.statusCode').should('equal', 202); + cy.wait(`@${as}`).its('response.statusCode').should('equal', 201); + } + before(() => { cy.openTask(taskName); cy.openJob(); @@ -25,23 +40,11 @@ context('Canvas 3D functionality. Export as a dataset.', () => { describe(`Testing case "${caseId}"`, () => { it('Export as a dataset with "Point Cloud" format.', () => { - cy.intercept('GET', '/api/v1/tasks/**/dataset**').as('exportDatasetPC'); - cy.interactMenu('Export as a dataset'); - cy.get('.cvat-menu-export-submenu-item').within(() => { - cy.contains(dumpTypePC).click(); - }); - cy.wait('@exportDatasetPC', { timeout: 5000 }).its('response.statusCode').should('equal', 202); - cy.wait('@exportDatasetPC').its('response.statusCode').should('equal', 201); + exportDataset(dumpTypePC, 'exportDatasetPC'); }); it('Export as a dataset with "Velodyne Points" format.', () => { - cy.intercept('GET', '/api/v1/tasks/**/dataset**').as('exportDatasetVC'); - cy.interactMenu('Export as a dataset'); - cy.get('.cvat-menu-export-submenu-item').within(() => { - cy.contains(dumpTypeVC).click(); - }); - cy.wait('@exportDatasetVC', { timeout: 5000 }).its('response.statusCode').should('equal', 202); - cy.wait('@exportDatasetVC').its('response.statusCode').should('equal', 201); + exportDataset(dumpTypeVC, 'exportDatasetVC'); cy.removeAnnotations(); cy.saveJob('PUT'); }); diff --git a/tests/cypress/integration/issues_prs2/issue_1568_cuboid_dump_annotation.js b/tests/cypress/integration/issues_prs2/issue_1568_cuboid_dump_annotation.js index bfc573b6b88..5565d906888 100644 --- a/tests/cypress/integration/issues_prs2/issue_1568_cuboid_dump_annotation.js +++ b/tests/cypress/integration/issues_prs2/issue_1568_cuboid_dump_annotation.js @@ -6,7 +6,7 @@ import { taskName, labelName } from '../../support/const'; -context('Dump annotation if cuboid created', () => { +context('Dump annotation if cuboid created.', () => { const issueId = '1568'; const createCuboidShape2Points = { points: 'From rectangle', @@ -17,30 +17,33 @@ context('Dump annotation if cuboid created', () => { secondX: 350, secondY: 450, }; + const dumpType = 'Datumaro'; before(() => { cy.openTaskJob(taskName); }); describe(`Testing issue "${issueId}"`, () => { - it('Create a cuboid', () => { + it('Create a cuboid.', () => { cy.createCuboid(createCuboidShape2Points); - cy.get('#cvat-objects-sidebar-state-item-1').should('contain', '1').and('contain', 'CUBOID SHAPE'); }); - it('Dump an annotation', () => { - cy.get('.cvat-annotation-header-left-group').within(() => { - cy.saveJob(); - cy.get('button').contains('Menu').trigger('mouseover', { force: true }); - }); - cy.get('.cvat-annotation-menu').within(() => { - cy.get('[title="Dump annotations"]').trigger('mouseover'); - }); - cy.get('.cvat-menu-dump-submenu-item').within(() => { - cy.contains('Datumaro').click(); - }); + + it('Dump an annotation.', () => { + cy.saveJob('PATCH', 200, `dump${dumpType}Format`); + cy.intercept('GET', '/api/v1/tasks/**/annotations**').as('dumpAnnotations'); + cy.interactMenu('Export task dataset'); + cy.get('.cvat-modal-export-task').find('.cvat-modal-export-select').click(); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .contains('.cvat-modal-export-option-item', dumpType) + .click(); + cy.get('.cvat-modal-export-select').should('contain.text', dumpType); + cy.get('.cvat-modal-export-task').contains('button', 'OK').click(); + cy.wait('@dumpAnnotations', { timeout: 5000 }).its('response.statusCode').should('equal', 202); + cy.wait('@dumpAnnotations').its('response.statusCode').should('equal', 201); }); - it('Error notification is not exists', () => { - cy.wait(5000); + + it('Error notification is not exists.', () => { cy.get('.ant-notification-notice').should('not.exist'); }); }); From c9559955e2b84e307dd5f4cf633ac463dcc8291f Mon Sep 17 00:00:00 2001 From: dvkruchinin Date: Thu, 29 Jul 2021 19:15:35 +0300 Subject: [PATCH 40/63] Some rework main.yaml --- .github/workflows/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 877a095f59e..9374a9758a9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,8 +5,7 @@ on: - 'master' - 'develop' pull_request: - branches: - - '*' + jobs: Unit_testing: runs-on: ubuntu-latest From 7e9fcaa44a09f8cf256bb6c2d0a4319f0c897b7f Mon Sep 17 00:00:00 2001 From: dvkruchinin Date: Fri, 30 Jul 2021 00:42:33 +0300 Subject: [PATCH 41/63] Added debug --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9374a9758a9..d4dbf068e71 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -174,7 +174,7 @@ jobs: if [ ${{ matrix.specs }} == 'canvas3d_functionality' ] || [ ${{ matrix.specs }} == 'canvas3d_functionality_2' ]; then npx cypress run --browser chrome --env coverage=false --config-file cypress_canvas3d.json --spec 'cypress/integration/${{ matrix.specs }}/**/*.js' else - npx cypress run --headless --browser chrome --env coverage=false --spec 'cypress/integration/${{ matrix.specs }}/**/*.js' + DEBUG=cypress:* npx cypress run --headless --browser chrome --env coverage=false --spec 'cypress/integration/${{ matrix.specs }}/**/*.js' fi fi - name: Creating a log file from "cvat" container logs From 4be054089d838d2080af83be8002af3e3e7eed3d Mon Sep 17 00:00:00 2001 From: dvkruchinin Date: Fri, 30 Jul 2021 01:00:09 +0300 Subject: [PATCH 42/63] Debug removed --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d4dbf068e71..9374a9758a9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -174,7 +174,7 @@ jobs: if [ ${{ matrix.specs }} == 'canvas3d_functionality' ] || [ ${{ matrix.specs }} == 'canvas3d_functionality_2' ]; then npx cypress run --browser chrome --env coverage=false --config-file cypress_canvas3d.json --spec 'cypress/integration/${{ matrix.specs }}/**/*.js' else - DEBUG=cypress:* npx cypress run --headless --browser chrome --env coverage=false --spec 'cypress/integration/${{ matrix.specs }}/**/*.js' + npx cypress run --headless --browser chrome --env coverage=false --spec 'cypress/integration/${{ matrix.specs }}/**/*.js' fi fi - name: Creating a log file from "cvat" container logs From 415e2751ff6380aaa55f0625eb47485f7baa2517 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Fri, 30 Jul 2021 12:47:00 +0300 Subject: [PATCH 43/63] Fixed late binding problem --- cvat/apps/dataset_manager/bindings.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 494a9874b61..a322de8f458 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -955,19 +955,23 @@ def __init__(self, project_data: ProjectData, include_images=False): if is_video: # optimization for videos: use numpy arrays instead of bytes # some formats or transforms can require image data - def _make_image(i, **kwargs): - loader = lambda _: frame_provider.get_frame(i, - quality=frame_provider.Quality.ORIGINAL, - out_type=frame_provider.Type.NUMPY_ARRAY)[0] - return Image(loader=loader, **kwargs) + def image_maker_factory(frame_provider): + def _make_image(i, **kwargs): + loader = lambda _: frame_provider.get_frame(i, + quality=frame_provider.Quality.ORIGINAL, + out_type=frame_provider.Type.NUMPY_ARRAY)[0] + return Image(loader=loader, **kwargs) + return _make_image else: # for images use encoded data to avoid recoding - def _make_image(i, **kwargs): - loader = lambda _: frame_provider.get_frame(i, - quality=frame_provider.Quality.ORIGINAL, - out_type=frame_provider.Type.BUFFER)[0].getvalue() - return ByteImage(data=loader, **kwargs) - image_maker_per_task[task.id] = _make_image + def image_maker_factory(frame_provider): + def _make_image(i, **kwargs): + loader = lambda _: frame_provider.get_frame(i, + quality=frame_provider.Quality.ORIGINAL, + out_type=frame_provider.Type.BUFFER)[0].getvalue() + return ByteImage(data=loader, **kwargs) + return _make_image + image_maker_per_task[task.id] = image_maker_factory(frame_provider) for frame_data in project_data.group_by_frame(include_empty=True): image_args = { From e6c95a27deee25f752ecf4ea2892a4fcc66491c9 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Fri, 30 Jul 2021 12:48:43 +0300 Subject: [PATCH 44/63] Fixed 3d default format --- .../src/components/export-dataset/export-dataset-modal.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx index 49f09578f5a..349ae94becd 100644 --- a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -45,9 +45,9 @@ function ExportDatasetModal(): JSX.Element { } else if (instance instanceof core.classes.Task) { setInstanceType('task'); setActivities(taskExportActivities[instance.id] || []); - if (instance.mode === 'interpolation') { + if (instance.mode === 'interpolation' && instance.dimension === '2d') { form.setFieldsValue({ selectedFormat: 'CVAT for video 1.1' }); - } else if (instance.mode === 'annotation') { + } else if (instance.mode === 'annotation' && instance.dimension === '2d') { form.setFieldsValue({ selectedFormat: 'CVAT for images 1.1' }); } } From 0687808750a0a076ab2bad030ac36a4e2445b2a8 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Mon, 2 Aug 2021 16:44:30 +0300 Subject: [PATCH 45/63] Added support for project with 3d tasks --- cvat/apps/dataset_manager/bindings.py | 115 ++++++++++-------- .../dataset_manager/formats/pointcloud.py | 7 +- .../dataset_manager/formats/velodynepoint.py | 7 +- 3 files changed, 71 insertions(+), 58 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index a322de8f458..581ada96ca9 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -513,7 +513,7 @@ class ProjectData(InstanceLabelData): Track = NamedTuple('Track', [('label', str), ('group', int), ('source', str), ('shapes', List[TrackedShape]), ('task_id', int)]) Tag = NamedTuple('Tag', [('frame', int), ('label', str), ('attributes', List[InstanceLabelData.Attribute]), ('source', str), ('group', int), ('task_id', int)]) Tag.__new__.__defaults__ = (0, ) - Frame = NamedTuple('Frame', [('task_id', int), ('subset', str), ('idx', int), ('frame', int), ('name', str), ('width', int), ('height', int), ('labeled_shapes', List[Union[LabeledShape, TrackedShape]]), ('tags', List[Tag])]) + Frame = NamedTuple('Frame', [('task_id', int), ('subset', str), ('idx', int), ('id', int), ('frame', int), ('name', str), ('width', int), ('height', int), ('labeled_shapes', List[Union[LabeledShape, TrackedShape]]), ('tags', List[Tag])]) def __init__(self, annotation_irs: Mapping[str, AnnotationIR], db_project: Project, host: str, create_callback: Callable = None): self._annotation_irs = annotation_irs @@ -579,6 +579,7 @@ def _init_frame_info(self): } for frame in range(task.data.size)}) else: self._frame_info.update({(task.id, self.rel_frame_id(task.id, db_image.frame)): { + "id": db_image.id, "path": mangle_image_name(db_image.path, task.subset, original_names), "width": db_image.width, "height": db_image.height, @@ -682,6 +683,7 @@ def get_frame(task_id: int, idx: int) -> ProjectData.Frame: task_id=task_id, subset=frame_info["subset"], idx=idx, + id=frame_info.get('id',0), frame=abs_frame, name=frame_info["path"], height=frame_info["height"], @@ -795,20 +797,25 @@ def __init__(self): self._categories: Dict[datumaro.AnnotationType, datumaro.LabelCategories] = {} @staticmethod - def _load_categories(labels: list): + def _load_categories(meta: dict, dimension): categories: Dict[datumaro.AnnotationType, datumaro.Categories] = {} label_categories = datumaro.LabelCategories(attributes=['occluded']) - for _, label in labels: + user_info = {} + if dimension == DimensionType.DIM_3D: + user_info = {"name": meta['owner']['username'], + "createdAt": meta['created'], + "updatedAt": meta['updated']} + for _, label in meta['labels']: label_categories.add(label['name']) for _, attr in label['attributes']: label_categories.attributes.add(attr['name']) - categories[datumaro.AnnotationType.label] = label_categories - return categories + categories[datumaro.AnnotationType.label] = label_categories + return categories, user_info def _read_cvat_anno(self, cvat_frame_anno: Union[ProjectData.Frame, TaskData.Frame], labels: list): categories = self.categories() @@ -823,9 +830,9 @@ def map_label(name): return label_cat.find(name)[0] class CvatTaskDataExtractor(CVATDataExtractor): - def __init__(self, task_data, include_images=False, format_type=None, dimension=DimensionType.DIM_2D): + def __init__(self, task_data, include_images, format_type, dimension): super().__init__() - self._categories, self._user = self._load_categories(task_data, dimension=dimension) + self._categories, self._user = self._load_categories(task_data.meta['task'], dimension=dimension) self._dimension = dimension self._format_type = format_type dm_items = [] @@ -891,11 +898,9 @@ def _make_image(i, **kwargs): attributes["createdAt"] = self._user["createdAt"] attributes["updatedAt"] = self._user["updatedAt"] attributes["labels"] = [] - index = 0 - for _, label in task_data.meta['task']['labels']: - attributes["labels"].append({"label_id": index, "name": label["name"], "color": label["color"]}) + for (idx, (_, label)) in enumerate(task_data.meta['task']['labels']): + attributes["labels"].append({"label_id": idx, "name": label["name"], "color": label["color"]}) attributes["track_id"] = -1 - index += 1 dm_item = datumaro.DatasetItem(id=osp.split(frame_data.name)[-1].split('.')[0], annotations=dm_anno, point_cloud=dm_image[0], related_images=dm_image[1], @@ -905,27 +910,6 @@ def _make_image(i, **kwargs): self._items = dm_items - @staticmethod - def _load_categories(cvat_anno, dimension): # pylint: disable=arguments-differ - categories = {} - - label_categories = datumaro.LabelCategories(attributes=['occluded']) - - user_info = {} - if dimension == DimensionType.DIM_3D: - user_info = {"name": cvat_anno.meta['task']['owner']['username'], - "createdAt": cvat_anno.meta['task']['created'], - "updatedAt": cvat_anno.meta['task']['updated']} - for _, label in cvat_anno.meta['task']['labels']: - label_categories.add(label['name']) - for _, attr in label['attributes']: - label_categories.attributes.add(attr['name']) - - - categories[datumaro.AnnotationType.label] = label_categories - - return categories, user_info - def _read_cvat_anno(self, cvat_frame_anno: TaskData.Frame, labels: list): categories = self.categories() label_cat = categories[datumaro.AnnotationType.label] @@ -938,9 +922,11 @@ def map_label(name): return label_cat.find(name)[0] return convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label, self._format_type, self._dimension) class CVATProjectDataExtractor(CVATDataExtractor): - def __init__(self, project_data: ProjectData, include_images=False): + def __init__(self, project_data: ProjectData, include_images: bool, format_type: str, dimension: DimensionType): super().__init__() - self._categories = self._load_categories(project_data.meta['project']['labels']) + self._categories, self._user = self._load_categories(project_data.meta['project'], dimension) + self._dimension = dimension + self._format_type = format_type dm_items: List[datumaro.DatasetItem] = [] @@ -950,12 +936,27 @@ def __init__(self, project_data: ProjectData, include_images=False): for task in project_data.tasks: is_video = task.mode == 'interpolation' ext_per_task[task.id] = FrameProvider.VIDEO_FRAME_EXT if is_video else '' - if include_images: - frame_provider = FrameProvider(task.data) + if self._dimension == DimensionType.DIM_3D: + def image_maker_factory(task): + def _make_image(i, **kwargs): + loader = osp.join( + task.data.get_upload_dirname(), kwargs['path'], + ) + related_images = [] + image = Img.objects.get(id=i) + for i in image.related_files.all(): + path = osp.realpath(str(i.path)) + if osp.isfile(path): + related_images.append(path) + return loader, related_images + return _make_image + image_maker_per_task[task.id] = image_maker_factory(task) + elif include_images: if is_video: # optimization for videos: use numpy arrays instead of bytes # some formats or transforms can require image data - def image_maker_factory(frame_provider): + def image_maker_factory(task): + frame_provider = FrameProvider(task.data) def _make_image(i, **kwargs): loader = lambda _: frame_provider.get_frame(i, quality=frame_provider.Quality.ORIGINAL, @@ -964,40 +965,58 @@ def _make_image(i, **kwargs): return _make_image else: # for images use encoded data to avoid recoding - def image_maker_factory(frame_provider): + def image_maker_factory(task): + frame_provider = FrameProvider(task.data) def _make_image(i, **kwargs): loader = lambda _: frame_provider.get_frame(i, quality=frame_provider.Quality.ORIGINAL, out_type=frame_provider.Type.BUFFER)[0].getvalue() return ByteImage(data=loader, **kwargs) return _make_image - image_maker_per_task[task.id] = image_maker_factory(frame_provider) + image_maker_per_task[task.id] = image_maker_factory(task) for frame_data in project_data.group_by_frame(include_empty=True): image_args = { 'path': frame_data.name + ext_per_task[frame_data.task_id], 'size': (frame_data.height, frame_data.width), } - if include_images: + if self._dimension == DimensionType.DIM_3D: + dm_image = image_maker_per_task[frame_data.task_id](frame_data.id, **image_args) + elif include_images: dm_image = image_maker_per_task[frame_data.task_id](frame_data.idx, **image_args) else: dm_image = Image(**image_args) dm_anno = self._read_cvat_anno(frame_data, project_data.meta['project']['labels']) - dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0], - annotations=dm_anno, image=dm_image, - subset=frame_data.subset, - attributes={'frame': frame_data.frame} - ) + if self._dimension == DimensionType.DIM_2D: + dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0], + annotations=dm_anno, image=dm_image, + subset=frame_data.subset, + attributes={'frame': frame_data.frame} + ) + else: + attributes = {'frame': frame_data.frame} + if format_type == "sly_pointcloud": + attributes["name"] = self._user["name"] + attributes["createdAt"] = self._user["createdAt"] + attributes["updatedAt"] = self._user["updatedAt"] + attributes["labels"] = [] + for (idx, (_, label)) in enumerate(project_data.meta['project']['labels']): + attributes["labels"].append({"label_id": idx, "name": label["name"], "color": label["color"]}) + attributes["track_id"] = -1 + + dm_item = datumaro.DatasetItem(id=osp.split(frame_data.name)[-1].split('.')[0], + annotations=dm_anno, point_cloud=dm_image[0], related_images=dm_image[1], + attributes=attributes) dm_items.append(dm_item) self._items = dm_items -def GetCVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_images: bool=False): +def GetCVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_images: bool = False, format_type: str = None, dimension: DimensionType = DimensionType.DIM_2D): if isinstance(instance_data, ProjectData): - return CVATProjectDataExtractor(instance_data, include_images) + return CVATProjectDataExtractor(instance_data, include_images, format_type, dimension) else: - return CvatTaskDataExtractor(instance_data, include_images) + return CvatTaskDataExtractor(instance_data, include_images, format_type, dimension) class CvatImportError(Exception): pass diff --git a/cvat/apps/dataset_manager/formats/pointcloud.py b/cvat/apps/dataset_manager/formats/pointcloud.py index 458029a132d..0009cd2f5c8 100644 --- a/cvat/apps/dataset_manager/formats/pointcloud.py +++ b/cvat/apps/dataset_manager/formats/pointcloud.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, TaskData, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive from cvat.apps.engine.models import DimensionType @@ -18,10 +18,7 @@ @exporter(name='Sly Point Cloud Format', ext='ZIP', version='1.0', dimension=DimensionType.DIM_3D) def _export_images(dst_file, task_data, save_images=False): - if not isinstance(task_data, TaskData): - raise Exception("Export to \"Sly Point Cloud\" format is working only with tasks temporarily") - - dataset = Dataset.from_extractors(CvatTaskDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( task_data, include_images=save_images, format_type='sly_pointcloud', dimension=DimensionType.DIM_3D), env=dm_env) with TemporaryDirectory() as temp_dir: diff --git a/cvat/apps/dataset_manager/formats/velodynepoint.py b/cvat/apps/dataset_manager/formats/velodynepoint.py index 7384f7beabe..747c47513d8 100644 --- a/cvat/apps/dataset_manager/formats/velodynepoint.py +++ b/cvat/apps/dataset_manager/formats/velodynepoint.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor, TaskData, \ +from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, \ import_dm_annotations from .registry import dm_env @@ -20,10 +20,7 @@ @exporter(name='Kitti Raw Format', ext='ZIP', version='1.0', dimension=DimensionType.DIM_3D) def _export_images(dst_file, task_data, save_images=False): - if not isinstance(task_data, TaskData): - raise Exception("Export to \"Kitti raw\" format is working only with tasks temporarily") - - dataset = Dataset.from_extractors(CvatTaskDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( task_data, include_images=save_images, format_type="kitti_raw", dimension=DimensionType.DIM_3D), env=dm_env) with TemporaryDirectory() as temp_dir: From 7e412f4ccb689c11cd0da2fb632c4968fefc5f88 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Mon, 2 Aug 2021 17:40:45 +0300 Subject: [PATCH 46/63] Fixed test --- cvat/apps/dataset_manager/bindings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 581ada96ca9..2b57ee30ad7 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -830,7 +830,7 @@ def map_label(name): return label_cat.find(name)[0] class CvatTaskDataExtractor(CVATDataExtractor): - def __init__(self, task_data, include_images, format_type, dimension): + def __init__(self, task_data, include_images = False, format_type = None, dimension = DimensionType.DIM_2D): super().__init__() self._categories, self._user = self._load_categories(task_data.meta['task'], dimension=dimension) self._dimension = dimension @@ -922,7 +922,7 @@ def map_label(name): return label_cat.find(name)[0] return convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label, self._format_type, self._dimension) class CVATProjectDataExtractor(CVATDataExtractor): - def __init__(self, project_data: ProjectData, include_images: bool, format_type: str, dimension: DimensionType): + def __init__(self, project_data: ProjectData, include_images: bool = False, format_type: str = None, dimension: DimensionType = DimensionType.DIM_2D): super().__init__() self._categories, self._user = self._load_categories(project_data.meta['project'], dimension) self._dimension = dimension From e635db3614f70467991d4ea5624e06df519ea3a6 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 3 Aug 2021 12:40:51 +0300 Subject: [PATCH 47/63] Update cvat/apps/dataset_manager/views.py Co-authored-by: Nikita Manovich --- cvat/apps/dataset_manager/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/dataset_manager/views.py b/cvat/apps/dataset_manager/views.py index 4541e7c56d6..5e95e1c96b0 100644 --- a/cvat/apps/dataset_manager/views.py +++ b/cvat/apps/dataset_manager/views.py @@ -103,7 +103,7 @@ def export_project_as_dataset(project_id, dst_format=None, server_url=None): return export(dst_format, project_id=project_id, server_url=server_url, save_images=True) -def export_project_annotation(project_id, dst_format=None, server_url=None): +def export_project_annotations(project_id, dst_format=None, server_url=None): return export(dst_format, project_id=project_id, server_url=server_url, save_images=False) def clear_export_cache(task_id, file_path, file_ctime): From e9f1fc152f5c5a3ee2f814993ef98ad842e59a63 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 3 Aug 2021 12:42:17 +0300 Subject: [PATCH 48/63] Fixed function name --- cvat/apps/engine/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 85ade1734eb..bb4dbed7235 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -361,7 +361,7 @@ def annotations(self, request, pk): rq_id="/api/v1/projects/{}/annotations/{}".format(pk, format_name), request=request, action=request.query_params.get("action", "").lower(), - callback=dm.views.export_project_annotation, + callback=dm.views.export_project_annotations, format_name=format_name, filename=request.query_params.get("filename", "").lower(), ) From 0a06baffc4e9d24e0ecf37f1d22675ffd715c447 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 3 Aug 2021 12:57:47 +0300 Subject: [PATCH 49/63] Revert "Added support for project with 3d tasks" This reverts commit 0687808750a0a076ab2bad030ac36a4e2445b2a8. --- cvat/apps/dataset_manager/bindings.py | 115 ++++++++---------- .../dataset_manager/formats/pointcloud.py | 7 +- .../dataset_manager/formats/velodynepoint.py | 7 +- 3 files changed, 58 insertions(+), 71 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 2b57ee30ad7..6d0c8badf51 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -513,7 +513,7 @@ class ProjectData(InstanceLabelData): Track = NamedTuple('Track', [('label', str), ('group', int), ('source', str), ('shapes', List[TrackedShape]), ('task_id', int)]) Tag = NamedTuple('Tag', [('frame', int), ('label', str), ('attributes', List[InstanceLabelData.Attribute]), ('source', str), ('group', int), ('task_id', int)]) Tag.__new__.__defaults__ = (0, ) - Frame = NamedTuple('Frame', [('task_id', int), ('subset', str), ('idx', int), ('id', int), ('frame', int), ('name', str), ('width', int), ('height', int), ('labeled_shapes', List[Union[LabeledShape, TrackedShape]]), ('tags', List[Tag])]) + Frame = NamedTuple('Frame', [('task_id', int), ('subset', str), ('idx', int), ('frame', int), ('name', str), ('width', int), ('height', int), ('labeled_shapes', List[Union[LabeledShape, TrackedShape]]), ('tags', List[Tag])]) def __init__(self, annotation_irs: Mapping[str, AnnotationIR], db_project: Project, host: str, create_callback: Callable = None): self._annotation_irs = annotation_irs @@ -579,7 +579,6 @@ def _init_frame_info(self): } for frame in range(task.data.size)}) else: self._frame_info.update({(task.id, self.rel_frame_id(task.id, db_image.frame)): { - "id": db_image.id, "path": mangle_image_name(db_image.path, task.subset, original_names), "width": db_image.width, "height": db_image.height, @@ -683,7 +682,6 @@ def get_frame(task_id: int, idx: int) -> ProjectData.Frame: task_id=task_id, subset=frame_info["subset"], idx=idx, - id=frame_info.get('id',0), frame=abs_frame, name=frame_info["path"], height=frame_info["height"], @@ -797,25 +795,20 @@ def __init__(self): self._categories: Dict[datumaro.AnnotationType, datumaro.LabelCategories] = {} @staticmethod - def _load_categories(meta: dict, dimension): + def _load_categories(labels: list): categories: Dict[datumaro.AnnotationType, datumaro.Categories] = {} label_categories = datumaro.LabelCategories(attributes=['occluded']) - user_info = {} - if dimension == DimensionType.DIM_3D: - user_info = {"name": meta['owner']['username'], - "createdAt": meta['created'], - "updatedAt": meta['updated']} - for _, label in meta['labels']: + for _, label in labels: label_categories.add(label['name']) for _, attr in label['attributes']: label_categories.attributes.add(attr['name']) - categories[datumaro.AnnotationType.label] = label_categories - return categories, user_info + return categories + def _read_cvat_anno(self, cvat_frame_anno: Union[ProjectData.Frame, TaskData.Frame], labels: list): categories = self.categories() @@ -830,9 +823,9 @@ def map_label(name): return label_cat.find(name)[0] class CvatTaskDataExtractor(CVATDataExtractor): - def __init__(self, task_data, include_images = False, format_type = None, dimension = DimensionType.DIM_2D): + def __init__(self, task_data, include_images=False, format_type=None, dimension=DimensionType.DIM_2D): super().__init__() - self._categories, self._user = self._load_categories(task_data.meta['task'], dimension=dimension) + self._categories, self._user = self._load_categories(task_data, dimension=dimension) self._dimension = dimension self._format_type = format_type dm_items = [] @@ -898,9 +891,11 @@ def _make_image(i, **kwargs): attributes["createdAt"] = self._user["createdAt"] attributes["updatedAt"] = self._user["updatedAt"] attributes["labels"] = [] - for (idx, (_, label)) in enumerate(task_data.meta['task']['labels']): - attributes["labels"].append({"label_id": idx, "name": label["name"], "color": label["color"]}) + index = 0 + for _, label in task_data.meta['task']['labels']: + attributes["labels"].append({"label_id": index, "name": label["name"], "color": label["color"]}) attributes["track_id"] = -1 + index += 1 dm_item = datumaro.DatasetItem(id=osp.split(frame_data.name)[-1].split('.')[0], annotations=dm_anno, point_cloud=dm_image[0], related_images=dm_image[1], @@ -910,6 +905,27 @@ def _make_image(i, **kwargs): self._items = dm_items + @staticmethod + def _load_categories(cvat_anno, dimension): # pylint: disable=arguments-differ + categories = {} + + label_categories = datumaro.LabelCategories(attributes=['occluded']) + + user_info = {} + if dimension == DimensionType.DIM_3D: + user_info = {"name": cvat_anno.meta['task']['owner']['username'], + "createdAt": cvat_anno.meta['task']['created'], + "updatedAt": cvat_anno.meta['task']['updated']} + for _, label in cvat_anno.meta['task']['labels']: + label_categories.add(label['name']) + for _, attr in label['attributes']: + label_categories.attributes.add(attr['name']) + + + categories[datumaro.AnnotationType.label] = label_categories + + return categories, user_info + def _read_cvat_anno(self, cvat_frame_anno: TaskData.Frame, labels: list): categories = self.categories() label_cat = categories[datumaro.AnnotationType.label] @@ -922,11 +938,9 @@ def map_label(name): return label_cat.find(name)[0] return convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label, self._format_type, self._dimension) class CVATProjectDataExtractor(CVATDataExtractor): - def __init__(self, project_data: ProjectData, include_images: bool = False, format_type: str = None, dimension: DimensionType = DimensionType.DIM_2D): + def __init__(self, project_data: ProjectData, include_images: bool = False): super().__init__() - self._categories, self._user = self._load_categories(project_data.meta['project'], dimension) - self._dimension = dimension - self._format_type = format_type + self._categories = self._load_categories(project_data.meta['project']['labels']) dm_items: List[datumaro.DatasetItem] = [] @@ -936,27 +950,12 @@ def __init__(self, project_data: ProjectData, include_images: bool = False, form for task in project_data.tasks: is_video = task.mode == 'interpolation' ext_per_task[task.id] = FrameProvider.VIDEO_FRAME_EXT if is_video else '' - if self._dimension == DimensionType.DIM_3D: - def image_maker_factory(task): - def _make_image(i, **kwargs): - loader = osp.join( - task.data.get_upload_dirname(), kwargs['path'], - ) - related_images = [] - image = Img.objects.get(id=i) - for i in image.related_files.all(): - path = osp.realpath(str(i.path)) - if osp.isfile(path): - related_images.append(path) - return loader, related_images - return _make_image - image_maker_per_task[task.id] = image_maker_factory(task) - elif include_images: + if include_images: + frame_provider = FrameProvider(task.data) if is_video: # optimization for videos: use numpy arrays instead of bytes # some formats or transforms can require image data - def image_maker_factory(task): - frame_provider = FrameProvider(task.data) + def image_maker_factory(frame_provider): def _make_image(i, **kwargs): loader = lambda _: frame_provider.get_frame(i, quality=frame_provider.Quality.ORIGINAL, @@ -965,58 +964,40 @@ def _make_image(i, **kwargs): return _make_image else: # for images use encoded data to avoid recoding - def image_maker_factory(task): - frame_provider = FrameProvider(task.data) + def image_maker_factory(frame_provider): def _make_image(i, **kwargs): loader = lambda _: frame_provider.get_frame(i, quality=frame_provider.Quality.ORIGINAL, out_type=frame_provider.Type.BUFFER)[0].getvalue() return ByteImage(data=loader, **kwargs) return _make_image - image_maker_per_task[task.id] = image_maker_factory(task) + image_maker_per_task[task.id] = image_maker_factory(frame_provider) for frame_data in project_data.group_by_frame(include_empty=True): image_args = { 'path': frame_data.name + ext_per_task[frame_data.task_id], 'size': (frame_data.height, frame_data.width), } - if self._dimension == DimensionType.DIM_3D: - dm_image = image_maker_per_task[frame_data.task_id](frame_data.id, **image_args) - elif include_images: + if include_images: dm_image = image_maker_per_task[frame_data.task_id](frame_data.idx, **image_args) else: dm_image = Image(**image_args) dm_anno = self._read_cvat_anno(frame_data, project_data.meta['project']['labels']) - if self._dimension == DimensionType.DIM_2D: - dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0], - annotations=dm_anno, image=dm_image, - subset=frame_data.subset, - attributes={'frame': frame_data.frame} - ) - else: - attributes = {'frame': frame_data.frame} - if format_type == "sly_pointcloud": - attributes["name"] = self._user["name"] - attributes["createdAt"] = self._user["createdAt"] - attributes["updatedAt"] = self._user["updatedAt"] - attributes["labels"] = [] - for (idx, (_, label)) in enumerate(project_data.meta['project']['labels']): - attributes["labels"].append({"label_id": idx, "name": label["name"], "color": label["color"]}) - attributes["track_id"] = -1 - - dm_item = datumaro.DatasetItem(id=osp.split(frame_data.name)[-1].split('.')[0], - annotations=dm_anno, point_cloud=dm_image[0], related_images=dm_image[1], - attributes=attributes) + dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0], + annotations=dm_anno, image=dm_image, + subset=frame_data.subset, + attributes={'frame': frame_data.frame} + ) dm_items.append(dm_item) self._items = dm_items -def GetCVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_images: bool = False, format_type: str = None, dimension: DimensionType = DimensionType.DIM_2D): +def GetCVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_images: bool=False): if isinstance(instance_data, ProjectData): - return CVATProjectDataExtractor(instance_data, include_images, format_type, dimension) + return CVATProjectDataExtractor(instance_data, include_images) else: - return CvatTaskDataExtractor(instance_data, include_images, format_type, dimension) + return CvatTaskDataExtractor(instance_data, include_images) class CvatImportError(Exception): pass diff --git a/cvat/apps/dataset_manager/formats/pointcloud.py b/cvat/apps/dataset_manager/formats/pointcloud.py index 0009cd2f5c8..458029a132d 100644 --- a/cvat/apps/dataset_manager/formats/pointcloud.py +++ b/cvat/apps/dataset_manager/formats/pointcloud.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, +from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, TaskData, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive from cvat.apps.engine.models import DimensionType @@ -18,7 +18,10 @@ @exporter(name='Sly Point Cloud Format', ext='ZIP', version='1.0', dimension=DimensionType.DIM_3D) def _export_images(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(GetCVATDataExtractor( + if not isinstance(task_data, TaskData): + raise Exception("Export to \"Sly Point Cloud\" format is working only with tasks temporarily") + + dataset = Dataset.from_extractors(CvatTaskDataExtractor( task_data, include_images=save_images, format_type='sly_pointcloud', dimension=DimensionType.DIM_3D), env=dm_env) with TemporaryDirectory() as temp_dir: diff --git a/cvat/apps/dataset_manager/formats/velodynepoint.py b/cvat/apps/dataset_manager/formats/velodynepoint.py index 747c47513d8..7384f7beabe 100644 --- a/cvat/apps/dataset_manager/formats/velodynepoint.py +++ b/cvat/apps/dataset_manager/formats/velodynepoint.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, \ +from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor, TaskData, \ import_dm_annotations from .registry import dm_env @@ -20,7 +20,10 @@ @exporter(name='Kitti Raw Format', ext='ZIP', version='1.0', dimension=DimensionType.DIM_3D) def _export_images(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(GetCVATDataExtractor( + if not isinstance(task_data, TaskData): + raise Exception("Export to \"Kitti raw\" format is working only with tasks temporarily") + + dataset = Dataset.from_extractors(CvatTaskDataExtractor( task_data, include_images=save_images, format_type="kitti_raw", dimension=DimensionType.DIM_3D), env=dm_env) with TemporaryDirectory() as temp_dir: From 1387b2fd5c45d0f5dc852c3b5d3344e543a10590 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 3 Aug 2021 13:27:01 +0300 Subject: [PATCH 50/63] Changed defaulted subset --- cvat/apps/dataset_manager/bindings.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 6d0c8badf51..fc777fc1883 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -570,19 +570,20 @@ def _init_frame_info(self): self._frame_info = dict() original_names = DefaultDict[Tuple[str, str], int](int) for task in self._db_tasks.values(): + defaulted_subset = get_defaulted_subset(task.subset, self._subsets) if hasattr(task.data, 'video'): self._frame_info.update({(task.id, frame): { "path": "frame_{:06d}".format(self.abs_frame_id(task.id, frame)), "width": task.data.video.width, "height": task.data.video.height, - "subset": get_defaulted_subset(task.subset, self._subsets), + "subset": defaulted_subset, } for frame in range(task.data.size)}) else: self._frame_info.update({(task.id, self.rel_frame_id(task.id, db_image.frame)): { - "path": mangle_image_name(db_image.path, task.subset, original_names), + "path": mangle_image_name(db_image.path, defaulted_subset, original_names), "width": db_image.width, "height": db_image.height, - "subset": get_defaulted_subset(task.subset, self._subsets) + "subset": defaulted_subset } for db_image in task.data.images.all()}) self._frame_mapping = { From 23d47dc32c8772c356b71fa6d4fe246229af0967 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Mon, 2 Aug 2021 16:44:30 +0300 Subject: [PATCH 51/63] Added support for project with 3d tasks (cherry picked from commit 0687808750a0a076ab2bad030ac36a4e2445b2a8) --- cvat/apps/dataset_manager/bindings.py | 115 ++++++++++-------- .../dataset_manager/formats/pointcloud.py | 7 +- .../dataset_manager/formats/velodynepoint.py | 7 +- 3 files changed, 71 insertions(+), 58 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index fc777fc1883..20f16495cb9 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -513,7 +513,7 @@ class ProjectData(InstanceLabelData): Track = NamedTuple('Track', [('label', str), ('group', int), ('source', str), ('shapes', List[TrackedShape]), ('task_id', int)]) Tag = NamedTuple('Tag', [('frame', int), ('label', str), ('attributes', List[InstanceLabelData.Attribute]), ('source', str), ('group', int), ('task_id', int)]) Tag.__new__.__defaults__ = (0, ) - Frame = NamedTuple('Frame', [('task_id', int), ('subset', str), ('idx', int), ('frame', int), ('name', str), ('width', int), ('height', int), ('labeled_shapes', List[Union[LabeledShape, TrackedShape]]), ('tags', List[Tag])]) + Frame = NamedTuple('Frame', [('task_id', int), ('subset', str), ('idx', int), ('id', int), ('frame', int), ('name', str), ('width', int), ('height', int), ('labeled_shapes', List[Union[LabeledShape, TrackedShape]]), ('tags', List[Tag])]) def __init__(self, annotation_irs: Mapping[str, AnnotationIR], db_project: Project, host: str, create_callback: Callable = None): self._annotation_irs = annotation_irs @@ -581,6 +581,7 @@ def _init_frame_info(self): else: self._frame_info.update({(task.id, self.rel_frame_id(task.id, db_image.frame)): { "path": mangle_image_name(db_image.path, defaulted_subset, original_names), + "id": db_image.id, "width": db_image.width, "height": db_image.height, "subset": defaulted_subset @@ -683,6 +684,7 @@ def get_frame(task_id: int, idx: int) -> ProjectData.Frame: task_id=task_id, subset=frame_info["subset"], idx=idx, + id=frame_info.get('id',0), frame=abs_frame, name=frame_info["path"], height=frame_info["height"], @@ -796,20 +798,25 @@ def __init__(self): self._categories: Dict[datumaro.AnnotationType, datumaro.LabelCategories] = {} @staticmethod - def _load_categories(labels: list): + def _load_categories(meta: dict, dimension): categories: Dict[datumaro.AnnotationType, datumaro.Categories] = {} label_categories = datumaro.LabelCategories(attributes=['occluded']) - for _, label in labels: + user_info = {} + if dimension == DimensionType.DIM_3D: + user_info = {"name": meta['owner']['username'], + "createdAt": meta['created'], + "updatedAt": meta['updated']} + for _, label in meta['labels']: label_categories.add(label['name']) for _, attr in label['attributes']: label_categories.attributes.add(attr['name']) - categories[datumaro.AnnotationType.label] = label_categories - return categories + categories[datumaro.AnnotationType.label] = label_categories + return categories, user_info def _read_cvat_anno(self, cvat_frame_anno: Union[ProjectData.Frame, TaskData.Frame], labels: list): categories = self.categories() @@ -824,9 +831,9 @@ def map_label(name): return label_cat.find(name)[0] class CvatTaskDataExtractor(CVATDataExtractor): - def __init__(self, task_data, include_images=False, format_type=None, dimension=DimensionType.DIM_2D): + def __init__(self, task_data, include_images, format_type, dimension): super().__init__() - self._categories, self._user = self._load_categories(task_data, dimension=dimension) + self._categories, self._user = self._load_categories(task_data.meta['task'], dimension=dimension) self._dimension = dimension self._format_type = format_type dm_items = [] @@ -892,11 +899,9 @@ def _make_image(i, **kwargs): attributes["createdAt"] = self._user["createdAt"] attributes["updatedAt"] = self._user["updatedAt"] attributes["labels"] = [] - index = 0 - for _, label in task_data.meta['task']['labels']: - attributes["labels"].append({"label_id": index, "name": label["name"], "color": label["color"]}) + for (idx, (_, label)) in enumerate(task_data.meta['task']['labels']): + attributes["labels"].append({"label_id": idx, "name": label["name"], "color": label["color"]}) attributes["track_id"] = -1 - index += 1 dm_item = datumaro.DatasetItem(id=osp.split(frame_data.name)[-1].split('.')[0], annotations=dm_anno, point_cloud=dm_image[0], related_images=dm_image[1], @@ -906,27 +911,6 @@ def _make_image(i, **kwargs): self._items = dm_items - @staticmethod - def _load_categories(cvat_anno, dimension): # pylint: disable=arguments-differ - categories = {} - - label_categories = datumaro.LabelCategories(attributes=['occluded']) - - user_info = {} - if dimension == DimensionType.DIM_3D: - user_info = {"name": cvat_anno.meta['task']['owner']['username'], - "createdAt": cvat_anno.meta['task']['created'], - "updatedAt": cvat_anno.meta['task']['updated']} - for _, label in cvat_anno.meta['task']['labels']: - label_categories.add(label['name']) - for _, attr in label['attributes']: - label_categories.attributes.add(attr['name']) - - - categories[datumaro.AnnotationType.label] = label_categories - - return categories, user_info - def _read_cvat_anno(self, cvat_frame_anno: TaskData.Frame, labels: list): categories = self.categories() label_cat = categories[datumaro.AnnotationType.label] @@ -939,9 +923,11 @@ def map_label(name): return label_cat.find(name)[0] return convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label, self._format_type, self._dimension) class CVATProjectDataExtractor(CVATDataExtractor): - def __init__(self, project_data: ProjectData, include_images: bool = False): + def __init__(self, project_data: ProjectData, include_images: bool = False, format_type: str = None, dimension: DimensionType = DimensionType.DIM_2D): super().__init__() - self._categories = self._load_categories(project_data.meta['project']['labels']) + self._categories, self._user = self._load_categories(project_data.meta['project'], dimension) + self._dimension = dimension + self._format_type = format_type dm_items: List[datumaro.DatasetItem] = [] @@ -951,12 +937,27 @@ def __init__(self, project_data: ProjectData, include_images: bool = False): for task in project_data.tasks: is_video = task.mode == 'interpolation' ext_per_task[task.id] = FrameProvider.VIDEO_FRAME_EXT if is_video else '' - if include_images: - frame_provider = FrameProvider(task.data) + if self._dimension == DimensionType.DIM_3D: + def image_maker_factory(task): + def _make_image(i, **kwargs): + loader = osp.join( + task.data.get_upload_dirname(), kwargs['path'], + ) + related_images = [] + image = Img.objects.get(id=i) + for i in image.related_files.all(): + path = osp.realpath(str(i.path)) + if osp.isfile(path): + related_images.append(path) + return loader, related_images + return _make_image + image_maker_per_task[task.id] = image_maker_factory(task) + elif include_images: if is_video: # optimization for videos: use numpy arrays instead of bytes # some formats or transforms can require image data - def image_maker_factory(frame_provider): + def image_maker_factory(task): + frame_provider = FrameProvider(task.data) def _make_image(i, **kwargs): loader = lambda _: frame_provider.get_frame(i, quality=frame_provider.Quality.ORIGINAL, @@ -965,40 +966,58 @@ def _make_image(i, **kwargs): return _make_image else: # for images use encoded data to avoid recoding - def image_maker_factory(frame_provider): + def image_maker_factory(task): + frame_provider = FrameProvider(task.data) def _make_image(i, **kwargs): loader = lambda _: frame_provider.get_frame(i, quality=frame_provider.Quality.ORIGINAL, out_type=frame_provider.Type.BUFFER)[0].getvalue() return ByteImage(data=loader, **kwargs) return _make_image - image_maker_per_task[task.id] = image_maker_factory(frame_provider) + image_maker_per_task[task.id] = image_maker_factory(task) for frame_data in project_data.group_by_frame(include_empty=True): image_args = { 'path': frame_data.name + ext_per_task[frame_data.task_id], 'size': (frame_data.height, frame_data.width), } - if include_images: + if self._dimension == DimensionType.DIM_3D: + dm_image = image_maker_per_task[frame_data.task_id](frame_data.id, **image_args) + elif include_images: dm_image = image_maker_per_task[frame_data.task_id](frame_data.idx, **image_args) else: dm_image = Image(**image_args) dm_anno = self._read_cvat_anno(frame_data, project_data.meta['project']['labels']) - dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0], - annotations=dm_anno, image=dm_image, - subset=frame_data.subset, - attributes={'frame': frame_data.frame} - ) + if self._dimension == DimensionType.DIM_2D: + dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0], + annotations=dm_anno, image=dm_image, + subset=frame_data.subset, + attributes={'frame': frame_data.frame} + ) + else: + attributes = {'frame': frame_data.frame} + if format_type == "sly_pointcloud": + attributes["name"] = self._user["name"] + attributes["createdAt"] = self._user["createdAt"] + attributes["updatedAt"] = self._user["updatedAt"] + attributes["labels"] = [] + for (idx, (_, label)) in enumerate(project_data.meta['project']['labels']): + attributes["labels"].append({"label_id": idx, "name": label["name"], "color": label["color"]}) + attributes["track_id"] = -1 + + dm_item = datumaro.DatasetItem(id=osp.split(frame_data.name)[-1].split('.')[0], + annotations=dm_anno, point_cloud=dm_image[0], related_images=dm_image[1], + attributes=attributes) dm_items.append(dm_item) self._items = dm_items -def GetCVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_images: bool=False): +def GetCVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_images: bool = False, format_type: str = None, dimension: DimensionType = DimensionType.DIM_2D): if isinstance(instance_data, ProjectData): - return CVATProjectDataExtractor(instance_data, include_images) + return CVATProjectDataExtractor(instance_data, include_images, format_type, dimension) else: - return CvatTaskDataExtractor(instance_data, include_images) + return CvatTaskDataExtractor(instance_data, include_images, format_type, dimension) class CvatImportError(Exception): pass diff --git a/cvat/apps/dataset_manager/formats/pointcloud.py b/cvat/apps/dataset_manager/formats/pointcloud.py index 458029a132d..0009cd2f5c8 100644 --- a/cvat/apps/dataset_manager/formats/pointcloud.py +++ b/cvat/apps/dataset_manager/formats/pointcloud.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, TaskData, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive from cvat.apps.engine.models import DimensionType @@ -18,10 +18,7 @@ @exporter(name='Sly Point Cloud Format', ext='ZIP', version='1.0', dimension=DimensionType.DIM_3D) def _export_images(dst_file, task_data, save_images=False): - if not isinstance(task_data, TaskData): - raise Exception("Export to \"Sly Point Cloud\" format is working only with tasks temporarily") - - dataset = Dataset.from_extractors(CvatTaskDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( task_data, include_images=save_images, format_type='sly_pointcloud', dimension=DimensionType.DIM_3D), env=dm_env) with TemporaryDirectory() as temp_dir: diff --git a/cvat/apps/dataset_manager/formats/velodynepoint.py b/cvat/apps/dataset_manager/formats/velodynepoint.py index 7384f7beabe..747c47513d8 100644 --- a/cvat/apps/dataset_manager/formats/velodynepoint.py +++ b/cvat/apps/dataset_manager/formats/velodynepoint.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor, TaskData, \ +from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, \ import_dm_annotations from .registry import dm_env @@ -20,10 +20,7 @@ @exporter(name='Kitti Raw Format', ext='ZIP', version='1.0', dimension=DimensionType.DIM_3D) def _export_images(dst_file, task_data, save_images=False): - if not isinstance(task_data, TaskData): - raise Exception("Export to \"Kitti raw\" format is working only with tasks temporarily") - - dataset = Dataset.from_extractors(CvatTaskDataExtractor( + dataset = Dataset.from_extractors(GetCVATDataExtractor( task_data, include_images=save_images, format_type="kitti_raw", dimension=DimensionType.DIM_3D), env=dm_env) with TemporaryDirectory() as temp_dir: From cbb7a1e1a39d3871e929003ef135ee4195eaf652 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 3 Aug 2021 23:38:57 +0300 Subject: [PATCH 52/63] Update cvat/apps/dataset_manager/bindings.py Co-authored-by: Maxim Zhiltsov --- cvat/apps/dataset_manager/bindings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index fc777fc1883..e7639740ec4 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -793,7 +793,7 @@ class CVATDataExtractor(datumaro.SourceExtractor): def __init__(self): super().__init__() self.dm_items: List[datumaro.DatasetItem] = [] - self._categories: Dict[datumaro.AnnotationType, datumaro.LabelCategories] = {} + self._categories: Dict[datumaro.AnnotationType, datumaro.Categories] = {} @staticmethod def _load_categories(labels: list): From 566e49034a20c85a022730632ca24a36d2b8f7a8 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 4 Aug 2021 02:47:14 +0300 Subject: [PATCH 53/63] Fixed project extractor --- cvat/apps/dataset_manager/bindings.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index e7639740ec4..6a61b2cf93c 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -789,11 +789,12 @@ def _get_filename(path): return osp.splitext(path)[0] -class CVATDataExtractor(datumaro.SourceExtractor): +class CVATDataExtractorMixin: def __init__(self): super().__init__() - self.dm_items: List[datumaro.DatasetItem] = [] - self._categories: Dict[datumaro.AnnotationType, datumaro.Categories] = {} + + def categories(self) -> dict: + raise NotImplementedError() @staticmethod def _load_categories(labels: list): @@ -823,7 +824,7 @@ def map_label(name): return label_cat.find(name)[0] return convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label) -class CvatTaskDataExtractor(CVATDataExtractor): +class CvatTaskDataExtractor(datumaro.SourceExtractor, CVATDataExtractorMixin): def __init__(self, task_data, include_images=False, format_type=None, dimension=DimensionType.DIM_2D): super().__init__() self._categories, self._user = self._load_categories(task_data, dimension=dimension) @@ -938,7 +939,7 @@ def map_label(name): return label_cat.find(name)[0] return convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label, self._format_type, self._dimension) -class CVATProjectDataExtractor(CVATDataExtractor): +class CVATProjectDataExtractor(datumaro.Extractor, CVATDataExtractorMixin): def __init__(self, project_data: ProjectData, include_images: bool = False): super().__init__() self._categories = self._load_categories(project_data.meta['project']['labels']) @@ -993,6 +994,15 @@ def _make_image(i, **kwargs): self._items = dm_items + def categories(self): + return self._categories + + def __iter__(self): + yield from self._items + + def __len__(self): + return len(self._items) + def GetCVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_images: bool=False): if isinstance(instance_data, ProjectData): From 333bc8850e4b07d21898ab8fcd41ae5303dcc1dd Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 5 Aug 2021 11:12:17 +0300 Subject: [PATCH 54/63] Fixed project export cache invalidation --- cvat/apps/dataset_manager/views.py | 7 +++++-- cvat/apps/engine/views.py | 9 ++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/cvat/apps/dataset_manager/views.py b/cvat/apps/dataset_manager/views.py index 5e95e1c96b0..4f51c69a91e 100644 --- a/cvat/apps/dataset_manager/views.py +++ b/cvat/apps/dataset_manager/views.py @@ -63,9 +63,12 @@ def export(dst_format, task_id=None, project_id=None, server_url=None, save_imag output_path = '%s.%s' % (output_base, exporter.EXT) output_path = osp.join(cache_dir, output_path) - task_time = timezone.localtime(db_instance.updated_date).timestamp() + instance_time = timezone.localtime(db_instance.updated_date).timestamp() + if isinstance(db_instance, Project): + tasks_update = list(map(lambda db_task: timezone.localtime(db_task.updated_date).timestamp(), db_instance.tasks.all())) + instance_time = max(tasks_update + [instance_time]) if not (osp.exists(output_path) and \ - task_time <= osp.getmtime(output_path)): + instance_time <= osp.getmtime(output_path)): os.makedirs(cache_dir, exist_ok=True) with tempfile.TemporaryDirectory(dir=cache_dir) as temp_dir: temp_file = osp.join(temp_dir, 'result') diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index bb4dbed7235..2cd724edcab 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -1460,9 +1460,12 @@ def _export_annotations(db_instance, rq_id, request, format_name, action, callba rq_job = queue.fetch_job(rq_id) if rq_job: - last_task_update_time = timezone.localtime(db_instance.updated_date) + last_instance_update_time = timezone.localtime(db_instance.updated_date) + if isinstance(db_instance, Project): + tasks_update = list(map(lambda db_task: timezone.localtime(db_task.updated_date), db_instance.tasks.all())) + last_instance_update_time = max(tasks_update + [last_instance_update_time]) request_time = rq_job.meta.get('request_time', None) - if request_time is None or request_time < last_task_update_time: + if request_time is None or request_time < last_instance_update_time: rq_job.cancel() rq_job.delete() else: @@ -1471,7 +1474,7 @@ def _export_annotations(db_instance, rq_id, request, format_name, action, callba if action == "download" and osp.exists(file_path): rq_job.delete() - timestamp = datetime.strftime(last_task_update_time, + timestamp = datetime.strftime(last_instance_update_time, "%Y_%m_%d_%H_%M_%S") filename = filename or \ "{}_{}-{}-{}{}".format( From f5c89efa0862dcaf965a56778e2543eeb312c160 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 5 Aug 2021 15:31:11 +0300 Subject: [PATCH 55/63] Added CHANGELOG, increased versions --- CHANGELOG.md | 1 + cvat-core/package-lock.json | 2 +- cvat-core/package.json | 2 +- cvat-ui/package-lock.json | 2 +- cvat-ui/package.json | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c610dca1103..09c4ffb73de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Notification if the browser does not support nesassary API +- Added ability to export project as a dataset () ### Changed diff --git a/cvat-core/package-lock.json b/cvat-core/package-lock.json index 6daac70214e..e0b9594df79 100644 --- a/cvat-core/package-lock.json +++ b/cvat-core/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.13.3", + "version": "3.14.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-core/package.json b/cvat-core/package.json index 1cf4c506da2..9c169475461 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.13.3", + "version": "3.14.0", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "babel.config.js", "scripts": { diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json index 2e0e519945e..f878779282c 100644 --- a/cvat-ui/package-lock.json +++ b/cvat-ui/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.21.2", + "version": "1.22.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-ui/package.json b/cvat-ui/package.json index a200543ff0c..977098b8531 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.21.2", + "version": "1.22.0", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { From ddf8c3b6fb7286212fe1cbe7af6f86f8e23d08c1 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Mon, 9 Aug 2021 14:01:01 +0300 Subject: [PATCH 56/63] separated user info function --- cvat/apps/dataset_manager/bindings.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index f3bb82e5dc5..2bd3942cec9 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -799,17 +799,12 @@ def categories(self) -> dict: raise NotImplementedError() @staticmethod - def _load_categories(meta: dict, dimension): + def _load_categories(labels: list): categories: Dict[datumaro.AnnotationType, datumaro.Categories] = {} label_categories = datumaro.LabelCategories(attributes=['occluded']) - user_info = {} - if dimension == DimensionType.DIM_3D: - user_info = {"name": meta['owner']['username'], - "createdAt": meta['created'], - "updatedAt": meta['updated']} - for _, label in meta['labels']: + for _, label in labels: label_categories.add(label['name']) for _, attr in label['attributes']: label_categories.attributes.add(attr['name']) @@ -817,7 +812,15 @@ def _load_categories(meta: dict, dimension): categories[datumaro.AnnotationType.label] = label_categories - return categories, user_info + return categories + + @staticmethod + def _load_user_info(meta: dict): + return { + "name": meta['owner']['username'], + "createdAt": meta['created'], + "updatedAt": meta['updated'] + } def _read_cvat_anno(self, cvat_frame_anno: Union[ProjectData.Frame, TaskData.Frame], labels: list): categories = self.categories() @@ -834,7 +837,8 @@ def map_label(name): return label_cat.find(name)[0] class CvatTaskDataExtractor(datumaro.SourceExtractor, CVATDataExtractorMixin): def __init__(self, task_data, include_images=False, format_type=None, dimension=DimensionType.DIM_2D): super().__init__() - self._categories, self._user = self._load_categories(task_data.meta['task'], dimension=dimension) + self._categories = self._load_categories(task_data.meta['task']['labels']) + self._user = self._load_user_info(task_data.meta['task']) if dimension == DimensionType.DIM_3D else {} self._dimension = dimension self._format_type = format_type dm_items = [] @@ -926,7 +930,8 @@ def map_label(name): return label_cat.find(name)[0] class CVATProjectDataExtractor(datumaro.Extractor, CVATDataExtractorMixin): def __init__(self, project_data: ProjectData, include_images: bool = False, format_type: str = None, dimension: DimensionType = DimensionType.DIM_2D): super().__init__() - self._categories, self._user = self._load_categories(project_data.meta['project'], dimension) + self._categories = self._load_categories(project_data.meta['project']['labels']) + self._user = self._load_user_info(project_data.meta['project']) if dimension == DimensionType.DIM_3D else {} self._dimension = dimension self._format_type = format_type From bed869cec66ac8b10d3c3de16a8f46ef1b8a418d Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 10 Aug 2021 09:42:26 +0300 Subject: [PATCH 57/63] Fixed subset missing --- cvat/apps/dataset_manager/bindings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 2bd3942cec9..e3f56bf4671 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -1013,7 +1013,7 @@ def _make_image(i, **kwargs): dm_item = datumaro.DatasetItem(id=osp.split(frame_data.name)[-1].split('.')[0], annotations=dm_anno, point_cloud=dm_image[0], related_images=dm_image[1], - attributes=attributes) + attributes=attributes, subset=frame_data.subset) dm_items.append(dm_item) self._items = dm_items From d2c03c9cfe4443922c86749e9d5f13d0fa007279 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 10 Aug 2021 12:45:04 +0300 Subject: [PATCH 58/63] Added prefetch --- cvat/apps/dataset_manager/bindings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index e3f56bf4671..6805197742f 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -945,12 +945,13 @@ def __init__(self, project_data: ProjectData, include_images: bool = False, form ext_per_task[task.id] = FrameProvider.VIDEO_FRAME_EXT if is_video else '' if self._dimension == DimensionType.DIM_3D: def image_maker_factory(task): + images_query = task.data.images.prefetch_related() def _make_image(i, **kwargs): loader = osp.join( task.data.get_upload_dirname(), kwargs['path'], ) related_images = [] - image = Img.objects.get(id=i) + image = images_query.get(id=i) for i in image.related_files.all(): path = osp.realpath(str(i.path)) if osp.isfile(path): From 88a3ace088d597c31ff7bfc774c1ca32bd5bd9d9 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 10 Aug 2021 13:24:40 +0300 Subject: [PATCH 59/63] Added CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e3f6fadc51..b2b403c7204 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Notification if the browser does not support nesassary API -- Added ability to export project as a dataset () +- Added ability to export project as a dataset () and dataset with 3D tasks () - Additional inline tips in interactors with demo gifs () ### Changed From f1cb156886f58491351ab66449edb2397c0e6bfb Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 10 Aug 2021 13:28:03 +0300 Subject: [PATCH 60/63] Fixed CHANGELOG --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2b403c7204..9ce412c1be2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Notification if the browser does not support nesassary API -- Added ability to export project as a dataset () and dataset with 3D tasks () +- Added ability to export project as a dataset () + and project with 3D tasks () - Additional inline tips in interactors with demo gifs () ### Changed From ac2790f95af8fa84e780aa3246f44329fc73f2f0 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 12 Aug 2021 12:16:23 +0300 Subject: [PATCH 61/63] Added dimension to project --- cvat-core/package-lock.json | 2 +- cvat-core/package.json | 2 +- cvat-core/src/project.js | 16 ++++++- .../export-dataset/export-dataset-modal.tsx | 47 ++++++++++--------- cvat/apps/engine/serializers.py | 11 +++-- 5 files changed, 50 insertions(+), 28 deletions(-) diff --git a/cvat-core/package-lock.json b/cvat-core/package-lock.json index e0b9594df79..15be69a6b42 100644 --- a/cvat-core/package-lock.json +++ b/cvat-core/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.14.0", + "version": "3.15.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-core/package.json b/cvat-core/package.json index 9c169475461..229b9bec6ae 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.14.0", + "version": "3.15.0", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "babel.config.js", "scripts": { diff --git a/cvat-core/src/project.js b/cvat-core/src/project.js index bd25fc0f1b1..7e324498b95 100644 --- a/cvat-core/src/project.js +++ b/cvat-core/src/project.js @@ -34,6 +34,7 @@ task_subsets: undefined, training_project: undefined, task_ids: undefined, + dimension: undefined, }; for (const property in data) { @@ -153,7 +154,7 @@ /** * @name createdDate * @type {string} - * @memberof module:API.cvat.classes.Task + * @memberof module:API.cvat.classes.Project * @readonly * @instance */ @@ -163,13 +164,24 @@ /** * @name updatedDate * @type {string} - * @memberof module:API.cvat.classes.Task + * @memberof module:API.cvat.classes.Project * @readonly * @instance */ updatedDate: { get: () => data.updated_date, }, + /** + * Dimesion of the tasks in the project, if no task dimension is null + * @name dimension + * @type {string} + * @memberof module:API.cvat.classes.Project + * @readonly + * @instance + */ + dimension: { + get: () => data.dimension, + }, /** * After project has been created value can be appended only. * @name labels diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx index 400cdc4e21c..d1660dfee78 100644 --- a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -34,9 +34,9 @@ function ExportDatasetModal(): JSX.Element { const instance = useSelector((state: CombinedState) => state.export.instance); const modalVisible = useSelector((state: CombinedState) => state.export.modalVisible); const dumpers = useSelector((state: CombinedState) => state.formats.annotationFormats.dumpers); - const { - tasks: taskExportActivities, projects: projectExportActivities, - } = useSelector((state: CombinedState) => state.export); + const { tasks: taskExportActivities, projects: projectExportActivities } = useSelector( + (state: CombinedState) => state.export, + ); const initActivities = (): void => { if (instance instanceof core.classes.Project) { @@ -62,19 +62,28 @@ function ExportDatasetModal(): JSX.Element { dispatch(exportActions.closeExportModal()); }; - const handleExport = useCallback((values: FormValues): void => { - // have to validate format before so it would not be undefined - dispatch( - exportDatasetAsync(instance, values.selectedFormat as string, values.customName ? `${values.customName}.zip` : '', values.saveImages), - ); - closeModal(); - Notification.info({ - message: 'Dataset export started', - description: `Dataset export was started for ${instanceType} #${instance?.id}. ` + - 'Download will start automaticly as soon as the dataset is ready.', - className: `cvat-notification-notice-export-${instanceType}-start`, - }); - }, [instance?.id, instance instanceof core.classes.Project, instanceType]); + const handleExport = useCallback( + (values: FormValues): void => { + // have to validate format before so it would not be undefined + dispatch( + exportDatasetAsync( + instance, + values.selectedFormat as string, + values.customName ? `${values.customName}.zip` : '', + values.saveImages, + ), + ); + closeModal(); + Notification.info({ + message: 'Dataset export started', + description: + `Dataset export was started for ${instanceType} #${instance?.id}. ` + + 'Download will start automaticly as soon as the dataset is ready.', + className: `cvat-notification-notice-export-${instanceType}-start`, + }); + }, + [instance?.id, instance instanceof core.classes.Project, instanceType], + ); return ( {dumpers .sort((a: any, b: any) => a.name.localeCompare(b.name)) - .filter( - (dumper: any): boolean => - !(instance instanceof core.classes.Task) || - dumper.dimension === instance?.dimension, - ) + .filter((dumper: any): boolean => dumper.dimension === instance?.dimension) .map( (dumper: any): JSX.Element => { const pending = (activities || []).includes(dumper.name); diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 767b393e1f3..f50e799bf42 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -504,13 +504,15 @@ class ProjectWithoutTaskSerializer(serializers.ModelSerializer): owner_id = serializers.IntegerField(write_only=True, allow_null=True, required=False) assignee = BasicUserSerializer(allow_null=True, required=False) assignee_id = serializers.IntegerField(write_only=True, allow_null=True, required=False) + task_subsets = serializers.ListField(child=serializers.CharField(), required=False) training_project = TrainingProjectSerializer(required=False, allow_null=True) + dimension = serializers.CharField(max_length=16, required=False) class Meta: model = models.Project fields = ('url', 'id', 'name', 'labels', 'tasks', 'owner', 'assignee', 'owner_id', 'assignee_id', - 'bug_tracker', 'created_date', 'updated_date', 'status', 'training_project') - read_only_fields = ('created_date', 'updated_date', 'status', 'owner', 'asignee') + 'bug_tracker', 'task_subsets', 'created_date', 'updated_date', 'status', 'training_project', 'dimension') + read_only_fields = ('created_date', 'updated_date', 'status', 'owner', 'asignee', 'task_subsets', 'dimension') ordering = ['-id'] @@ -519,6 +521,7 @@ def to_representation(self, instance): task_subsets = set(instance.tasks.values_list('subset', flat=True)) task_subsets.discard('') response['task_subsets'] = list(task_subsets) + response['dimension'] = instance.tasks.first().dimension if instance.tasks.count() else None return response class ProjectSerializer(ProjectWithoutTaskSerializer): @@ -580,7 +583,9 @@ def validate_labels(self, value): return value def to_representation(self, instance): - return serializers.ModelSerializer.to_representation(self, instance) # ignoring subsets here + response = serializers.ModelSerializer.to_representation(self, instance) # ignoring subsets here + response['dimension'] = instance.tasks.first().dimension if instance.tasks.count() else None + return response class ExceptionSerializer(serializers.Serializer): system = serializers.CharField(max_length=255) From 84f328c99fae106651a3a3e6d0db08c09e107550 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 12 Aug 2021 13:01:15 +0300 Subject: [PATCH 62/63] Fixed serilazer getting --- cvat/apps/engine/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 2cd724edcab..afe9ef504fa 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -239,6 +239,8 @@ class ProjectViewSet(auth.ProjectGetQuerySetMixin, viewsets.ModelViewSet): http_method_names = ['get', 'post', 'head', 'patch', 'delete'] def get_serializer_class(self): + if self.request.path.endswith('tasks'): + return TaskSerializer if self.request.query_params and self.request.query_params.get("names_only") == "true": return ProjectSearchSerializer if self.request.query_params and self.request.query_params.get("without_tasks") == "true": From 579d2b41b5e8f189a1792396d6ee5bf9e794f598 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 17 Aug 2021 09:28:09 +0300 Subject: [PATCH 63/63] Fixed comments --- cvat-ui/src/components/projects-page/actions-menu.tsx | 3 ++- cvat/apps/dataset_manager/bindings.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cvat-ui/src/components/projects-page/actions-menu.tsx b/cvat-ui/src/components/projects-page/actions-menu.tsx index 75d71508652..c85684ad6db 100644 --- a/cvat-ui/src/components/projects-page/actions-menu.tsx +++ b/cvat-ui/src/components/projects-page/actions-menu.tsx @@ -37,12 +37,13 @@ export default function ProjectActionsMenuComponent(props: Props): JSX.Element { return ( - Delete dispatch(exportActions.openExportModal(projectInstance))} > Export project dataset +
+ Delete ); } diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 6805197742f..2062e0333fb 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -1012,7 +1012,7 @@ def _make_image(i, **kwargs): attributes["labels"].append({"label_id": idx, "name": label["name"], "color": label["color"]}) attributes["track_id"] = -1 - dm_item = datumaro.DatasetItem(id=osp.split(frame_data.name)[-1].split('.')[0], + dm_item = datumaro.DatasetItem(id=osp.splitext(osp.split(frame_data.name)[-1])[0], annotations=dm_anno, point_cloud=dm_image[0], related_images=dm_image[1], attributes=attributes, subset=frame_data.subset) dm_items.append(dm_item)