Project export with 3d tasks (#3502)

Co-authored-by: Maxim Zhiltsov <maxim.zhiltsov@intel.com>
Co-authored-by: dvkruchinin <dvkruchinin@gmail.com>
Co-authored-by: Nikita Manovich <nikita.manovich@intel.com>

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added ability to import data from share with cli without copying the data (<https://github.com/openvinotoolkit/cvat/issues/2862>)
 - Notification if the browser does not support nesassary API
 - Added ability to export project as a dataset (<https://github.com/openvinotoolkit/cvat/pull/3365>)
+  and project with 3D tasks (<https://github.com/openvinotoolkit/cvat/pull/3502>)
 - Additional inline tips in interactors with demo gifs (<https://github.com/openvinotoolkit/cvat/pull/3473>)
 
 ### Changed

cvat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "3.14.0",
+  "version": "3.15.0",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "babel.config.js",
   "scripts": {

cvat-core/src/project.js

@@ -34,6 +34,7 @@
                 task_subsets: undefined,
                 training_project: undefined,
                 task_ids: undefined,
+                dimension: undefined,
             };
 
             for (const property in data) {
@@ -153,7 +154,7 @@
                     /**
                      * @name createdDate
                      * @type {string}
-                     * @memberof module:API.cvat.classes.Task
+                     * @memberof module:API.cvat.classes.Project
                      * @readonly
                      * @instance
                      */
@@ -163,13 +164,24 @@
                     /**
                      * @name updatedDate
                      * @type {string}
-                     * @memberof module:API.cvat.classes.Task
+                     * @memberof module:API.cvat.classes.Project
                      * @readonly
                      * @instance
                      */
                     updatedDate: {
                         get: () => data.updated_date,
                     },
+                    /**
+                     * Dimesion of the tasks in the project, if no task dimension is null
+                     * @name dimension
+                     * @type {string}
+                     * @memberof module:API.cvat.classes.Project
+                     * @readonly
+                     * @instance
+                     */
+                    dimension: {
+                        get: () => data.dimension,
+                    },
                     /**
                      * After project has been created value can be appended only.
                      * @name labels

cvat-ui/src/components/export-dataset/export-dataset-modal.tsx

@@ -34,9 +34,9 @@ function ExportDatasetModal(): JSX.Element {
     const instance = useSelector((state: CombinedState) => state.export.instance);
     const modalVisible = useSelector((state: CombinedState) => state.export.modalVisible);
     const dumpers = useSelector((state: CombinedState) => state.formats.annotationFormats.dumpers);
-    const {
-        tasks: taskExportActivities, projects: projectExportActivities,
-    } = useSelector((state: CombinedState) => state.export);
+    const { tasks: taskExportActivities, projects: projectExportActivities } = useSelector(
+        (state: CombinedState) => state.export,
+    );
 
     const initActivities = (): void => {
         if (instance instanceof core.classes.Project) {
@@ -62,19 +62,28 @@ function ExportDatasetModal(): JSX.Element {
         dispatch(exportActions.closeExportModal());
     };
 
-    const handleExport = useCallback((values: FormValues): void => {
-        // have to validate format before so it would not be undefined
-        dispatch(
-            exportDatasetAsync(instance, values.selectedFormat as string, values.customName ? `${values.customName}.zip` : '', values.saveImages),
-        );
-        closeModal();
-        Notification.info({
-            message: 'Dataset export started',
-            description: `Dataset export was started for ${instanceType} #${instance?.id}. ` +
-                'Download will start automaticly as soon as the dataset is ready.',
-            className: `cvat-notification-notice-export-${instanceType}-start`,
-        });
-    }, [instance?.id, instance instanceof core.classes.Project, instanceType]);
+    const handleExport = useCallback(
+        (values: FormValues): void => {
+            // have to validate format before so it would not be undefined
+            dispatch(
+                exportDatasetAsync(
+                    instance,
+                    values.selectedFormat as string,
+                    values.customName ? `${values.customName}.zip` : '',
+                    values.saveImages,
+                ),
+            );
+            closeModal();
+            Notification.info({
+                message: 'Dataset export started',
+                description:
+                    `Dataset export was started for ${instanceType} #${instance?.id}. ` +
+                    'Download will start automaticly as soon as the dataset is ready.',
+                className: `cvat-notification-notice-export-${instanceType}-start`,
+            });
+        },
+        [instance?.id, instance instanceof core.classes.Project, instanceType],
+    );
 
     return (
         <Modal
@@ -106,11 +115,7 @@ function ExportDatasetModal(): JSX.Element {
                     <Select placeholder='Select dataset format' className='cvat-modal-export-select'>
                         {dumpers
                             .sort((a: any, b: any) => a.name.localeCompare(b.name))
-                            .filter(
-                                (dumper: any): boolean =>
-                                    !(instance instanceof core.classes.Task) ||
-                                    dumper.dimension === instance?.dimension,
-                            )
+                            .filter((dumper: any): boolean => dumper.dimension === instance?.dimension)
                             .map(
                                 (dumper: any): JSX.Element => {
                                     const pending = (activities || []).includes(dumper.name);

cvat-ui/src/components/projects-page/actions-menu.tsx

@@ -37,12 +37,13 @@ export default function ProjectActionsMenuComponent(props: Props): JSX.Element {
 
     return (
         <Menu className='cvat-project-actions-menu'>
-            <Menu.Item onClick={onDeleteProject}>Delete</Menu.Item>
             <Menu.Item
                 onClick={() => dispatch(exportActions.openExportModal(projectInstance))}
             >
                 Export project dataset
             </Menu.Item>
+            <hr />
+            <Menu.Item onClick={onDeleteProject}>Delete</Menu.Item>
         </Menu>
     );
 }

cvat/apps/dataset_manager/bindings.py

@@ -513,7 +513,7 @@ class ProjectData(InstanceLabelData):
     Track = NamedTuple('Track', [('label', str), ('group', int), ('source', str), ('shapes', List[TrackedShape]), ('task_id', int)])
     Tag = NamedTuple('Tag', [('frame', int), ('label', str), ('attributes', List[InstanceLabelData.Attribute]), ('source', str), ('group', int), ('task_id', int)])
     Tag.__new__.__defaults__ = (0, )
-    Frame = NamedTuple('Frame', [('task_id', int), ('subset', str), ('idx', int), ('frame', int), ('name', str), ('width', int), ('height', int), ('labeled_shapes', List[Union[LabeledShape, TrackedShape]]), ('tags', List[Tag])])
+    Frame = NamedTuple('Frame', [('task_id', int), ('subset', str), ('idx', int), ('id', int), ('frame', int), ('name', str), ('width', int), ('height', int), ('labeled_shapes', List[Union[LabeledShape, TrackedShape]]), ('tags', List[Tag])])
 
     def __init__(self, annotation_irs: Mapping[str, AnnotationIR], db_project: Project, host: str, create_callback: Callable = None):
         self._annotation_irs = annotation_irs
@@ -581,6 +581,7 @@ def _init_frame_info(self):
             else:
                 self._frame_info.update({(task.id, self.rel_frame_id(task.id, db_image.frame)): {
                     "path": mangle_image_name(db_image.path, defaulted_subset, original_names),
+                    "id": db_image.id,
                     "width": db_image.width,
                     "height": db_image.height,
                     "subset": defaulted_subset
@@ -683,6 +684,7 @@ def get_frame(task_id: int, idx: int) -> ProjectData.Frame:
                     task_id=task_id,
                     subset=frame_info["subset"],
                     idx=idx,
+                    id=frame_info.get('id',0),
                     frame=abs_frame,
                     name=frame_info["path"],
                     height=frame_info["height"],
@@ -807,10 +809,18 @@ def _load_categories(labels: list):
             for _, attr in label['attributes']:
                 label_categories.attributes.add(attr['name'])
 
+
         categories[datumaro.AnnotationType.label] = label_categories
 
         return categories
 
+    @staticmethod
+    def _load_user_info(meta: dict):
+        return {
+            "name": meta['owner']['username'],
+            "createdAt": meta['created'],
+            "updatedAt": meta['updated']
+        }
 
     def _read_cvat_anno(self, cvat_frame_anno: Union[ProjectData.Frame, TaskData.Frame], labels: list):
         categories = self.categories()
@@ -827,7 +837,8 @@ def map_label(name): return label_cat.find(name)[0]
 class CvatTaskDataExtractor(datumaro.SourceExtractor, CVATDataExtractorMixin):
     def __init__(self, task_data, include_images=False, format_type=None, dimension=DimensionType.DIM_2D):
         super().__init__()
-        self._categories, self._user = self._load_categories(task_data, dimension=dimension)
+        self._categories = self._load_categories(task_data.meta['task']['labels'])
+        self._user = self._load_user_info(task_data.meta['task']) if dimension == DimensionType.DIM_3D else {}
         self._dimension = dimension
         self._format_type = format_type
         dm_items = []
@@ -893,11 +904,9 @@ def _make_image(i, **kwargs):
                     attributes["createdAt"] = self._user["createdAt"]
                     attributes["updatedAt"] = self._user["updatedAt"]
                     attributes["labels"] = []
-                    index = 0
-                    for _, label in task_data.meta['task']['labels']:
-                        attributes["labels"].append({"label_id": index, "name": label["name"], "color": label["color"]})
+                    for (idx, (_, label)) in enumerate(task_data.meta['task']['labels']):
+                        attributes["labels"].append({"label_id": idx, "name": label["name"], "color": label["color"]})
                         attributes["track_id"] = -1
-                        index += 1
 
                 dm_item = datumaro.DatasetItem(id=osp.split(frame_data.name)[-1].split('.')[0],
                                                annotations=dm_anno, point_cloud=dm_image[0], related_images=dm_image[1],
@@ -907,27 +916,6 @@ def _make_image(i, **kwargs):
 
         self._items = dm_items
 
-    @staticmethod
-    def _load_categories(cvat_anno, dimension): # pylint: disable=arguments-differ
-        categories = {}
-
-        label_categories = datumaro.LabelCategories(attributes=['occluded'])
-
-        user_info = {}
-        if dimension == DimensionType.DIM_3D:
-            user_info = {"name": cvat_anno.meta['task']['owner']['username'],
-                         "createdAt": cvat_anno.meta['task']['created'],
-                         "updatedAt": cvat_anno.meta['task']['updated']}
-        for _, label in cvat_anno.meta['task']['labels']:
-            label_categories.add(label['name'])
-            for _, attr in label['attributes']:
-                label_categories.attributes.add(attr['name'])
-
-
-        categories[datumaro.AnnotationType.label] = label_categories
-
-        return categories, user_info
-
     def _read_cvat_anno(self, cvat_frame_anno: TaskData.Frame, labels: list):
         categories = self.categories()
         label_cat = categories[datumaro.AnnotationType.label]
@@ -940,9 +928,12 @@ def map_label(name): return label_cat.find(name)[0]
         return convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label, self._format_type, self._dimension)
 
 class CVATProjectDataExtractor(datumaro.Extractor, CVATDataExtractorMixin):
-    def __init__(self, project_data: ProjectData, include_images: bool = False):
+    def __init__(self, project_data: ProjectData, include_images: bool = False, format_type: str = None, dimension: DimensionType = DimensionType.DIM_2D):
         super().__init__()
         self._categories = self._load_categories(project_data.meta['project']['labels'])
+        self._user = self._load_user_info(project_data.meta['project']) if dimension == DimensionType.DIM_3D else {}
+        self._dimension = dimension
+        self._format_type = format_type
 
         dm_items: List[datumaro.DatasetItem] = []
 
@@ -952,12 +943,28 @@ def __init__(self, project_data: ProjectData, include_images: bool = False):
         for task in project_data.tasks:
             is_video = task.mode == 'interpolation'
             ext_per_task[task.id] = FrameProvider.VIDEO_FRAME_EXT if is_video else ''
-            if include_images:
-                frame_provider = FrameProvider(task.data)
+            if self._dimension == DimensionType.DIM_3D:
+                def image_maker_factory(task):
+                    images_query = task.data.images.prefetch_related()
+                    def _make_image(i, **kwargs):
+                        loader = osp.join(
+                            task.data.get_upload_dirname(), kwargs['path'],
+                        )
+                        related_images = []
+                        image = images_query.get(id=i)
+                        for i in image.related_files.all():
+                            path = osp.realpath(str(i.path))
+                            if osp.isfile(path):
+                                related_images.append(path)
+                        return loader, related_images
+                    return _make_image
+                image_maker_per_task[task.id] = image_maker_factory(task)
+            elif include_images:
                 if is_video:
                     # optimization for videos: use numpy arrays instead of bytes
                     # some formats or transforms can require image data
-                    def image_maker_factory(frame_provider):
+                    def image_maker_factory(task):
+                        frame_provider = FrameProvider(task.data)
                         def _make_image(i, **kwargs):
                             loader = lambda _: frame_provider.get_frame(i,
                                 quality=frame_provider.Quality.ORIGINAL,
@@ -966,30 +973,48 @@ def _make_image(i, **kwargs):
                         return _make_image
                 else:
                     # for images use encoded data to avoid recoding
-                    def image_maker_factory(frame_provider):
+                    def image_maker_factory(task):
+                        frame_provider = FrameProvider(task.data)
                         def _make_image(i, **kwargs):
                             loader = lambda _: frame_provider.get_frame(i,
                                 quality=frame_provider.Quality.ORIGINAL,
                                 out_type=frame_provider.Type.BUFFER)[0].getvalue()
                             return ByteImage(data=loader, **kwargs)
                         return _make_image
-                image_maker_per_task[task.id] = image_maker_factory(frame_provider)
+                image_maker_per_task[task.id] = image_maker_factory(task)
 
         for frame_data in project_data.group_by_frame(include_empty=True):
             image_args = {
                 'path': frame_data.name + ext_per_task[frame_data.task_id],
                 'size': (frame_data.height, frame_data.width),
             }
-            if include_images:
+            if self._dimension == DimensionType.DIM_3D:
+                dm_image = image_maker_per_task[frame_data.task_id](frame_data.id, **image_args)
+            elif include_images:
                 dm_image = image_maker_per_task[frame_data.task_id](frame_data.idx, **image_args)
             else:
                 dm_image = Image(**image_args)
             dm_anno = self._read_cvat_anno(frame_data, project_data.meta['project']['labels'])
-            dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0],
-                annotations=dm_anno, image=dm_image,
-                subset=frame_data.subset,
-                attributes={'frame': frame_data.frame}
-            )
+            if self._dimension == DimensionType.DIM_2D:
+                dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0],
+                    annotations=dm_anno, image=dm_image,
+                    subset=frame_data.subset,
+                    attributes={'frame': frame_data.frame}
+                )
+            else:
+                attributes = {'frame': frame_data.frame}
+                if format_type == "sly_pointcloud":
+                    attributes["name"] = self._user["name"]
+                    attributes["createdAt"] = self._user["createdAt"]
+                    attributes["updatedAt"] = self._user["updatedAt"]
+                    attributes["labels"] = []
+                    for (idx, (_, label)) in enumerate(project_data.meta['project']['labels']):
+                        attributes["labels"].append({"label_id": idx, "name": label["name"], "color": label["color"]})
+                        attributes["track_id"] = -1
+
+                dm_item = datumaro.DatasetItem(id=osp.splitext(osp.split(frame_data.name)[-1])[0],
+                                               annotations=dm_anno, point_cloud=dm_image[0], related_images=dm_image[1],
+                                               attributes=attributes, subset=frame_data.subset)
             dm_items.append(dm_item)
 
         self._items = dm_items
@@ -1004,11 +1029,11 @@ def __len__(self):
         return len(self._items)
 
 
-def GetCVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_images: bool=False):
+def GetCVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_images: bool = False, format_type: str = None, dimension: DimensionType = DimensionType.DIM_2D):
     if isinstance(instance_data, ProjectData):
-        return CVATProjectDataExtractor(instance_data, include_images)
+        return CVATProjectDataExtractor(instance_data, include_images, format_type, dimension)
     else:
-        return CvatTaskDataExtractor(instance_data, include_images)
+        return CvatTaskDataExtractor(instance_data, include_images, format_type, dimension)
 
 class CvatImportError(Exception):
     pass

cvat/apps/dataset_manager/formats/pointcloud.py

@@ -7,7 +7,7 @@
 
 from datumaro.components.dataset import Dataset
 
-from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, TaskData,
+from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor,
                                                 import_dm_annotations)
 from cvat.apps.dataset_manager.util import make_zip_archive
 from cvat.apps.engine.models import DimensionType
@@ -18,10 +18,7 @@
 @exporter(name='Sly Point Cloud Format', ext='ZIP', version='1.0', dimension=DimensionType.DIM_3D)
 def _export_images(dst_file, task_data, save_images=False):
 
-    if not isinstance(task_data, TaskData):
-        raise Exception("Export to \"Sly Point Cloud\" format is working only with tasks temporarily")
-
-    dataset = Dataset.from_extractors(CvatTaskDataExtractor(
+    dataset = Dataset.from_extractors(GetCVATDataExtractor(
         task_data, include_images=save_images, format_type='sly_pointcloud', dimension=DimensionType.DIM_3D), env=dm_env)
 
     with TemporaryDirectory() as temp_dir: