[Datumaro] Support relative paths

cvat/apps/dataset_manager/bindings.py

@@ -11,6 +11,7 @@
 import datumaro.components.extractor as datumaro
 from cvat.apps.engine.frame_provider import FrameProvider
 from cvat.apps.engine.models import AttributeType, ShapeType
+from datumaro.util import cast
 from datumaro.util.image import Image
 
 from .annotation import AnnotationManager, TrackManager
@@ -422,8 +423,9 @@ def __init__(self, task_data, include_images=False):
                 size=(frame_data.height, frame_data.width)
             )
             dm_anno = self._read_cvat_anno(frame_data, task_data)
-            dm_item = datumaro.DatasetItem(id=frame_data.frame,
-                annotations=dm_anno, image=dm_image)
+            dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0],
+                annotations=dm_anno, image=dm_image,
+                attributes={'frame': frame_data.frame})
             dm_items.append(dm_item)
 
         self._items = dm_items
@@ -533,23 +535,21 @@ def match_frame(item, task_data):
     is_video = task_data.meta['task']['mode'] == 'interpolation'
 
     frame_number = None
-    if frame_number is None:
-        try:
-            frame_number = task_data.match_frame(item.id)
-        except Exception:
-            pass
     if frame_number is None and item.has_image:
         try:
-            frame_number = task_data.match_frame(item.image.filename)
+            frame_number = task_data.match_frame(item.image.path)
         except Exception:
             pass
     if frame_number is None:
         try:
-            frame_number = int(item.id)
+            frame_number = task_data.match_frame(item.id)
         except Exception:
             pass
-    if frame_number is None and is_video and item.id.startswith('frame_'):
-        frame_number = int(item.id[len('frame_'):])
+    if frame_number is None:
+        frame_number = cast(item.attributes.get('frame', item.id), int)
+    if frame_number is None and is_video:
+        frame_number = cast(osp.basename(item.id)[len('frame_'):], int)
+
     if not frame_number in task_data.frame_info:
         raise Exception("Could not match item id: '%s' with any task frame" %
             item.id)

cvat/apps/dataset_manager/formats/labelme.py

@@ -17,8 +17,6 @@
 @exporter(name='LabelMe', ext='ZIP', version='3.0')
 def _export(dst_file, task_data, save_images=False):
     extractor = CvatTaskDataExtractor(task_data, include_images=save_images)
-    envt = dm_env.transforms
-    extractor = extractor.transform(envt.get('id_from_image_name'))
     extractor = Dataset.from_extractors(extractor) # apply lazy transforms
     with TemporaryDirectory() as temp_dir:
         converter = dm_env.make_converter('label_me', save_images=save_images)

cvat/apps/dataset_manager/formats/mask.py

@@ -24,7 +24,6 @@ def _export(dst_file, task_data, save_images=False):
     extractor = extractor.transform(envt.get('polygons_to_masks'))
     extractor = extractor.transform(envt.get('boxes_to_masks'))
     extractor = extractor.transform(envt.get('merge_instance_segments'))
-    extractor = extractor.transform(envt.get('id_from_image_name'))
     extractor = Dataset.from_extractors(extractor) # apply lazy transforms
     with TemporaryDirectory() as temp_dir:
         converter = dm_env.make_converter('voc_segmentation',

cvat/apps/dataset_manager/formats/mot.py

@@ -7,8 +7,7 @@
 from pyunpack import Archive
 
 import datumaro.components.extractor as datumaro
-from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor,
-    match_frame)
+from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor
 from cvat.apps.dataset_manager.util import make_zip_archive
 from datumaro.components.project import Dataset
 
@@ -18,8 +17,6 @@
 @exporter(name='MOT', ext='ZIP', version='1.1')
 def _export(dst_file, task_data, save_images=False):
     extractor = CvatTaskDataExtractor(task_data, include_images=save_images)
-    envt = dm_env.transforms
-    extractor = extractor.transform(envt.get('id_from_image_name'))
     extractor = Dataset.from_extractors(extractor) # apply lazy transforms
     with TemporaryDirectory() as temp_dir:
         converter = dm_env.make_converter('mot_seq_gt',
@@ -39,8 +36,8 @@ def _import(src_file, task_data):
         label_cat = dataset.categories()[datumaro.AnnotationType.label]
 
         for item in dataset:
-            item = item.wrap(id=int(item.id) - 1) # NOTE: MOT frames start from 1
-            frame_number = task_data.abs_frame_id(match_frame(item, task_data))
+            frame_number = int(item.id) - 1 # NOTE: MOT frames start from 1
+            frame_number = task_data.abs_frame_id(frame_number)
 
             for ann in item.annotations:
                 if ann.type != datumaro.AnnotationType.bbox:

cvat/apps/dataset_manager/formats/pascal_voc.py

@@ -22,8 +22,6 @@
 @exporter(name='PASCAL VOC', ext='ZIP', version='1.1')
 def _export(dst_file, task_data, save_images=False):
     extractor = CvatTaskDataExtractor(task_data, include_images=save_images)
-    envt = dm_env.transforms
-    extractor = extractor.transform(envt.get('id_from_image_name'))
     extractor = Dataset.from_extractors(extractor) # apply lazy transforms
     with TemporaryDirectory() as temp_dir:
         converter = dm_env.make_converter('voc', label_map='source',

datumaro/datumaro/components/extractor.py

@@ -578,7 +578,7 @@ class DatasetItem:
     def __init__(self, id=None, annotations=None,
             subset=None, path=None, image=None, attributes=None):
         assert id is not None
-        self._id = str(id)
+        self._id = str(id).replace('\\', '/')
 
         if subset is None:
             subset = ''

datumaro/datumaro/plugins/coco_format/converter.py

@@ -496,25 +496,19 @@ def _make_task_converters(self):
     def _get_image_id(self, item):
         image_id = self._image_ids.get(item.id)
         if image_id is None:
-            image_id = cast(item.id, int, len(self._image_ids) + 1)
+            image_id = cast(item.attributes.get('id'), int,
+                len(self._image_ids) + 1)
             self._image_ids[item.id] = image_id
         return image_id
 
-    def _save_image(self, item):
+    def _save_image(self, item, filename):
         image = item.image.data
         if image is None:
             log.warning("Item '%s' has no image" % item.id)
             return ''
 
-        filename = item.image.filename
-        if filename:
-            filename = osp.splitext(filename)[0]
-        else:
-            filename = item.id
-        filename += CocoPath.IMAGE_EXT
-        path = osp.join(self._images_dir, filename)
-        save_image(path, image)
-        return path
+        save_image(osp.join(self._images_dir, filename), image,
+            create_dir=True)
 
     def convert(self):
         self._make_dirs()
@@ -534,12 +528,10 @@ def convert(self):
             for task_conv in task_converters.values():
                 task_conv.save_categories(subset)
             for item in subset:
-                filename = ''
-                if item.has_image:
-                    filename = item.image.path
+                filename = item.id + CocoPath.IMAGE_EXT
                 if self._save_images:
                     if item.has_image:
-                        filename = self._save_image(item)
+                        self._save_image(item, filename)
                     else:
                         log.debug("Item '%s' has no image info" % item.id)
                 for task_conv in task_converters.values():

datumaro/datumaro/plugins/coco_format/extractor.py

@@ -125,8 +125,10 @@ def _load_items(self, loader):
             anns = loader.loadAnns(anns)
             anns = sum((self._load_annotations(a, image_info) for a in anns), [])
 
-            items[img_id] = DatasetItem(id=img_id, subset=self._subset,
-                image=image, annotations=anns)
+            items[img_id] = DatasetItem(
+                id=osp.splitext(image_info['file_name'])[0],
+                subset=self._subset, image=image, annotations=anns,
+                attributes={'id': img_id})
 
         return items
 

datumaro/datumaro/plugins/cvat_format/converter.py

@@ -163,37 +163,30 @@ def write(self):
 
         self._writer.close_root()
 
-    def _save_image(self, item):
+    def _save_image(self, item, filename):
         image = item.image.data
         if image is None:
             log.warning("Item '%s' has no image" % item.id)
             return ''
 
-        filename = item.image.filename
-        if filename:
-            filename = osp.splitext(filename)[0]
-        else:
-            filename = item.id
-        filename += CvatPath.IMAGE_EXT
-        image_path = osp.join(self._context._images_dir, filename)
-        save_image(image_path, image)
-        return filename
+        save_image(osp.join(self._context._images_dir, filename), image,
+            create_dir=True)
 
     def _write_item(self, item, index):
         image_info = OrderedDict([
-            ("id", str(cast(item.id, int, index))),
+            ("id", str(cast(item.attributes.get('frame'), int, index))),
         ])
+        filename = item.id + CvatPath.IMAGE_EXT
+        image_info["name"] = filename
         if item.has_image:
             size = item.image.size
             if size:
                 h, w = size
                 image_info["width"] = str(w)
                 image_info["height"] = str(h)
 
-            filename = item.image.filename
             if self._context._save_images:
-                filename = self._save_image(item)
-            image_info["name"] = filename
+                self._save_image(item, filename)
         else:
             log.debug("Item '%s' has no image info" % item.id)
         self._writer.open_image(image_info)

datumaro/datumaro/plugins/cvat_format/extractor.py

@@ -303,17 +303,14 @@ def _parse_tag_ann(cls, ann, categories):
 
     def _load_items(self, parsed):
         for frame_id, item_desc in parsed.items():
-            path = item_desc.get('name', 'frame_%06d.png' % int(frame_id))
+            name = item_desc.get('name', 'frame_%06d.png' % int(frame_id))
+            image = osp.join(self._images_dir, name)
             image_size = (item_desc.get('height'), item_desc.get('width'))
             if all(image_size):
-                image_size = (int(image_size[0]), int(image_size[1]))
-            else:
-                image_size = None
-            image = None
-            if path:
-                image = Image(path=osp.join(self._images_dir, path),
-                    size=image_size)
-
-            parsed[frame_id] = DatasetItem(id=frame_id, subset=self._subset,
-                image=image, annotations=item_desc.get('annotations'))
+                image = Image(path=image, size=tuple(map(int, image_size)))
+
+            parsed[frame_id] = DatasetItem(id=osp.splitext(name)[0],
+                subset=self._subset, image=image,
+                annotations=item_desc.get('annotations'),
+                attributes={'frame': int(frame_id)})
         return parsed

datumaro/datumaro/plugins/datumaro_format/converter.py

@@ -253,12 +253,7 @@ def _save_image(self, item):
         if image is None:
             return ''
 
-        filename = item.image.filename
-        if filename:
-            filename = osp.splitext(filename)[0]
-        else:
-            filename = item.id
-        filename += DatumaroPath.IMAGE_EXT
+        filename = item.id + DatumaroPath.IMAGE_EXT
         image_path = osp.join(self._images_dir, filename)
         save_image(image_path, image, create_dir=True)
         return filename

datumaro/datumaro/plugins/datumaro_format/extractor.py

@@ -82,10 +82,11 @@ def _load_items(self, parsed):
             item_id = item_desc['id']
 
             image = None
-            image_info = item_desc.get('image', {})
+            image_info = item_desc.get('image')
             if image_info:
-                image_path = osp.join(self._images_dir,
-                    image_info.get('path', '')) # relative or absolute fits
+                image_path = image_info.get('path') or \
+                    item_id + DatumaroPath.IMAGE_EXT
+                image_path = osp.join(self._images_dir, image_path)
                 image = Image(path=image_path, size=image_info.get('size'))
 
             annotations = self._load_annotations(item_desc)

datumaro/datumaro/plugins/image_dir.py

@@ -3,7 +3,6 @@
 #
 # SPDX-License-Identifier: MIT
 
-from collections import OrderedDict
 import os
 import os.path as osp
 
@@ -41,29 +40,24 @@ def __init__(self, url):
         assert osp.isdir(url), url
 
         items = []
-        for name in os.listdir(url):
-            path = osp.join(url, name)
-            if self._is_image(path):
-                item_id = osp.splitext(name)[0]
-                item = DatasetItem(id=item_id, image=path)
-                items.append((item.id, item))
-
-        items = sorted(items, key=lambda e: e[0])
-        items = OrderedDict(items)
+        for dirpath, _, filenames in os.walk(url):
+            for name in filenames:
+                path = osp.join(dirpath, name)
+                if not self._is_image(path):
+                    continue
+
+                item_id = osp.relpath(osp.splitext(path)[0], url)
+                items.append(DatasetItem(id=item_id, image=path))
+
         self._items = items
 
     def __iter__(self):
-        for item in self._items.values():
+        for item in self._items:
             yield item
 
     def __len__(self):
         return len(self._items)
 
-    def get(self, item_id, subset=None, path=None):
-        if path or subset:
-            raise KeyError()
-        return self._items[item_id]
-
     def _is_image(self, path):
         if not osp.isfile(path):
             return False
@@ -79,11 +73,5 @@ def __call__(self, extractor, save_dir):
 
         for item in extractor:
             if item.has_image and item.image.has_data:
-                filename = item.image.filename
-                if filename:
-                    filename = osp.splitext(filename)[0]
-                else:
-                    filename = item.id
-                filename += '.jpg'
-                save_image(osp.join(save_dir, filename), item.image.data,
-                    create_dir=True)
+                save_image(osp.join(save_dir, item.id + '.jpg'),
+                    item.image.data, create_dir=True)

datumaro/datumaro/plugins/labelme_format.py

@@ -331,16 +331,13 @@ def _save_item(self, item, subset_dir):
 
         log.debug("Converting item '%s'", item.id)
 
-        image_filename = ''
-        if item.has_image:
-            image_filename = item.image.filename
+        if '/' in item.id:
+            raise Exception("Can't export item '%s': "
+                "LabelMe format only supports flat image layout" % item.id)
+
+        image_filename = item.id + LabelMePath.IMAGE_EXT
         if self._save_images:
             if item.has_image and item.image.has_data:
-                if image_filename:
-                    image_filename = osp.splitext(image_filename)[0]
-                else:
-                    image_filename = item.id
-                image_filename += LabelMePath.IMAGE_EXT
                 save_image(osp.join(subset_dir, image_filename),
                     item.image.data, create_dir=True)
             else:

datumaro/datumaro/plugins/mot_format.py

@@ -311,7 +311,9 @@ def __call__(self, extractor, save_dir):
 
                 if self._save_images:
                     if item.has_image and item.image.has_data:
-                        self._save_image(item, index=frame_id)
+                        save_image(osp.join(self._images_dir,
+                                '%06d%s' % (frame_id, MotPath.IMAGE_EXT)),
+                            item.image.data)
                     else:
                         log.debug("Item '%s' has no image" % item.id)
 
@@ -320,13 +322,3 @@ def __call__(self, extractor, save_dir):
             f.write('\n'.join(l.name
                 for l in extractor.categories()[AnnotationType.label].items)
             )
-
-    def _save_image(self, item, index):
-        if item.image.filename:
-            frame_id = osp.splitext(item.image.filename)[0]
-        else:
-            frame_id = item.id
-        frame_id = cast(frame_id, int, index)
-        image_filename = '%06d%s' % (frame_id, MotPath.IMAGE_EXT)
-        save_image(osp.join(self._images_dir, image_filename),
-            item.image.data)