Remove max file count (#6292)

cvat/apps/engine/serializers.py

@@ -16,7 +16,6 @@
 from rest_framework import serializers, exceptions
 from django.contrib.auth.models import User, Group
 from django.db import transaction
-from django.conf import settings
 
 from cvat.apps.dataset_manager.formats.utils import get_label_color
 from cvat.apps.engine import models
@@ -951,15 +950,9 @@ def validate(self, attrs):
             and attrs['start_frame'] > attrs['stop_frame']:
             raise serializers.ValidationError('Stop frame must be more or equal start frame')
 
-        if (
-            (server_files := attrs.get('server_files'))
-            and attrs.get('cloud_storage_id')
-            and sum(1 for f in server_files if not f['file'].endswith('.jsonl')) > settings.CLOUD_STORAGE_MAX_FILES_COUNT
-        ):
-            raise serializers.ValidationError(f'The maximum number of the cloud storage attached files is {settings.CLOUD_STORAGE_MAX_FILES_COUNT}')
-
         filename_pattern = attrs.get('filename_pattern')
         server_files_exclude = attrs.get('server_files_exclude')
+        server_files = attrs.get('server_files', [])
 
         if filename_pattern and len(list(filter(lambda x: not x['file'].endswith('.jsonl'), server_files))):
             raise serializers.ValidationError('The filename_pattern can only be used with specified manifest or without server_files')

cvat/apps/engine/task.py

@@ -580,24 +580,15 @@ def _create_thread(
                     )
                 if cloud_storage_manifest_prefix:
                     additional_files = [os.path.join(cloud_storage_manifest_prefix, f) for f in additional_files]
-                if len(data['server_files']) + len(additional_files) > settings.CLOUD_STORAGE_MAX_FILES_COUNT:
-                    raise ValidationError(
-                        'The maximum number of the cloud storage attached files '
-                        f'is {settings.CLOUD_STORAGE_MAX_FILES_COUNT}')
             else:
-                number_of_files = len(data['server_files'])
                 while len(dirs):
                     directory = dirs.pop()
                     for f in cloud_storage_instance.list_files(prefix=directory, _use_flat_listing=True):
                         if f['type'] == 'REG':
                             additional_files.append(f['name'])
                         else:
                             dirs.append(f['name'])
-                    # we check the limit of files on each iteration to reduce the number of possible requests to the bucket
-                    if (len(additional_files) + len(dirs) + number_of_files) > settings.CLOUD_STORAGE_MAX_FILES_COUNT:
-                        raise ValidationError(
-                            'The maximum number of the cloud storage attached files '
-                            f'is {settings.CLOUD_STORAGE_MAX_FILES_COUNT}')
+
             data['server_files'].extend(additional_files)
             del additional_files
 
@@ -637,10 +628,6 @@ def _create_thread(
                 if not data['filename_pattern'] == '*':
                     additional_files = fnmatch.filter(additional_files, data['filename_pattern'])
 
-            if (len(additional_files)) > settings.CLOUD_STORAGE_MAX_FILES_COUNT:
-                raise ValidationError(
-                    'The maximum number of the cloud storage attached files '
-                    f'is {settings.CLOUD_STORAGE_MAX_FILES_COUNT}')
             data['server_files'].extend(additional_files)
 
         if db_data.storage_method == models.StorageMethodChoice.FILE_SYSTEM or not settings.USE_CACHE:

cvat/settings/base.py

@@ -522,9 +522,6 @@ class CVAT_QUEUES(Enum):
 DATA_UPLOAD_MAX_MEMORY_SIZE = 100 * 1024 * 1024  # 100 MB
 DATA_UPLOAD_MAX_NUMBER_FIELDS = None   # this django check disabled
 DATA_UPLOAD_MAX_NUMBER_FILES = None
-LOCAL_LOAD_MAX_FILES_COUNT = 500
-LOCAL_LOAD_MAX_FILES_SIZE = 512 * 1024 * 1024  # 512 MB
-CLOUD_STORAGE_MAX_FILES_COUNT = LOCAL_LOAD_MAX_FILES_COUNT
 
 RESTRICTIONS = {
     # allow access to analytics component to users with business role