Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove max file count #6292

Merged
merged 5 commits into from
Jun 13, 2023
Merged

Remove max file count #6292

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3c48a61
remove CLOUD_STORAGE_MAX_FILES_COUNT and LOCAL_LOAD_MAX_FILES_COUNT
sizov-kirill Jun 12, 2023
8c62ac0
remove CLOUD_STORAGE_MAX_FILES_COUNT and LOCAL_LOAD_MAX_FILES_COUNT
sizov-kirill Jun 12, 2023
b00fa74
remove unused imports
sizov-kirill Jun 12, 2023
5a548e4
remove LOCAL_LOAD_MAX_FILES_SIZE
sizov-kirill Jun 13, 2023
8f77270
Update cvat/apps/engine/serializers.py
Jun 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 1 addition & 8 deletions cvat/apps/engine/serializers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
from rest_framework import serializers, exceptions
from django.contrib.auth.models import User, Group
from django.db import transaction
from django.conf import settings

from cvat.apps.dataset_manager.formats.utils import get_label_color
from cvat.apps.engine import models
Expand Down Expand Up @@ -836,15 +835,9 @@ def validate(self, attrs):
and attrs['start_frame'] > attrs['stop_frame']:
raise serializers.ValidationError('Stop frame must be more or equal start frame')

if (
(server_files := attrs.get('server_files'))
and attrs.get('cloud_storage_id')
and sum(1 for f in server_files if not f['file'].endswith('.jsonl')) > settings.CLOUD_STORAGE_MAX_FILES_COUNT
):
raise serializers.ValidationError(f'The maximum number of the cloud storage attached files is {settings.CLOUD_STORAGE_MAX_FILES_COUNT}')

filename_pattern = attrs.get('filename_pattern')
server_files_exclude = attrs.get('server_files_exclude')
server_files = attrs.get('server_files')
sizov-kirill marked this conversation as resolved.
Show resolved Hide resolved

if filename_pattern and len(list(filter(lambda x: not x['file'].endswith('.jsonl'), server_files))):
raise serializers.ValidationError('The filename_pattern can only be used with specified manifest or without server_files')
Expand Down
15 changes: 1 addition & 14 deletions cvat/apps/engine/task.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -584,24 +584,15 @@ def _create_thread(
)
if cloud_storage_manifest_prefix:
additional_files = [os.path.join(cloud_storage_manifest_prefix, f) for f in additional_files]
if len(data['server_files']) + len(additional_files) > settings.CLOUD_STORAGE_MAX_FILES_COUNT:
raise ValidationError(
'The maximum number of the cloud storage attached files '
f'is {settings.CLOUD_STORAGE_MAX_FILES_COUNT}')
else:
number_of_files = len(data['server_files'])
while len(dirs):
directory = dirs.pop()
for f in cloud_storage_instance.list_files(prefix=directory, _use_flat_listing=True):
if f['type'] == 'REG':
additional_files.append(f['name'])
else:
dirs.append(f['name'])
# we check the limit of files on each iteration to reduce the number of possible requests to the bucket
if (len(additional_files) + len(dirs) + number_of_files) > settings.CLOUD_STORAGE_MAX_FILES_COUNT:
raise ValidationError(
'The maximum number of the cloud storage attached files '
f'is {settings.CLOUD_STORAGE_MAX_FILES_COUNT}')

data['server_files'].extend(additional_files)
del additional_files

Expand Down Expand Up @@ -641,10 +632,6 @@ def _create_thread(
if not data['filename_pattern'] == '*':
additional_files = fnmatch.filter(additional_files, data['filename_pattern'])

if (len(additional_files)) > settings.CLOUD_STORAGE_MAX_FILES_COUNT:
raise ValidationError(
'The maximum number of the cloud storage attached files '
f'is {settings.CLOUD_STORAGE_MAX_FILES_COUNT}')
data['server_files'].extend(additional_files)

if db_data.storage_method == models.StorageMethodChoice.FILE_SYSTEM or not settings.USE_CACHE:
Expand Down
2 changes: 0 additions & 2 deletions cvat/settings/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -514,9 +514,7 @@ class CVAT_QUEUES(Enum):
DATA_UPLOAD_MAX_MEMORY_SIZE = 100 * 1024 * 1024 # 100 MB
DATA_UPLOAD_MAX_NUMBER_FIELDS = None # this django check disabled
DATA_UPLOAD_MAX_NUMBER_FILES = None
LOCAL_LOAD_MAX_FILES_COUNT = 500
LOCAL_LOAD_MAX_FILES_SIZE = 512 * 1024 * 1024 # 512 MB
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What about LOCAL_LOAD_MAX_FILES_SIZE variable? It doesn't seem to be used anywhere. Should we delete it too?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed

CLOUD_STORAGE_MAX_FILES_COUNT = LOCAL_LOAD_MAX_FILES_COUNT

RESTRICTIONS = {
# allow access to analytics component to users with business role
Expand Down