Use PostgreSQL in debug mode (#5999)

Fix #5996

We have multiple problems with sqlite on different operation systems.
The idea here is to use the same DB in debug and in production.

sqlite database is locked errors:
- https://www2.sqlite.org/cvstrac/wiki?p=DatabaseIsLocked
-
https://docs.djangoproject.com/en/4.2/ref/databases/#database-is-locked-errors

---------

Co-authored-by: kirill-sizov <sizow.k.d@gmail.com>

.github/workflows/full.yml

@@ -212,7 +212,7 @@ jobs:
           HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
           CONTAINER_COVERAGE_DATA_DIR: "/coverage_data"
         run: |
-          docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server
+          docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server cvat_db
 
           max_tries=12
           while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done

.github/workflows/main.yml

@@ -210,7 +210,7 @@ jobs:
           HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
           CONTAINER_COVERAGE_DATA_DIR: "/coverage_data"
         run: |
-          docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server
+          docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server cvat_db
 
           max_tries=12
           while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done

.github/workflows/schedule.yml

@@ -181,7 +181,7 @@ jobs:
           HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
           CONTAINER_COVERAGE_DATA_DIR: "/coverage_data"
         run: |
-          docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server
+          docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server cvat_db
           max_tries=12
           while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
 

cvat/requirements/base.txt

@@ -62,3 +62,4 @@ psutil==5.9.4
 clickhouse-connect==0.5.10
 django-crum==0.7.9
 wheel>=0.38.0 # not directly required, pinned by Snyk to avoid a vulnerability
+psycopg2-binary==2.9.5

cvat/requirements/production.txt

@@ -1,3 +1,2 @@
 -r base.txt
-psycopg2-binary==2.9.5
 mod-wsgi==4.9.4

cvat/settings/base.py

@@ -646,3 +646,16 @@ class CVAT_QUEUES(Enum):
         'PASSWORD': os.getenv('CLICKHOUSE_PASSWORD', 'user'),
     }
 }
+
+# Database
+# https://docs.djangoproject.com/en/3.2/ref/settings/#databases
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql',
+        'HOST': os.getenv('CVAT_POSTGRES_HOST', 'cvat_db'),
+        'NAME': os.getenv('CVAT_POSTGRES_DBNAME', 'cvat'),
+        'USER': os.getenv('CVAT_POSTGRES_USER', 'root'),
+        'PASSWORD': os.getenv('CVAT_POSTGRES_PASSWORD', ''),
+        'PORT': os.getenv('CVAT_POSTGRES_PORT', 5432),
+    }
+}

cvat/settings/development.py

@@ -17,16 +17,6 @@
 # https://github.com/moggers87/django-sendfile2
 SENDFILE_BACKEND = 'django_sendfile.backends.development'
 
-# Database
-# https://docs.djangoproject.com/en/2.0/ref/settings/#databases
-
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
-    },
-}
-
 # Cross-Origin Resource Sharing settings for CVAT UI
 UI_SCHEME = os.environ.get('UI_SCHEME', 'http')
 UI_HOST = os.environ.get('UI_HOST', 'localhost')
@@ -67,3 +57,7 @@
 SILKY_MAX_RESPONSE_BODY_SIZE = 1024
 SILKY_IGNORE_PATHS = ['/admin', '/documentation', '/django-rq', '/auth']
 SILKY_MAX_RECORDED_REQUESTS = 10**4
+
+# Database
+# https://docs.djangoproject.com/en/3.2/ref/settings/#databases
+DATABASES['default']['HOST'] = os.getenv('CVAT_POSTGRES_HOST', 'localhost')

cvat/settings/production.py

@@ -18,17 +18,3 @@
 # Django-sendfile:
 # https://github.com/moggers87/django-sendfile2
 SENDFILE_BACKEND = 'django_sendfile.backends.xsendfile'
-
-# Database
-# https://docs.djangoproject.com/en/2.0/ref/settings/#databases
-
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.postgresql',
-        'HOST': os.getenv('CVAT_POSTGRES_HOST', 'cvat_db'),
-        'NAME': os.getenv('CVAT_POSTGRES_DBNAME', 'cvat'),
-        'USER': os.getenv('CVAT_POSTGRES_USER', 'root'),
-        'PASSWORD': os.getenv('CVAT_POSTGRES_PASSWORD', ''),
-        'PORT': os.getenv('CVAT_POSTGRES_PORT', 5432),
-    }
-}

cvat/settings/testing.py

@@ -5,6 +5,13 @@
 from .development import *
 import tempfile
 
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+    },
+}
+
 _temp_dir = tempfile.TemporaryDirectory(dir=BASE_DIR, suffix="cvat")
 BASE_DIR = _temp_dir.name
 

docker-compose.dev.yml

@@ -6,6 +6,10 @@
 #
 
 services:
+  cvat_db:
+    ports:
+      - '5432:5432'
+
   cvat_server:
     build:
       context: .

site/content/en/docs/contributing/development-environment.md

@@ -163,14 +163,24 @@ description: 'Installing a development environment for different operating syste
 
 - Install [Docker Engine](https://docs.docker.com/engine/install/ubuntu/) and [Docker-Compose](https://docs.docker.com/compose/install/)
 
-- Pull and run OpenPolicyAgent Docker image:
+- Pull and run Open Policy Agent docker image:
 
   ```bash
    docker run -d --rm --name cvat_opa_debug -p 8181:8181 openpolicyagent/opa:0.45.0-rootless \
    run --server --set=decision_logs.console=true --set=services.cvat.url=http://host.docker.internal:7000 \
    --set=bundles.cvat.service=cvat --set=bundles.cvat.resource=/api/auth/rules
   ```
 
+- Pull and run PostgreSQL docker image:
+
+  ```bash
+  docker run --name cvat_db_debug -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_USER=root \
+  -e POSTGRES_DB=cvat -p 5432:5432 -d postgres
+  ```
+
+  Note: use `docker start/stop cvat_db_debug` commands to start and stop the container.
+  If it is removed, data will be removed together with the container.
+
 ### Run CVAT
 - Start npm UI debug server (run the following command from CVAT root directory):
   - If you want to run CVAT in localhost:

site/content/en/docs/contributing/running-tests.md

@@ -130,22 +130,54 @@ Extra options:
    ```
    yarn --frozen-lockfile
    ```
-1. Run CVAT instance
+1. Build CVAT server image
    ```
-   docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d
+   docker compose -f docker-compose.yml -f docker-compose.dev.yml build cvat_server
+   ```
+1. Run cvat_opa container
+   ```
+   docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa
    ```
 
 **Running tests**
 1. Python tests
    ```
-   python manage.py test --settings cvat.settings.testing cvat/apps
+   python manage.py test --settings cvat.settings.testing cvat/apps -v 2
    ```
 1. JS tests
    ```
    cd cvat-core
    yarn run test
    ```
 
+**Debug python unit tests**
+1. Run `server: tests` debug task in VSCode
+1. If you want to debug particular tests then change the configuration
+of the corresponding task in `./vscode/launch.json`, for example:
+   ```json
+   {
+       "name": "server: tests",
+       "type": "python",
+       "request": "launch",
+       "justMyCode": false,
+       "stopOnEntry": false,
+       "python": "${command:python.interpreterPath}",
+       "program": "${workspaceRoot}/manage.py",
+       "args": [
+           "test",
+           "--settings",
+           "cvat.settings.testing",
+           "cvat/apps/engine",
+           "-v", "2",
+           "-k", "test_api_v2_projects_",
+       ],
+       "django": true,
+       "cwd": "${workspaceFolder}",
+       "env": {},
+       "console": "internalConsole"
+   }
+   ```
+
 
 <a id="opa-tests"></a>
 ## IAM and Open Policy Agent tests