Code coverage

.coveragerc

@@ -1,6 +1,5 @@
 [run]
 branch = true
-# relative_files = true # does not work?
 
 source =
     cvat/apps/

.github/codecov.yml

@@ -0,0 +1,20 @@
+comment:
+  layout: "header, diff, components"
+
+component_management:
+  individual_components:
+    - component_id: cvat-ui
+      name: cvat-ui
+      paths:
+        - cvat-canvas/**
+        - cvat-core/**
+        - cvat-data/**
+        - cvat-ui/**
+    - component_id: cvat-server
+      name: cvat-server
+      paths:
+        - cvat/**
+        - cvat-cli/**
+        - cvat-sdk/**
+        - utils/**
+

.github/workflows/full.yml

@@ -52,12 +52,19 @@ jobs:
       - name: CVAT server. Build and push
         uses: docker/build-push-action@v3
         with:
+          build-args: |
+              "COVERAGE_PROCESS_START=.coveragerc"
           cache-from: type=local,src=/tmp/cvat_cache_server
           context: .
           file: Dockerfile
           tags: cvat/server
           outputs: type=docker,dest=/tmp/cvat_server/image.tar
 
+      - name: Instrumentation of the code then rebuilding the CVAT UI
+        run: |
+          yarn --frozen-lockfile
+          yarn run coverage
+
       - name: CVAT UI. Build and push
         uses: docker/build-push-action@v3
         with:
@@ -154,11 +161,20 @@ jobs:
 
       - name: Running REST API and SDK tests
         id: run_tests
+        env:
+          COVERAGE_PROCESS_START: ".coveragerc"
         run: |
-          pip3 install --user '/tmp/cvat_sdk/[pytorch]'
-          pip3 install --user cvat-cli/
-          pip3 install --user -r tests/python/requirements.txt
-          pytest tests/python/ -s -v
+          pip3 install -r cvat-sdk/gen/requirements.txt
+          ./cvat-sdk/gen/generate.sh
+
+          pip3 install -r ./tests/python/requirements.txt
+          pip3 install -e ./cvat-sdk
+          pip3 install -e ./cvat-cli
+
+          pytest tests/python/ --cov --cov-report xml
+
+      - name: Upload coverage reports to Codecov with GitHub Action
+        uses: codecov/codecov-action@v3
 
       - name: Creating a log file from cvat containers
         if: failure() && steps.run_tests.conclusion == 'failure'
@@ -218,10 +234,13 @@ jobs:
           while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
 
           docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
-            -c 'python manage.py test cvat/apps -v 2'
+            -c 'coverage run -a manage.py test cvat/apps && coverage json && mv coverage.json ${CONTAINER_COVERAGE_DATA_DIR}'
 
           docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
-            -c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test'
+            -c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test && mv cvat-core/reports/coverage/coverage-final.json ${CONTAINER_COVERAGE_DATA_DIR}'
+
+      - name: Upload coverage reports to Codecov with GitHub Action
+        uses: codecov/codecov-action@v3
 
       - name: Creating a log file from cvat containers
         if: failure()
@@ -317,16 +336,17 @@ jobs:
             npx cypress run \
               --headed \
               --browser chrome \
-              --env coverage=false \
               --config-file cypress_canvas3d.config.js \
               --spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
           else
             npx cypress run \
               --browser chrome \
-              --env coverage=false \
               --spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
           fi
 
+      - name: Upload coverage reports to Codecov with GitHub Action
+        uses: codecov/codecov-action@v3
+
       - name: Creating a log file from "cvat" container logs
         if: failure()
         run: |

.github/workflows/main.yml

@@ -52,12 +52,19 @@ jobs:
       - name: CVAT server. Build and push
         uses: docker/build-push-action@v3
         with:
+          build-args: |
+              "COVERAGE_PROCESS_START=.coveragerc"
           cache-from: type=local,src=/tmp/cvat_cache_server
           context: .
           file: Dockerfile
           tags: cvat/server
           outputs: type=docker,dest=/tmp/cvat_server/image.tar
 
+      - name: Instrumentation of the code then rebuilding the CVAT UI
+        run: |
+          yarn --frozen-lockfile
+          yarn run coverage
+
       - name: CVAT UI. Build and push
         uses: docker/build-push-action@v3
         with:
@@ -130,12 +137,6 @@ jobs:
           name: cvat_ui
           path: /tmp/cvat_ui/
 
-      - name: Download CVAT SDK package
-        uses: actions/download-artifact@v3
-        with:
-          name: cvat_sdk
-          path: /tmp/cvat_sdk/
-
       - name: Load Docker images
         run: |
           docker load --input /tmp/cvat_server/image.tar
@@ -144,7 +145,6 @@ jobs:
           docker tag cvat/ui:latest cvat/ui:${CVAT_VERSION}
           docker image ls -a
 
-
       - name: Upload expected schema as an artifact
         if: failure() && steps.verify_schema.conclusion == 'failure'
         uses: actions/upload-artifact@v3.1.2
@@ -154,11 +154,20 @@ jobs:
 
       - name: Running REST API and SDK tests
         id: run_tests
+        env:
+          COVERAGE_PROCESS_START: ".coveragerc"
         run: |
-          pip3 install --user '/tmp/cvat_sdk/[pytorch]'
-          pip3 install --user cvat-cli/
-          pip3 install --user -r tests/python/requirements.txt
-          pytest tests/python/ -s -v
+          pip3 install -r cvat-sdk/gen/requirements.txt
+          ./cvat-sdk/gen/generate.sh
+
+          pip3 install -r ./tests/python/requirements.txt
+          pip3 install -e ./cvat-sdk
+          pip3 install -e ./cvat-cli
+
+          pytest tests/python/ --cov --cov-report xml
+
+      - name: Upload coverage reports to Codecov with GitHub Action
+        uses: codecov/codecov-action@v3
 
       - name: Creating a log file from cvat containers
         if: failure() && steps.run_tests.conclusion == 'failure'
@@ -216,10 +225,13 @@ jobs:
           while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
 
           docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
-            -c 'python manage.py test cvat/apps -v 2'
+            -c 'coverage run -a manage.py test cvat/apps && coverage json && mv coverage.json ${CONTAINER_COVERAGE_DATA_DIR}'
 
           docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
-            -c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test'
+            -c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test && mv cvat-core/reports/coverage/coverage-final.json ${CONTAINER_COVERAGE_DATA_DIR}'
+
+      - name: Upload coverage reports to Codecov with GitHub Action
+        uses: codecov/codecov-action@v3
 
       - name: Creating a log file from cvat containers
         if: failure()
@@ -313,16 +325,17 @@ jobs:
             npx cypress run \
               --headed \
               --browser chrome \
-              --env coverage=false \
               --config-file cypress_canvas3d.config.js \
               --spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
           else
             npx cypress run \
               --browser chrome \
-              --env coverage=false \
               --spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
           fi
 
+      - name: Upload coverage reports to Codecov with GitHub Action
+        uses: codecov/codecov-action@v3
+
       - name: Creating a log file from "cvat" container logs
         if: failure()
         run: |

.github/workflows/schedule.yml

@@ -88,6 +88,8 @@ jobs:
       - name: CVAT server. Build and push
         uses: docker/build-push-action@v3
         with:
+          build-args: |
+              "COVERAGE_PROCESS_START=.coveragerc"
           cache-from: type=local,src=/tmp/cvat_cache_server
           context: .
           file: Dockerfile
@@ -163,14 +165,17 @@ jobs:
           ./opa test cvat/apps/iam/rules
 
       - name: REST API and SDK tests
+        env:
+          COVERAGE_PROCESS_START: ".coveragerc"
         run: |
-          pip3 install --user -r cvat-sdk/gen/requirements.txt
+          pip3 install -r cvat-sdk/gen/requirements.txt
           ./cvat-sdk/gen/generate.sh
 
-          pip3 install --user 'cvat-sdk/[pytorch]'
-          pip3 install --user cvat-cli/
-          pip3 install --user -r tests/python/requirements.txt
-          pytest tests/python/
+          pip3 install -r ./tests/python/requirements.txt
+          pip3 install -e ./cvat-sdk
+          pip3 install -e ./cvat-cli
+
+          pytest tests/python/ --cov --cov-report xml
           pytest tests/python/ --stop-services
 
       - name: Unit tests
@@ -183,20 +188,15 @@ jobs:
           while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
 
           docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
-          -c 'coverage run -a manage.py test cvat/apps && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
+            -c 'coverage run -a manage.py test cvat/apps && coverage json && mv coverage.json ${CONTAINER_COVERAGE_DATA_DIR}'
 
           docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
-          -c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test'
+            -c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test && mv cvat-core/reports/coverage/coverage-final.json ${CONTAINER_COVERAGE_DATA_DIR}'
 
           docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml down -v
 
-      - name: Uploading code coverage results as an artifact
-        uses: actions/upload-artifact@v3.1.1
-        with:
-          name: coverage_results
-          path: |
-            ${{ github.workspace }}/lcov.info
-            ${{ github.workspace }}/.coverage
+      - name: Upload coverage reports to Codecov with GitHub Action
+        uses: codecov/codecov-action@v3
 
   e2e_testing:
     needs: build
@@ -304,14 +304,15 @@ jobs:
               --browser chrome \
               --config-file cypress_canvas3d.config.js \
               --spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
-            mv ./.nyc_output/out.json ./.nyc_output/out_${{ matrix.specs }}.json
           else
             npx cypress run \
               --browser chrome \
               --spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
-            mv ./.nyc_output/out.json ./.nyc_output/out_${{ matrix.specs }}.json
           fi
 
+      - name: Upload coverage reports to Codecov with GitHub Action
+        uses: codecov/codecov-action@v3
+
       - name: Creating a log file from "cvat" container logs
         if: failure()
         run: |
@@ -330,72 +331,3 @@ jobs:
         with:
           name: cvat_container_logs
           path: ${{ github.workspace }}/tests/cvat.log
-
-      - name: Uploading code coverage results as an artifact
-        uses: actions/upload-artifact@v3.1.1
-        with:
-          name: coverage_results
-          path: ${{ github.workspace }}/tests/.nyc_output
-
-  coveralls:
-    runs-on: ubuntu-latest
-    needs: [unit_testing, e2e_testing]
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: CVAT server. Extract metadata (tags, labels) for Docker
-        id: meta-server
-        uses: docker/metadata-action@master
-        with:
-          images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.SERVER_IMAGE_TEST_REPO }}
-          tags:
-            type=raw,value=nightly
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_CI_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_CI_TOKEN }}
-
-      - name: Pull CVAT server image
-        run: |
-          docker pull ${{ steps.meta-server.outputs.tags }}
-          docker tag ${{ steps.meta-server.outputs.tags }} cvat/server:local
-
-      - name: Downloading coverage results
-        uses: actions/download-artifact@v2
-        with:
-          name: coverage_results
-
-      - name: Combining coverage results
-        run: |
-          mkdir -p ./nyc_output_tmp
-          mv ./out_*.json ./nyc_output_tmp
-          mkdir -p ./.nyc_output
-          yarn --frozen-lockfile
-          npx nyc merge ./nyc_output_tmp ./.nyc_output/out.json
-
-      - name: Sending results to Coveralls
-        env:
-          HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
-          CONTAINER_COVERAGE_DATA_DIR: "/coverage_data"
-          COVERALLS_SERVICE_NAME: github
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-            npx nyc report --reporter=text-lcov >> ${HOST_COVERAGE_DATA_DIR}/lcov.info
-
-            docker compose \
-              -f docker-compose.yml \
-              -f docker-compose.dev.yml \
-              -f docker-compose.ci.yml \
-              run cvat_ci /bin/bash -c 'cd ${CONTAINER_COVERAGE_DATA_DIR} && coveralls-lcov -v -n lcov.info > ${CONTAINER_COVERAGE_DATA_DIR}/coverage.json'
-
-            docker compose \
-              -f docker-compose.yml \
-              -f docker-compose.dev.yml \
-              -f docker-compose.ci.yml \
-              run cvat_ci /bin/bash -c '\
-                ln -s ${CONTAINER_COVERAGE_DATA_DIR}/.git . \
-                && ln -s ${CONTAINER_COVERAGE_DATA_DIR}/.coverage . \
-                && ln -s ${CONTAINER_COVERAGE_DATA_DIR}/coverage.json . \
-                && coveralls --merge=coverage.json'

Dockerfile

@@ -177,6 +177,12 @@ COPY --chown=${USER} ssh ${HOME}/.ssh
 COPY --chown=${USER} wait-for-it.sh manage.py backend_entrypoint.sh ${HOME}/
 COPY --chown=${USER} utils/ ${HOME}/utils
 COPY --chown=${USER} cvat/ ${HOME}/cvat
+COPY --chown=${USER} tests/python/.coveragerc ${HOME}/
+
+ARG COVERAGE_PROCESS_START
+RUN if [ "${COVERAGE_PROCESS_START}" ]; then \
+        echo "import coverage; coverage.process_startup()" > /opt/venv/lib/python3.10/site-packages/coverage_subprocess.pth; \
+    fi
 
 # RUN all commands below as 'django' user
 USER ${USER}

Dockerfile.ci

@@ -25,9 +25,7 @@ RUN apt-get update && \
 COPY cvat/requirements/ /tmp/cvat/requirements/
 COPY utils/dataset_manifest/requirements.txt /tmp/utils/dataset_manifest/requirements.txt
 
-RUN DATUMARO_HEADLESS=1 python3 -m pip install --no-cache-dir -r /tmp/cvat/requirements/${DJANGO_CONFIGURATION}.txt && \
-    python3 -m pip install --no-cache-dir coveralls
-RUN gem install coveralls-lcov
+RUN DATUMARO_HEADLESS=1 python3 -m pip install --no-cache-dir -r /tmp/cvat/requirements/${DJANGO_CONFIGURATION}.txt
 
 COPY cvat-core ${HOME}/cvat-core
 COPY cvat-data ${HOME}/cvat-data

README.md

@@ -256,8 +256,8 @@ questions and get our support.
 [ci-url]: https://github.com/opencv/cvat/actions
 [gitter-img]: https://img.shields.io/gitter/room/opencv-cvat/public?style=flat
 [gitter-url]: https://gitter.im/opencv-cvat
-[coverage-img]: https://coveralls.io/repos/github/cvat-ai/cvat/badge.svg?branch=develop
-[coverage-url]: https://coveralls.io/github/cvat-ai/cvat?branch=develop
+[coverage-img]: https://codecov.io/github/opencv/cvat/branch/develop/graph/badge.svg
+[coverage-url]: https://codecov.io/github/opencv/cvat
 [doi-img]: https://zenodo.org/badge/139156354.svg
 [doi-url]: https://zenodo.org/badge/latestdoi/139156354
 [discord-img]: https://img.shields.io/discord/1000789942802337834?label=discord