diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index 79dc6949944..e317bac7cdc 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -35,24 +35,6 @@ jobs: ${{ runner.os }}-build-ui-${{ needs.get-sha.outputs.sha }} ${{ runner.os }}-build-ui- - - uses: actions/cache@v3 - id: elasticsearch-cache-action - with: - path: /tmp/cvat_cache_elasticsearch - key: ${{ runner.os }}-build-elasticsearch-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-build-elasticsearch-${{ needs.get-sha.outputs.sha }} - ${{ runner.os }}-build-elasticsearch- - - - uses: actions/cache@v3 - id: logstash-cache-action - with: - path: /tmp/cvat_cache_logstash - key: ${{ runner.os }}-build-logstash-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-build-logstash-${{ needs.get-sha.outputs.sha }} - ${{ runner.os }}-build-logstash- - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -72,24 +54,6 @@ jobs: cache-from: type=local,src=/tmp/cvat_cache_ui cache-to: type=local,dest=/tmp/cvat_cache_ui-new - - name: Caching CVAT Elasticsearch - uses: docker/build-push-action@v2 - with: - context: ./components/analytics/elasticsearch/ - file: ./components/analytics/elasticsearch/Dockerfile - cache-from: type=local,src=/tmp/cvat_cache_elasticsearch - cache-to: type=local,dest=/tmp/cvat_cache_elasticsearch-new - build-args: ELK_VERSION=6.8.23 - - - name: Caching CVAT Logstash - uses: docker/build-push-action@v2 - with: - context: ./components/analytics/logstash/ - file: ./components/analytics/logstash/Dockerfile - cache-from: type=local,src=/tmp/cvat_cache_logstash - cache-to: type=local,dest=/tmp/cvat_cache_logstash-new - build-args: ELK_VERSION=6.8.23 - - name: Moving cache run: | rm -rf /tmp/cvat_cache_server @@ -97,9 +61,3 @@ jobs: rm -rf /tmp/cvat_cache_ui mv /tmp/cvat_cache_ui-new /tmp/cvat_cache_ui - - rm -rf /tmp/cvat_cache_elasticsearch - mv /tmp/cvat_cache_elasticsearch-new /tmp/cvat_cache_elasticsearch - - rm -rf /tmp/cvat_cache_logstash - mv /tmp/cvat_cache_logstash-new /tmp/cvat_cache_logstash diff --git a/.github/workflows/full.yml b/.github/workflows/full.yml index b6fd0362609..ca53141c24a 100644 --- a/.github/workflows/full.yml +++ b/.github/workflows/full.yml @@ -115,38 +115,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@master - - name: Getting CVAT Elasticsearch cache from the default branch - uses: actions/cache@v3 - with: - path: /tmp/cvat_cache_elasticsearch - key: ${{ runner.os }}-build-elasticsearch-${{ needs.search_cache.outputs.sha }} - - - name: Getting CVAT Logstash cache from the default branch - uses: actions/cache@v3 - with: - path: /tmp/cvat_cache_logstash - key: ${{ runner.os }}-build-logstash-${{ needs.search_cache.outputs.sha }} - - - name: Building CVAT Elasticsearch - uses: docker/build-push-action@v2 - with: - context: ./components/analytics/elasticsearch/ - file: ./components/analytics/elasticsearch/Dockerfile - cache-from: type=local,src=/tmp/cvat_cache_elasticsearch - tags: cvat_elasticsearch:latest - load: true - build-args: ELK_VERSION=6.8.23 - - - name: Building CVAT Logstash - uses: docker/build-push-action@v2 - with: - context: ./components/analytics/logstash/ - file: ./components/analytics/logstash/Dockerfile - cache-from: type=local,src=/tmp/cvat_cache_logstash - tags: cvat_logstash:latest - load: true - build-args: ELK_VERSION=6.8.23 - - name: Download CVAT server image uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml index 0bb46cced85..fbee3918a70 100644 --- a/.github/workflows/schedule.yml +++ b/.github/workflows/schedule.yml @@ -129,18 +129,6 @@ jobs: path: /tmp/cvat_cache_ui key: ${{ runner.os }}-build-ui-${{ needs.search_cache.outputs.sha }} - - name: Getting CVAT Logstash cache from the default branch - uses: actions/cache@v3 - with: - path: /tmp/cvat_cache_logstash - key: ${{ runner.os }}-build-logstash-${{ needs.search_cache.outputs.sha }} - - - name: Getting CVAT Elasticsearch cache from the default branch - uses: actions/cache@v3 - with: - path: /tmp/cvat_cache_elasticsearch - key: ${{ runner.os }}-build-elasticsearch-${{ needs.search_cache.outputs.sha }} - - name: Building CVAT UI image uses: docker/build-push-action@v2 with: @@ -150,26 +138,6 @@ jobs: tags: cvat/ui:latest load: true - - name: Building CVAT Logstash image - uses: docker/build-push-action@v2 - with: - context: ./components/analytics/logstash/ - file: ./components/analytics/logstash/Dockerfile - cache-from: type=local,src=/tmp/cvat_cache_logstash - build-args: ELK_VERSION=6.8.23 - tags: cvat_logstash - load: true - - - name: Building CVAT Elasticsearch image - uses: docker/build-push-action@v2 - with: - context: ./components/analytics/elasticsearch/ - file: ./components/analytics/elasticsearch/Dockerfile - cache-from: type=local,src=/tmp/cvat_cache_elasticsearch - build-args: ELK_VERSION=6.8.23 - tags: cvat_elasticsearch - load: true - - name: CVAT server. Extract metadata (tags, labels) for Docker id: meta-server uses: docker/metadata-action@master diff --git a/.vscode/launch.json b/.vscode/launch.json index 03216ea1b95..501b1bf3075 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -107,7 +107,9 @@ "env": { "CVAT_SERVERLESS": "1", "ALLOWED_HOSTS": "*", - "IAM_OPA_BUNDLE": "1" + "IAM_OPA_BUNDLE": "1", + "DJANGO_LOG_SERVER_HOST": "localhost", + "DJANGO_LOG_SERVER_PORT": "8282" }, "args": [ "runserver", @@ -144,11 +146,14 @@ "rqworker", "import", "--worker-class", - "cvat.rqworker.SimpleWorker", + "cvat.rqworker.SimpleWorker" ], "django": true, "cwd": "${workspaceFolder}", - "env": {}, + "env": { + "DJANGO_LOG_SERVER_HOST": "localhost", + "DJANGO_LOG_SERVER_PORT": "8282" + }, "console": "internalConsole" }, { @@ -167,7 +172,10 @@ ], "django": true, "cwd": "${workspaceFolder}", - "env": {}, + "env": { + "DJANGO_LOG_SERVER_HOST": "localhost", + "DJANGO_LOG_SERVER_PORT": "8282" + }, "console": "internalConsole" }, { diff --git a/CHANGELOG.md b/CHANGELOG.md index 66844e5854d..3994c8e5759 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ Tracks can be exported/imported to/from Datumaro and Sly Pointcloud formats () - \[Server API\] Simple filters for object collection endpoints () +- Analytics based on Clickhouse, Vector and Grafana instead of the ELK stack () ### Changed - The Docker Compose files now use the Compose Specification version diff --git a/components/analytics/clickhouse/init.sh b/components/analytics/clickhouse/init.sh new file mode 100755 index 00000000000..81c9ae19b58 --- /dev/null +++ b/components/analytics/clickhouse/init.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +CLICKHOUSE_DB="${CLICKHOUSE_DB:-cvat}"; +CLICKHOUSE_USER="${CLICKHOUSE_USER:-user}"; +CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-password}"; + +cat < /etc/clickhouse-server/users.d/user.xml + + + + <${CLICKHOUSE_USER}> + default + + ::/0 + + ${CLICKHOUSE_PASSWORD} + default + + + +EOT + +clickhouse-client --query "CREATE DATABASE IF NOT EXISTS ${CLICKHOUSE_DB}"; + +echo " +CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_DB}.events +( + \`scope\` String NOT NULL, + \`obj_name\` String NULL, + \`obj_id\` UInt64 NULL, + \`obj_val\` String NULL, + \`source\` String NOT NULL, + \`timestamp\` DateTime64(3, 'Etc/UTC') NOT NULL, + \`count\` UInt16 NULL, + \`duration\` UInt32 DEFAULT toUInt32(0), + \`project_id\` UInt64 NULL, + \`task_id\` UInt64 NULL, + \`job_id\` UInt64 NULL, + \`user_id\` UInt64 NULL, + \`user_name\` String NULL, + \`user_email\` String NULL, + \`org_id\` UInt64 NULL, + \`org_slug\` String NULL, + \`payload\` String NULL +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(timestamp) +ORDER BY (timestamp) +SETTINGS index_granularity = 8192 +;" | clickhouse-client diff --git a/components/analytics/docker-compose.analytics.yml b/components/analytics/docker-compose.analytics.yml deleted file mode 100644 index 1cb0aa3cd28..00000000000 --- a/components/analytics/docker-compose.analytics.yml +++ /dev/null @@ -1,91 +0,0 @@ -services: - elasticsearch: - container_name: cvat_elasticsearch - image: cvat_elasticsearch - networks: - - cvat - build: - context: ./components/analytics/elasticsearch - args: - ELK_VERSION: 6.8.23 - volumes: - - cvat_events:/usr/share/elasticsearch/data - restart: always - - kibana: - container_name: cvat_kibana - image: cvat_kibana - networks: - - cvat - build: - context: ./components/analytics/kibana - args: - ELK_VERSION: 6.8.23 - depends_on: ['elasticsearch'] - environment: - - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 - restart: always - - cvat_kibana_setup: - container_name: cvat_kibana_setup - image: cvat/server:${CVAT_VERSION:-dev} - volumes: ['./components/analytics/kibana:/home/django/kibana:ro'] - depends_on: ['cvat_server'] - working_dir: '/home/django' - networks: - - cvat - entrypoint: - [ - 'bash', - 'wait-for-it.sh', - 'elasticsearch:9200', - '-t', - '0', - '--', - '/bin/bash', - 'wait-for-it.sh', - 'kibana:5601', - '-t', - '0', - '--', - 'python3', - 'kibana/setup.py', - 'kibana/export.json', - ] - environment: - no_proxy: elasticsearch,kibana,${no_proxy:-} - - logstash: - container_name: cvat_logstash - image: cvat_logstash - networks: - - cvat - build: - context: ./components/analytics/logstash - args: - ELK_VERSION: 6.8.23 - http_proxy: ${http_proxy:-} - https_proxy: ${https_proxy:-} - environment: - LOGSTASH_OUTPUT_HOST: elasticsearch:9200 - LOGSTASH_OUTPUT_USER: - LOGSTASH_OUTPUT_PASS: - depends_on: ['elasticsearch'] - restart: always - - cvat_server: - environment: - DJANGO_LOG_SERVER_HOST: logstash - DJANGO_LOG_SERVER_PORT: 8080 - CVAT_ANALYTICS: 1 - - traefik: - environment: - CVAT_HOST: ${CVAT_HOST:-localhost} - DJANGO_LOG_VIEWER_HOST: kibana - DJANGO_LOG_VIEWER_PORT: 5601 - volumes: - - ./components/analytics/kibana_conf.yml:/etc/traefik/rules/kibana_conf.yml:ro - -volumes: - cvat_events: diff --git a/components/analytics/elasticsearch/Dockerfile b/components/analytics/elasticsearch/Dockerfile deleted file mode 100644 index 2c2704263a7..00000000000 --- a/components/analytics/elasticsearch/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -ARG ELK_VERSION -FROM docker.elastic.co/elasticsearch/elasticsearch-oss:${ELK_VERSION} -COPY --chown=elasticsearch:elasticsearch elasticsearch.yml /usr/share/elasticsearch/config/ - diff --git a/components/analytics/elasticsearch/elasticsearch.yml b/components/analytics/elasticsearch/elasticsearch.yml deleted file mode 100644 index 2d19b17ffab..00000000000 --- a/components/analytics/elasticsearch/elasticsearch.yml +++ /dev/null @@ -1,3 +0,0 @@ -http.host: 0.0.0.0 -script.painless.regex.enabled: true -path.repo: ['/usr/share/elasticsearch/data/backup'] diff --git a/components/analytics/grafana/dashboards/all_events.json b/components/analytics/grafana/dashboards/all_events.json new file mode 100644 index 00000000000..9bb340b8bd0 --- /dev/null +++ b/components/analytics/grafana/dashboards/all_events.json @@ -0,0 +1,523 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 90, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "builderOptions": { + "fields": [], + "filters": [], + "metrics": [ + { + "aggregation": "count", + "field": "*" + } + ], + "mode": "trend", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 0, + "meta": { + "builderOptions": { + "fields": [], + "filters": [], + "metrics": [ + { + "aggregation": "count", + "field": "*" + } + ], + "mode": "trend", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + } + }, + "queryType": "sql", + "rawSql": "SELECT $__timeInterval(timestamp) as time, count(*)\r\nFROM events\r\nWHERE $__timeFilter(timestamp)\r\nAND scope IN (${scopes})\r\nAND source IN (${sources})\r\nAND (-1 IN (${users}) OR user_id IN (${users}))\r\nAND (-1 IN (${organizations}) OR org_id IN (${organizations}))\r\nAND (-1 IN (${projects}) OR project_id IN (${projects}))\r\nAND (-1 IN (${tasks}) OR task_id IN (${tasks}))\r\nAND (-1 IN (${jobs}) OR job_id IN (${jobs}))\r\nGROUP BY time ORDER BY time ASC", + "refId": "A" + } + ], + "title": "Overall Activity", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 158 + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 4, + "options": { + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "builderOptions": { + "fields": [ + "*" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime64(3, 'Etc/UTC')", + "value": "TODAY" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "scope", + "operator": "IN", + "type": "String", + "value": [ + "" + ] + } + ], + "mode": "list", + "orderBy": [ + { + "dir": "ASC", + "name": "timestamp" + } + ], + "table": "events" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [ + "*" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime64(3, 'Etc/UTC')", + "value": "TODAY" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "scope", + "operator": "IN", + "type": "String", + "value": [ + "" + ] + } + ], + "mode": "list", + "orderBy": [ + { + "dir": "ASC", + "name": "timestamp" + } + ], + "table": "events" + } + }, + "queryType": "sql", + "rawSql": "SELECT * \r\nFROM events \r\nWHERE $__timeFilter(timestamp)\r\nORDER BY timestamp DESC", + "refId": "A" + } + ], + "title": "All events", + "type": "table" + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT scope\nFROM events\nWHERE $__timeFilter(timestamp)", + "hide": 0, + "includeAll": true, + "label": "Scope", + "multi": true, + "name": "scopes", + "options": [], + "query": "SELECT scope\nFROM events\nWHERE $__timeFilter(timestamp)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT source\nFROM events\nWHERE $__timeFilter(timestamp)", + "hide": 0, + "includeAll": true, + "label": "Source", + "multi": true, + "name": "sources", + "options": [], + "query": "SELECT source\nFROM events\nWHERE $__timeFilter(timestamp)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT user_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND user_id IS NOT NULL", + "hide": 0, + "includeAll": true, + "label": "User", + "multi": true, + "name": "users", + "options": [], + "query": "SELECT user_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND user_id IS NOT NULL", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT project_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND project_id IS NOT NULL", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "projects", + "options": [], + "query": "SELECT project_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND project_id IS NOT NULL", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT task_id\nFROM events\nWHERE $__timeFilter(timestamp) \n AND task_id IS NOT NULL", + "hide": 0, + "includeAll": true, + "label": "Task", + "multi": true, + "name": "tasks", + "options": [], + "query": "SELECT task_id\nFROM events\nWHERE $__timeFilter(timestamp) \n AND task_id IS NOT NULL", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT job_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND job_id IS NOT NULL", + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "jobs", + "options": [], + "query": "SELECT job_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND job_id IS NOT NULL", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT org_id\nFROM events\nWHERE $__timeFilter(timestamp)\nAND org_id IS NOT NULL", + "hide": 0, + "includeAll": true, + "label": "Organization", + "multi": true, + "name": "organizations", + "options": [], + "query": "SELECT org_id\nFROM events\nWHERE $__timeFilter(timestamp)\nAND org_id IS NOT NULL", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "All events", + "uid": "EIGSTDAVz", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/components/analytics/grafana/dashboards/management.json b/components/analytics/grafana/dashboards/management.json new file mode 100644 index 00000000000..94c60707a75 --- /dev/null +++ b/components/analytics/grafana/dashboards/management.json @@ -0,0 +1,509 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "User 1" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "builderOptions": { + "fields": [], + "filters": [], + "groupBy": [ + "user_id" + ], + "limit": null, + "metrics": [ + { + "aggregation": "count", + "alias": "value", + "field": "*" + } + ], + "mode": "trend", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 0, + "hide": false, + "meta": { + "builderOptions": { + "fields": [], + "filters": [], + "groupBy": [ + "user_id" + ], + "limit": null, + "metrics": [ + { + "aggregation": "count", + "alias": "value", + "field": "*" + } + ], + "mode": "trend", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + } + }, + "queryType": "sql", + "rawSql": "SELECT $__timeInterval(timestamp) as time, toString(user_id), count() as User\r\nFROM events\r\nWHERE $__timeFilter(timestamp)\r\nGROUP BY time, user_id\r\nORDER BY time ASC, user_id ASC", + "refId": "A" + } + ], + "title": "User Activity", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 90, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "builderOptions": { + "fields": [], + "filters": [], + "limit": 100, + "metrics": [ + { + "aggregation": "count", + "alias": "Count", + "field": "*" + } + ], + "mode": "trend", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 0, + "queryType": "builder", + "rawSql": "SELECT $__timeInterval(timestamp) as time, count(*) Count FROM events WHERE $__timeFilter(timestamp) GROUP BY time ORDER BY time ASC LIMIT 100", + "refId": "A" + } + ], + "title": "Overall Activity", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": false, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "footer": { + "enablePagination": true, + "fields": [ + "Working time(h)", + "Activity" + ], + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "SELECT\r\n user_id as User,\r\n project_id as Project,\r\n task_id as Task,\r\n job_id as Job, sum(JSONExtractUInt(payload, 'working_time')) / 1000 / 3600 as \"Working time(h)\",\r\n count() as Activity\r\nFROM events\r\nWHERE JSONHas(payload, 'working_time')\r\n AND $__timeFilter(timestamp)\r\n AND user_id IN (${users})\r\n AND (-1 IN (${projects}) OR project_id IN (${projects}))\r\n AND task_id IN (${tasks})\r\n AND job_id IN (${jobs})\r\n AND source = 'client'\r\nGROUP BY user_id, project_id, task_id, job_id", + "refId": "A" + } + ], + "title": "Working time", + "type": "table" + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT user_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND source = 'client'", + "hide": 0, + "includeAll": true, + "label": "User", + "multi": true, + "name": "users", + "options": [], + "query": "SELECT user_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND source = 'client'", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT project_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND project_id IS NOT NULL\n AND source = 'client'", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "projects", + "options": [], + "query": "SELECT project_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND project_id IS NOT NULL\n AND source = 'client'", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT task_id\nFROM events\nWHERE $__timeFilter(timestamp) \n AND task_id IS NOT NULL\n AND source = 'client'\n AND (-1 IN (${projects}) OR project_id IN (${projects}))", + "description": "", + "hide": 0, + "includeAll": true, + "label": "Task", + "multi": true, + "name": "tasks", + "options": [], + "query": "SELECT task_id\nFROM events\nWHERE $__timeFilter(timestamp) \n AND task_id IS NOT NULL\n AND source = 'client'\n AND (-1 IN (${projects}) OR project_id IN (${projects}))", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT job_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND job_id IS NOT NULL\n AND source = 'client'\n AND task_id in (${tasks})", + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "jobs", + "options": [], + "query": "SELECT job_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND job_id IS NOT NULL\n AND source = 'client'\n AND task_id in (${tasks})", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Management", + "uid": "w0if6WAVz", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/components/analytics/grafana/dashboards/monitoring.json b/components/analytics/grafana/dashboards/monitoring.json new file mode 100644 index 00000000000..55003970d15 --- /dev/null +++ b/components/analytics/grafana/dashboards/monitoring.json @@ -0,0 +1,902 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "builderOptions": { + "fields": [], + "filters": [], + "limit": 100, + "metrics": [ + { + "aggregation": "count", + "field": "" + } + ], + "mode": "aggregate", + "orderBy": [], + "table": "events" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "filters": [], + "limit": 100, + "metrics": [ + { + "aggregation": "count", + "field": "" + } + ], + "mode": "aggregate", + "orderBy": [], + "table": "events" + } + }, + "queryType": "sql", + "rawSql": "SELECT\r\n uniqExact(user_id) as \"Active users (now)\"\r\nFROM\r\n cvat.events\r\nWHERE\r\n user_id IS NOT NULL AND\r\n timestamp >= now() - INTERVAL 10 MINUTE", + "refId": "A" + } + ], + "title": "Active users (now)", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "Show active users calculated by 15 minutes time interval", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 21, + "x": 3, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "SELECT\r\n time,\r\n uniqExact(user_id) Users\r\nFROM\r\n(\r\n SELECT\r\n user_id,\r\n toStartOfInterval(timestamp, INTERVAL 15 minute) as time\r\n FROM cvat.events\r\n WHERE\r\n user_id IS NOT NULL\r\n GROUP BY\r\n user_id,\r\n time\r\n ORDER BY time ASC WITH FILL STEP toIntervalMinute(15)\r\n)\r\nGROUP BY time\r\nORDER BY time", + "refId": "A" + } + ], + "title": "Active users", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 2, + "options": { + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "builderOptions": { + "database": "cvat", + "fields": [ + "scope as Scope" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "JSONHas", + "operator": "", + "type": "string", + "value": "" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "$__timeFilter", + "operator": "", + "type": "datetime", + "value": "" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "user_id", + "operator": "IN", + "type": "", + "value": [ + "users" + ] + }, + { + "condition": "AND", + "filterType": "custom", + "key": "1", + "operator": "IN", + "type": "", + "value": [ + "projects", + "OR", + "project_id", + "IN", + "projects" + ] + }, + { + "condition": "AND", + "filterType": "custom", + "key": "task_id", + "operator": "IN", + "type": "", + "value": [ + "tasks" + ] + }, + { + "condition": "AND", + "filterType": "custom", + "key": "job_id", + "operator": "IN", + "type": "", + "value": [ + "jobs" + ] + }, + { + "condition": "AND", + "filterType": "custom", + "key": "source", + "operator": "=", + "type": "string", + "value": [ + "client" + ] + } + ], + "groupBy": [ + "user_id", + "project_id", + "task_id", + "job_id" + ], + "limit": 100, + "metrics": [ + { + "aggregation": "min", + "alias": "working_time", + "field": "JSONExtractUInt" + }, + { + "aggregation": "count", + "alias": "Activity", + "field": "as" + } + ], + "mode": "aggregate", + "table": "events" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "meta": { + "builderOptions": { + "database": "cvat", + "fields": [ + "scope as Scope" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "JSONHas", + "operator": "", + "type": "string", + "value": "" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "$__timeFilter", + "operator": "", + "type": "datetime", + "value": "" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "user_id", + "operator": "IN", + "type": "", + "value": [ + "users" + ] + }, + { + "condition": "AND", + "filterType": "custom", + "key": "1", + "operator": "IN", + "type": "", + "value": [ + "projects", + "OR", + "project_id", + "IN", + "projects" + ] + }, + { + "condition": "AND", + "filterType": "custom", + "key": "task_id", + "operator": "IN", + "type": "", + "value": [ + "tasks" + ] + }, + { + "condition": "AND", + "filterType": "custom", + "key": "job_id", + "operator": "IN", + "type": "", + "value": [ + "jobs" + ] + }, + { + "condition": "AND", + "filterType": "custom", + "key": "source", + "operator": "=", + "type": "string", + "value": [ + "client" + ] + } + ], + "groupBy": [ + "user_id", + "project_id", + "task_id", + "job_id" + ], + "limit": 100, + "metrics": [ + { + "aggregation": "min", + "alias": "working_time", + "field": "JSONExtractUInt" + }, + { + "aggregation": "count", + "alias": "Activity", + "field": "as" + } + ], + "mode": "aggregate", + "table": "events" + } + }, + "queryType": "sql", + "rawSql": "SELECT\r\n scope as Scope,\r\n source as Source,\r\n avg(duration) as \"Average duration (ms)\",\r\n min(duration) as \"Min duration (ms)\",\r\n max(duration) as \"Max duration (ms)\"\r\nFROM events\r\nWHERE duration > 0\r\n AND $__timeFilter(timestamp)\r\nGROUP BY scope, source", + "refId": "A" + } + ], + "title": "Duration of events", + "type": "table" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 4, + "options": { + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "SELECT\r\n scope as Scope,\r\n source as Source,\r\n count() as Count\r\nFROM events\r\nWHERE $__timeFilter(timestamp)\r\nGROUP BY scope, source", + "refId": "A" + } + ], + "title": "Number of events", + "type": "table" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "builderOptions": { + "fields": [], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "scope", + "operator": "=", + "type": "String", + "value": "send:exception" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime64(3, 'Etc/UTC')", + "value": "TODAY" + } + ], + "metrics": [ + { + "aggregation": "count", + "field": "*" + } + ], + "mode": "trend", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 0, + "queryType": "builder", + "rawSql": "SELECT $__timeInterval(timestamp) as time, count(*) FROM events WHERE $__timeFilter(timestamp) AND ( scope = 'send:exception' ) AND ( timestamp >= $__fromTime AND timestamp <= $__toTime ) GROUP BY time ORDER BY time ASC", + "refId": "A" + } + ], + "title": "Exceptions", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": true, + "minWidth": 80 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "user_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 28 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "source" + }, + "properties": [ + { + "id": "custom.width", + "value": 62 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "project_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 61 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "task_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 41 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 27 + } + ] + } + ] + }, + "gridPos": { + "h": 17, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 8, + "options": { + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "builderOptions": { + "fields": [ + "user_id", + "project_id", + "task_id", + "job_id", + "payload" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime64(3, 'Etc/UTC')", + "value": "TODAY" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "scope", + "operator": "=", + "type": "String", + "value": "send:exception" + } + ], + "mode": "list", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [ + "user_id", + "project_id", + "task_id", + "job_id", + "payload" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime64(3, 'Etc/UTC')", + "value": "TODAY" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "scope", + "operator": "=", + "type": "String", + "value": "send:exception" + } + ], + "mode": "list", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + } + }, + "queryType": "sql", + "rawSql": "SELECT\r\n user_id,\r\n source,\r\n project_id,\r\n task_id,\r\n job_id,\r\n JSONExtractString(payload, 'message') as error,\r\n JSONExtractString(payload, 'stack') as stack\r\nFROM events\r\nWHERE\r\n ( timestamp >= $__fromTime AND timestamp <= $__toTime )\r\n AND scope = 'send:exception'", + "refId": "A" + } + ], + "title": "Exceptions table", + "type": "table" + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Monitoring", + "uid": "WvDvWK04k", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/components/analytics/kibana_conf.yml b/components/analytics/grafana_conf.yml similarity index 89% rename from components/analytics/kibana_conf.yml rename to components/analytics/grafana_conf.yml index e5ccc75b323..f36a300379d 100644 --- a/components/analytics/kibana_conf.yml +++ b/components/analytics/grafana_conf.yml @@ -1,20 +1,20 @@ http: routers: - kibana: + grafana: entryPoints: - web middlewares: - analytics-auth - strip-prefix - service: kibana + service: grafana rule: Host(`{{ env "CVAT_HOST" }}`) && PathPrefix(`/analytics`) - kibana_https: + grafana_https: entryPoints: - websecure middlewares: - analytics-auth - strip-prefix - service: kibana + service: grafana tls: {} rule: Host(`{{ env "CVAT_HOST" }}`) && PathPrefix(`/analytics`) @@ -32,7 +32,7 @@ http: - /analytics services: - kibana: + grafana: loadBalancer: servers: - url: http://{{ env "DJANGO_LOG_VIEWER_HOST" }}:{{ env "DJANGO_LOG_VIEWER_PORT" }} diff --git a/components/analytics/kibana/Dockerfile b/components/analytics/kibana/Dockerfile deleted file mode 100644 index f7982dffc29..00000000000 --- a/components/analytics/kibana/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -ARG ELK_VERSION -FROM docker.elastic.co/kibana/kibana-oss:${ELK_VERSION} -COPY kibana.yml /usr/share/kibana/config/ - - diff --git a/components/analytics/kibana/export.json b/components/analytics/kibana/export.json deleted file mode 100644 index 7ba8d6372a9..00000000000 --- a/components/analytics/kibana/export.json +++ /dev/null @@ -1,197 +0,0 @@ -[ - { - "_id": "7e8996e0-c23d-11e8-8e1b-758ef07f6de8", - "_type": "dashboard", - "_source": { - "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":21,\"w\":48,\"h\":13,\"i\":\"1\"},\"id\":\"3ade53d0-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":34,\"w\":48,\"h\":27,\"i\":\"2\"},\"id\":\"9397f350-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"2\",\"type\":\"search\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":21,\"i\":\"3\"},\"id\":\"1ec6a660-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":21,\"i\":\"4\"},\"id\":\"65918380-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"version\":\"6.4.0\"}]", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" - }, - "timeRestore": false, - "description": "", - "title": "Monitoring", - "optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}", - "version": 1 - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "d92524b0-c25c-11e8-8e1b-758ef07f6de8", - "_type": "visualization", - "_source": { - "visState": "{\"title\":\"Activity of users\",\"type\":\"metrics\",\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"count\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"User\",\"terms_field\":\"userid.keyword\",\"terms_size\":\"100\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"cvat*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1},\"aggs\":[]}", - "uiStateJSON": "{}", - "description": "", - "title": "Activity of users", - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" - }, - "version": 1 - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "9397f350-c23e-11e8-8e1b-758ef07f6de8", - "_type": "search", - "_source": { - "hits": 0, - "sort": ["@timestamp", "desc"], - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"event:\\\"Send exception\\\"\"},\"filter\":[]}" - }, - "columns": ["task", "type", "userid", "stack"], - "description": "", - "title": "Table with exceptions", - "version": 1 - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "3ade53d0-c23e-11e8-8e1b-758ef07f6de8", - "_type": "visualization", - "_source": { - "title": "Timeline for exceptions", - "visState": "{\"title\":\"Timeline for exceptions\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum_bucket\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"filters\",\"params\":{\"filters\":[{\"input\":{\"query\":\"event:\\\"Send exception\\\"\",\"language\":\"lucene\"},\"label\":\"\"}]}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"count\",\"params\":{\"customLabel\":\"Exceptions\"}},\"customLabel\":\"Exceptions\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-15h\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Time\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":false,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Exceptions\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Exceptions\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"orderBucketsBySum\":false}}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" - } - }, - "references": [ - { - "id": "ec510550-c238-11e8-8e1b-758ef07f6de8", - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern" - } - ], - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "1ec6a660-c244-11e8-8e1b-758ef07f6de8", - "_type": "visualization", - "_source": { - "title": "Duration of events", - "visState": "{\"title\":\"Duration of events\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"event.keyword\",\"size\":1000,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Action\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\",\"customLabel\":\"\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\",\"customLabel\":\"\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\"}}]}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"exists\":{\"field\":\"duration\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"key\":\"duration\",\"negate\":false,\"type\":\"exists\",\"value\":\"exists\"}}]}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "ec510550-c238-11e8-8e1b-758ef07f6de8", - "_type": "index-pattern", - "_source": { - "fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"application\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"application.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"box count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"event\",\"type\":\"string\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"event.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"frame count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"object count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"points count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polygon count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polyline count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"task\",\"type\":\"string\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"task\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"track count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"userid\",\"type\":\"string\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"working_time\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", - "title": "cvat*", - "timeFieldName": "@timestamp", - "fieldFormatMap": "{\"duration\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asSeconds\"}},\"working_time\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asHours\"}}}" - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "65918380-c244-11e8-8e1b-758ef07f6de8", - "_type": "visualization", - "_source": { - "title": "Number of events", - "visState": "{\"title\":\"Number of events\",\"type\":\"table\",\"params\":{\"perPage\":20,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"event.keyword\",\"size\":1000,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Action\"}}]}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "543f6260-c25c-11e8-8e1b-758ef07f6de8", - "_type": "visualization", - "_source": { - "title": "Working day", - "visState": "{\"title\":\"Working day\",\"type\":\"table\",\"params\":{\"perPage\":20,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"Start\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userid.keyword\",\"size\":1000,\"order\":\"asc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"End\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"split\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"day\",\"row\":true}}]}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "31ac2d60-c25b-11e8-8e1b-758ef07f6de8", - "_type": "visualization", - "_source": { - "title": "List of users", - "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"User\",\"field\":\"userid.keyword\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"asc\",\"orderBy\":\"_key\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"size\":1000},\"schema\":\"bucket\",\"type\":\"terms\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customBucket\":{\"enabled\":true,\"id\":\"3-bucket\",\"params\":{\"customInterval\":\"2h\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":{\"aggFilter\":[],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"bucketAgg\",\"params\":[],\"title\":\"Bucket Agg\"},\"type\":\"date_histogram\"},\"customLabel\":\"Activity\",\"customMetric\":{\"enabled\":true,\"id\":\"3-metric\",\"params\":{},\"schema\":{\"aggFilter\":[\"!top_hits\",\"!percentiles\",\"!percentile_ranks\",\"!median\",\"!std_dev\",\"!sum_bucket\",\"!avg_bucket\",\"!min_bucket\",\"!max_bucket\",\"!derivative\",\"!moving_avg\",\"!serial_diff\",\"!cumulative_sum\"],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"metricAgg\",\"params\":[],\"title\":\"Metric Agg\"},\"type\":\"count\"}},\"schema\":\"metric\",\"type\":\"sum_bucket\"},{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Working Time (h)\",\"field\":\"working_time\"},\"schema\":\"metric\",\"type\":\"sum\"}],\"params\":{\"perPage\":20,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"title\":\"List of users\",\"type\":\"table\"}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "7f637200-d068-11e8-9320-a3c87be2b433", - "_type": "visualization", - "_source": { - "title": "List of tasks", - "visState": "{\"title\":\"List of tasks\",\"type\":\"table\",\"params\":{\"perPage\":20,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"working_time\",\"customLabel\":\"Working time (h)\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"task\",\"size\":1000,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Task\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userid.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}}]}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "22250a40-c25d-11e8-8e1b-758ef07f6de8", - "_type": "dashboard", - "_source": { - "title": "Managment", - "hits": 0, - "description": "", - "panelsJSON": "[{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":33,\"i\":\"1\"},\"id\":\"31ac2d60-c25b-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":33,\"w\":48,\"h\":33,\"i\":\"2\"},\"id\":\"543f6260-c25c-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"2\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":66,\"w\":48,\"h\":33,\"i\":\"3\"},\"id\":\"d92524b0-c25c-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":33,\"i\":\"4\"},\"id\":\"7f637200-d068-11e8-9320-a3c87be2b433\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"version\":\"6.4.0\"}]", - "optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}", - "version": 1, - "timeRestore": false, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - } -] diff --git a/components/analytics/kibana/kibana.yml b/components/analytics/kibana/kibana.yml deleted file mode 100644 index 0459f3441cb..00000000000 --- a/components/analytics/kibana/kibana.yml +++ /dev/null @@ -1,4 +0,0 @@ -server.host: 0.0.0.0 -elasticsearch.requestHeadersWhitelist: ['cookie', 'authorization', 'x-forwarded-user'] -kibana.defaultAppId: 'discover' -server.basePath: /analytics diff --git a/components/analytics/kibana/setup.py b/components/analytics/kibana/setup.py deleted file mode 100644 index 2335fcb7da1..00000000000 --- a/components/analytics/kibana/setup.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (C) 2021-2022 Intel Corporation -# -# SPDX-License-Identifier: MIT - -#/usr/bin/env python - -import argparse -import json -from time import sleep -import requests - -def import_resources(host, port, cfg_file): - with open(cfg_file, 'r') as f: - for saved_object in json.load(f): - _id = saved_object["_id"] - _type = saved_object["_type"] - _doc = saved_object["_source"] - import_saved_object(host, port, _type, _id, _doc) - -def import_saved_object(host, port, _type, _id, data): - saved_objects_api = "http://{}:{}/api/saved_objects/{}/{}".format( - host, port, _type, _id) - request = requests.get(saved_objects_api) - if request.status_code == 404: - print("Creating {} as {}".format(_type, _id)) - request = requests.post(saved_objects_api, json={"attributes": data}, - headers={'kbn-xsrf': 'true'}) - else: - print("Updating {} named {}".format(_type, _id)) - request = requests.put(saved_objects_api, json={"attributes": data}, - headers={'kbn-xsrf': 'true'}) - request.raise_for_status() - -def wait_for_status(host, port, status='green', max_attempts=10, delay=3): - for _ in range(max_attempts): - response = requests.get('http://{}:{}/api/status'.format(host, port)) - if response.status_code != 200: - sleep(delay) - continue - - response = response.json() - if status == response['status']['overall']['state']: - return True - - return False - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='import Kibana 6.x resources', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('export_file', metavar='FILE', - help='JSON export file with resources') - parser.add_argument('-p', '--port', metavar='PORT', default=5601, type=int, - help='port of Kibana instance') - parser.add_argument('-H', '--host', metavar='HOST', default='kibana', - help='host of Kibana instance') - args = parser.parse_args() - - if wait_for_status(args.host, args.port): - import_resources(args.host, args.port, args.export_file) - else: - exit('Cannot setup Kibana objects') diff --git a/components/analytics/logstash/Dockerfile b/components/analytics/logstash/Dockerfile deleted file mode 100644 index f7c869875b5..00000000000 --- a/components/analytics/logstash/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -ARG ELK_VERSION -FROM docker.elastic.co/logstash/logstash-oss:${ELK_VERSION} -RUN logstash-plugin install logstash-input-http logstash-filter-aggregate \ - logstash-filter-prune logstash-output-email - -COPY logstash.yml /usr/share/logstash/config/ -COPY logstash.conf /usr/share/logstash/pipeline/ -EXPOSE 8080 diff --git a/components/analytics/logstash/logstash.conf b/components/analytics/logstash/logstash.conf deleted file mode 100644 index 67ef24b3973..00000000000 --- a/components/analytics/logstash/logstash.conf +++ /dev/null @@ -1,141 +0,0 @@ -input { - http { - port => 8080 - codec => json - } -} - -filter { - mutate { - add_field => {"logger_name" => ""} - add_field => {"path" =>""} - } - mutate { - copy => {"[extra][logger_name]" => "logger_name" } - copy => {"[extra][path]"=>"path"} - } - prune { - blacklist_names => ["type","logsource","extra","program","pid","headers"] - } - if [logger_name] =~ /cvat.client/ { - # 1. Decode the event from json in 'message' field - # 2. Remove unnecessary field from it - # 3. Type it as client - - mutate { - rename => { "message" => "source_message" } - } - mutate { - add_field => {"[@metadata][target_index_client]" => "cvat.client.%{+YYYY}.%{+MM}"} - } - - json { - source => "source_message" - } - - date { - match => ["time", "ISO8601"] - remove_field => "time" - } - - if [payload] { - ruby { - code => " - event.get('payload').each { |key, value| - event.set(key, value) - } - " - } - } - - if [name] == "Send exception" { - aggregate { - task_id => "%{username}_%{message}_%{filename}_%{line}" - code => " - require 'time' - - map['username'] ||= event.get('username'); - map['error'] ||= event.get('message'); - map['filename'] ||= event.get('filename'); - map['line'] ||= event.get('line'); - map['task_id'] ||= event.get('task_id'); - map['job_id'] ||= event.get('job_id'); - map['error_count'] ||= 0; - map['error_count'] += 1; - - map['aggregated_stack'] ||= ''; - map['aggregated_stack'] += event.get('stack') + '\n\n\n'; - " - timeout => 3600 - timeout_tags => ['aggregated_exception'] - push_map_as_event_on_timeout => true - } - } - - prune { - blacklist_names => ["level", "host", "logger_name", "path", - "port", "stack_info", "payload", "source_message"] - } - - mutate { - replace => { "type" => "client" } - rename => ["working time", "working_time"] - copy => { - "job_id" => "task" - "username" => "userid" - "name" => "event" - } - } - } else if [logger_name] =~ /cvat.server/ { - # 1. Remove 'logger_name' field and create 'task' field - # 2. Remove unnecessary field from it - # 3. Type it as server - if [logger_name] =~ /cvat\.server\.task_[0-9]+/ { - mutate { - add_field => {"[@metadata][target_index_server]" => "cvat.server.%{+YYYY}.%{+MM}"} - } - mutate { - rename => { "logger_name" => "task_id" } - gsub => [ "task_id", "cvat.server.task_", "" ] - } - - # Need to split the mutate because otherwise the conversion - # doesn't work. - mutate { - convert => { "task_id" => "integer" } - } - } - - prune { - blacklist_names => ["host", "port", "stack_info"] - } - - mutate { - replace => { "type" => "server" } - } - } -} - -output { - stdout { - codec => rubydebug - } - - if [type] == "client" { - elasticsearch { - hosts => ["${LOGSTASH_OUTPUT_HOST}"] - index => "%{[@metadata][target_index_client]}" - user => "${LOGSTASH_OUTPUT_USER:}" - password => "${LOGSTASH_OUTPUT_PASS:}" - manage_template => false - } - } else if [type] == "server" { - elasticsearch { - hosts => ["${LOGSTASH_OUTPUT_HOST}"] - index => "%{[@metadata][target_index_server]}" - user => "${LOGSTASH_OUTPUT_USER:}" - password => "${LOGSTASH_OUTPUT_PASS:}" - manage_template => false - } - } -} diff --git a/components/analytics/logstash/logstash.yml b/components/analytics/logstash/logstash.yml deleted file mode 100644 index 01d2873030c..00000000000 --- a/components/analytics/logstash/logstash.yml +++ /dev/null @@ -1,4 +0,0 @@ -queue.type: persisted -queue.max_bytes: 1gb -queue.checkpoint.writes: 20 -http.host: 0.0.0.0 diff --git a/components/analytics/vector/vector.toml b/components/analytics/vector/vector.toml new file mode 100644 index 00000000000..0f56058543f --- /dev/null +++ b/components/analytics/vector/vector.toml @@ -0,0 +1,44 @@ +[sources.http-events] +type = "http_server" +address = "0.0.0.0:80" +encoding = "json" + +# Uncomment for debug +# [sinks.console] +# type = "console" +# inputs = [ "http-events" ] +# target = "stdout" + +# [sinks.console.encoding] +# codec = "json" + +[sinks.clickhouse] +inputs = [ "http-events" ] +type = "clickhouse" +database = "${CLICKHOUSE_DB}" +table = "events" +auth.strategy = "basic" +auth.user = "${CLICKHOUSE_USER}" +auth.password = "${CLICKHOUSE_PASSWORD}" +endpoint = "http://clickhouse:8123" +batch.timeout_secs = 10 +request.concurrency = "adaptive" +encoding.only_fields = [ + "scope", + "obj_name", + "obj_id", + "obj_val", + "source", + "timestamp", + "count", + "duration", + "project_id", + "task_id", + "job_id", + "user_id", + "user_name", + "user_email", + "org_id", + "org_slug", + "payload", +] diff --git a/cvat-core/package.json b/cvat-core/package.json index a85702d8c5c..a6f3ffc5a9c 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "8.1.0", + "version": "8.2.0", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "src/api.ts", "scripts": { diff --git a/cvat-core/src/api.ts b/cvat-core/src/api.ts index ec7488d05fa..c3c9f1dce08 100644 --- a/cvat-core/src/api.ts +++ b/cvat-core/src/api.ts @@ -10,7 +10,7 @@ import PluginRegistry from './plugins'; import loggerStorage from './logger-storage'; -import { Log } from './log'; +import { EventLogger } from './log'; import ObjectState from './object-state'; import Statistics from './statistics'; import Comment from './comment'; @@ -283,7 +283,7 @@ function build() { Project: implementProject(Project), Task: implementTask(Task), Job: implementJob(Job), - Log, + EventLogger, Attribute, Label, Statistics, diff --git a/cvat-core/src/enums.ts b/cvat-core/src/enums.ts index 386f41459e8..78b451cbe6c 100644 --- a/cvat-core/src/enums.ts +++ b/cvat-core/src/enums.ts @@ -76,37 +76,38 @@ export enum Source { } export enum LogType { - loadJob = 'Load job', - saveJob = 'Save job', - restoreJob = 'Restore job', - uploadAnnotations = 'Upload annotations', - sendUserActivity = 'Send user activity', - sendException = 'Send exception', - sendTaskInfo = 'Send task info', - - drawObject = 'Draw object', - pasteObject = 'Paste object', - copyObject = 'Copy object', - propagateObject = 'Propagate object', - dragObject = 'Drag object', - resizeObject = 'Resize object', - deleteObject = 'Delete object', - lockObject = 'Lock object', - mergeObjects = 'Merge objects', - changeAttribute = 'Change attribute', - changeLabel = 'Change label', - - changeFrame = 'Change frame', - moveImage = 'Move image', - zoomImage = 'Zoom image', - fitImage = 'Fit image', - rotateImage = 'Rotate image', - - undoAction = 'Undo action', - redoAction = 'Redo action', - - pressShortcut = 'Press shortcut', - debugInfo = 'Debug info', + loadJob = 'load:job', + saveJob = 'save:job', + restoreJob = 'restore:job', + uploadAnnotations = 'upload:annotations', + exception = 'send:exception', + sendTaskInfo = 'send:task_info', + + drawObject = 'draw:object', + pasteObject = 'paste:object', + copyObject = 'copy:object', + propagateObject = 'propagate:object', + dragObject = 'drag:object', + resizeObject = 'resize:object', + deleteObject = 'delete:object', + lockObject = 'lock:object', + mergeObjects = 'merge:objects', + changeAttribute = 'change:attribute', + changeLabel = 'change:label', + + changeFrame = 'change:frame', + moveImage = 'move:image', + zoomImage = 'zoom:image', + fitImage = 'fit:image', + rotateImage = 'rotate:image', + + undoAction = 'action:undo', + redoAction = 'action:redo', + + pressShortcut = 'press:shortcut', + debugInfo = 'send:debug_info', + + clickElement = 'click:element', } export enum HistoryActions { diff --git a/cvat-core/src/log.ts b/cvat-core/src/log.ts index 40d7aea585f..cdc5ee1e3b9 100644 --- a/cvat-core/src/log.ts +++ b/cvat-core/src/log.ts @@ -8,9 +8,9 @@ import PluginRegistry from './plugins'; import { LogType } from './enums'; import { ArgumentError } from './exceptions'; -export class Log { +export class EventLogger { public readonly id: number; - public readonly type: LogType; + public readonly scope: LogType; public readonly time: Date; public payload: any; @@ -20,7 +20,7 @@ export class Log { constructor(logType: LogType, payload: any) { this.onCloseCallback = null; - this.type = logType; + this.scope = logType; this.payload = { ...payload }; this.time = new Date(); } @@ -45,11 +45,22 @@ export class Log { public dump(): any { const payload = { ...this.payload }; const body = { - name: this.type, - time: this.time.toISOString(), + scope: this.scope, + timestamp: this.time.toISOString(), }; - for (const field of ['client_id', 'job_id', 'task_id', 'is_active']) { + for (const field of [ + 'obj_name', + 'obj_id', + 'obj_val', + 'count', + 'duration', + 'project_id', + 'task_id', + 'job_id', + 'user_id', + 'organization', + ]) { if (field in payload) { body[field] = payload[field]; delete payload[field]; @@ -58,7 +69,7 @@ export class Log { return { ...body, - payload, + payload: JSON.stringify(payload), }; } @@ -67,12 +78,12 @@ export class Log { // Log duration will be computed based on the latest call // All payloads will be shallowly combined (all top level properties will exist) public async close(payload = {}): Promise { - const result = await PluginRegistry.apiWrapper.call(this, Log.prototype.close, payload); + const result = await PluginRegistry.apiWrapper.call(this, EventLogger.prototype.close, payload); return result; } } -Object.defineProperties(Log.prototype.close, { +Object.defineProperties(EventLogger.prototype.close, { implementation: { writable: false, enumerable: false, @@ -86,131 +97,71 @@ Object.defineProperties(Log.prototype.close, { }, }); -class LogWithCount extends Log { +class LogWithCount extends EventLogger { public validatePayload(): void { super.validatePayload.call(this); if (!Number.isInteger(this.payload.count) || this.payload.count < 1) { - const message = `The field "count" is required for "${this.type}" log. It must be a positive integer`; + const message = `The field "count" is required for "${this.scope}" log. It must be a positive integer`; throw new ArgumentError(message); } } } -class LogWithObjectsInfo extends Log { - public validatePayload(): void { - const generateError = (name: string, range: string): void => { - const message = `The field "${name}" is required for "${this.type}" log. ${range}`; - throw new ArgumentError(message); - }; - - if (!Number.isInteger(this.payload['track count']) || this.payload['track count'] < 0) { - generateError('track count', 'It must be an integer not less than 0'); - } - - if (!Number.isInteger(this.payload['tag count']) || this.payload['tag count'] < 0) { - generateError('tag count', 'It must be an integer not less than 0'); - } - - if (!Number.isInteger(this.payload['object count']) || this.payload['object count'] < 0) { - generateError('object count', 'It must be an integer not less than 0'); - } - - if (!Number.isInteger(this.payload['frame count']) || this.payload['frame count'] < 1) { - generateError('frame count', 'It must be an integer not less than 1'); - } - - if (!Number.isInteger(this.payload['box count']) || this.payload['box count'] < 0) { - generateError('box count', 'It must be an integer not less than 0'); - } - - if (!Number.isInteger(this.payload['polygon count']) || this.payload['polygon count'] < 0) { - generateError('polygon count', 'It must be an integer not less than 0'); - } - - if (!Number.isInteger(this.payload['polyline count']) || this.payload['polyline count'] < 0) { - generateError('polyline count', 'It must be an integer not less than 0'); - } - - if (!Number.isInteger(this.payload['points count']) || this.payload['points count'] < 0) { - generateError('points count', 'It must be an integer not less than 0'); - } - } -} - -class LogWithWorkingTime extends Log { - public validatePayload(): void { - super.validatePayload.call(this); - - if ( - !('working_time' in this.payload) || - !(typeof this.payload.working_time === 'number') || - this.payload.working_time < 0 - ) { - const message = ` - The field "working_time" is required for ${this.type} log. It must be a number not less than 0 - `; - throw new ArgumentError(message); - } - } -} - -class LogWithExceptionInfo extends Log { +class LogWithExceptionInfo extends EventLogger { public validatePayload(): void { super.validatePayload.call(this); if (typeof this.payload.message !== 'string') { - const message = `The field "message" is required for ${this.type} log. It must be a string`; + const message = `The field "message" is required for ${this.scope} log. It must be a string`; throw new ArgumentError(message); } if (typeof this.payload.filename !== 'string') { - const message = `The field "filename" is required for ${this.type} log. It must be a string`; + const message = `The field "filename" is required for ${this.scope} log. It must be a string`; throw new ArgumentError(message); } if (typeof this.payload.line !== 'number') { - const message = `The field "line" is required for ${this.type} log. It must be a number`; + const message = `The field "line" is required for ${this.scope} log. It must be a number`; throw new ArgumentError(message); } if (typeof this.payload.column !== 'number') { - const message = `The field "column" is required for ${this.type} log. It must be a number`; + const message = `The field "column" is required for ${this.scope} log. It must be a number`; throw new ArgumentError(message); } if (typeof this.payload.stack !== 'string') { - const message = `The field "stack" is required for ${this.type} log. It must be a string`; + const message = `The field "stack" is required for ${this.scope} log. It must be a string`; throw new ArgumentError(message); } } public dump(): any { - let body = super.dump(); - const { payload } = body; + const body = super.dump(); const client = detect(); - body = { - ...body, - message: payload.message, - filename: payload.filename, - line: payload.line, - column: payload.column, - stack: payload.stack, + body.payload = JSON.stringify({ + ...JSON.parse(body.payload), system: client.os, client: client.name, version: client.version, - }; - - delete payload.message; - delete payload.filename; - delete payload.line; - delete payload.column; - delete payload.stack; + }); return body; } } -export default function logFactory(logType: LogType, payload: any): Log { +class LogWithControlsInfo extends EventLogger { + public dump(): any { + this.payload = { + obj_val: this.payload?.text, + obj_name: this.payload?.classes, + }; + return super.dump(); + } +} + +export default function logFactory(logType: LogType, payload: any): EventLogger { const logsWithCount = [ LogType.deleteObject, LogType.mergeObjects, @@ -222,17 +173,14 @@ export default function logFactory(logType: LogType, payload: any): Log { if (logsWithCount.includes(logType)) { return new LogWithCount(logType, payload); } - if ([LogType.sendTaskInfo, LogType.loadJob, LogType.uploadAnnotations].includes(logType)) { - return new LogWithObjectsInfo(logType, payload); - } - if (logType === LogType.sendUserActivity) { - return new LogWithWorkingTime(logType, payload); + if (logType === LogType.exception) { + return new LogWithExceptionInfo(logType, payload); } - if (logType === LogType.sendException) { - return new LogWithExceptionInfo(logType, payload); + if (logType === LogType.clickElement) { + return new LogWithControlsInfo(logType, payload); } - return new Log(logType, payload); + return new EventLogger(logType, payload); } diff --git a/cvat-core/src/logger-storage.ts b/cvat-core/src/logger-storage.ts index c9a5593a94e..a542a999e59 100644 --- a/cvat-core/src/logger-storage.ts +++ b/cvat-core/src/logger-storage.ts @@ -5,12 +5,10 @@ import PluginRegistry from './plugins'; import serverProxy from './server-proxy'; -import logFactory, { Log } from './log'; +import logFactory, { EventLogger } from './log'; import { LogType } from './enums'; import { ArgumentError } from './exceptions'; -const WORKING_TIME_THRESHOLD = 100000; // ms, 1.66 min - function sleep(ms): Promise { return new Promise((resolve) => { setTimeout(resolve, ms); @@ -18,24 +16,20 @@ function sleep(ms): Promise { } interface IgnoreRule { - lastLog: Log | null; + lastLog: EventLogger | null; timeThreshold?: number; - ignore: (previousLog: Log, currentPayload: any) => boolean; + ignore: (previousLog: EventLogger, currentPayload: any) => boolean; } class LoggerStorage { public clientID: string; - public lastLogTime: number; - public workingTime: number; - public collection: Array; + public collection: Array; public ignoreRules: Record; public isActiveChecker: (() => boolean) | null; public saving: boolean; constructor() { this.clientID = Date.now().toString().substr(-6); - this.lastLogTime = Date.now(); - this.workingTime = 0; this.collection = []; this.isActiveChecker = null; this.saving = false; @@ -43,13 +37,13 @@ class LoggerStorage { [LogType.zoomImage]: { lastLog: null, timeThreshold: 1000, - ignore(previousLog: Log) { + ignore(previousLog: EventLogger) { return (Date.now() - previousLog.time.getTime()) < this.timeThreshold; }, }, [LogType.changeAttribute]: { lastLog: null, - ignore(previousLog: Log, currentPayload: any) { + ignore(previousLog: EventLogger, currentPayload: any) { return ( currentPayload.object_id === previousLog.payload.object_id && currentPayload.id === previousLog.payload.id @@ -59,15 +53,6 @@ class LoggerStorage { }; } - protected updateWorkingTime(): void { - if (!this.isActiveChecker || this.isActiveChecker()) { - const lastLogTime = Date.now(); - const diff = lastLogTime - this.lastLogTime; - this.workingTime += diff < WORKING_TIME_THRESHOLD ? diff : 0; - this.lastLogTime = lastLogTime; - } - } - public async configure(isActiveChecker, activityHelper): Promise { const result = await PluginRegistry.apiWrapper.call( this, @@ -78,7 +63,7 @@ class LoggerStorage { return result; } - public async log(logType: LogType, payload = {}, wait = false): Promise { + public async log(logType: LogType, payload = {}, wait = false): Promise { const result = await PluginRegistry.apiWrapper.call(this, LoggerStorage.prototype.log, logType, payload, wait); return result; } @@ -103,7 +88,6 @@ Object.defineProperties(LoggerStorage.prototype.configure, { } this.isActiveChecker = () => !!isActiveChecker(); - userActivityCallback.push(this.updateWorkingTime.bind(this)); }, }, }); @@ -130,7 +114,6 @@ Object.defineProperties(LoggerStorage.prototype.log, { ...payload, }; - this.updateWorkingTime(); return ignoreRule.lastLog; } } @@ -147,15 +130,15 @@ Object.defineProperties(LoggerStorage.prototype.log, { } const pushEvent = (): void => { - this.updateWorkingTime(); log.validatePayload(); log.onClose(null); this.collection.push(log); }; - if (log.type === LogType.sendException) { - serverProxy.server.exception(log.dump()).catch(() => { - pushEvent(); + if (log.scope === LogType.exception) { + await serverProxy.events.save({ + events: [log.dump()], + timestamp: new Date().toISOString(), }); return log; @@ -177,39 +160,33 @@ Object.defineProperties(LoggerStorage.prototype.save, { writable: false, enumerable: false, value: async function implementation() { + if (!this.collection) { + return; + } + while (this.saving) { await sleep(100); } const collectionToSend = [...this.collection]; - const lastLog = this.collection[this.collection.length - 1]; - const logPayload: any = { client_id: this.clientID, - working_time: this.workingTime, }; if (this.isActiveChecker) { logPayload.is_active = this.isActiveChecker(); } - if (lastLog && lastLog.type === LogType.sendTaskInfo) { - logPayload.job_id = lastLog.payload.job_id; - logPayload.task_id = lastLog.payload.task_id; - } - - const userActivityLog = logFactory(LogType.sendUserActivity, logPayload); - collectionToSend.push(userActivityLog); - try { this.saving = true; - await serverProxy.logs.save(collectionToSend.map((log) => log.dump())); + await serverProxy.events.save({ + events: collectionToSend.map((log) => log.dump()), + timestamp: new Date().toISOString(), + }); for (const rule of Object.values(this.ignoreRules)) { rule.lastLog = null; } this.collection = []; - this.workingTime = 0; - this.lastLogTime = Date.now(); } finally { this.saving = false; } diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index c3cae83fd0c..5a1e2e30859 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -267,21 +267,6 @@ async function share(directoryArg) { return response.data; } -async function exception(exceptionObject) { - const { backendAPI } = config; - - try { - await Axios.post(`${backendAPI}/server/exception`, JSON.stringify(exceptionObject), { - proxy: config.proxy, - headers: { - 'Content-Type': 'application/json', - }, - }); - } catch (errorData) { - throw generateError(errorData); - } -} - async function formats(): Promise { const { backendAPI } = config; @@ -1884,11 +1869,11 @@ async function createFunction(functionData: any) { } } -async function saveLogs(logs) { +async function saveEvents(events) { const { backendAPI } = config; try { - await Axios.post(`${backendAPI}/server/logs`, JSON.stringify(logs), { + await Axios.post(`${backendAPI}/events`, JSON.stringify(events), { proxy: config.proxy, headers: { 'Content-Type': 'application/json', @@ -2539,7 +2524,6 @@ export default Object.freeze({ about, share, formats, - exception, login, logout, socialAuthentication, @@ -2606,8 +2590,8 @@ export default Object.freeze({ uploadAnnotations, }), - logs: Object.freeze({ - save: saveLogs, + events: Object.freeze({ + save: saveEvents, }), lambda: Object.freeze({ diff --git a/cvat-core/src/session-implementation.ts b/cvat-core/src/session-implementation.ts index e05b90067d5..99ec80b9a92 100644 --- a/cvat-core/src/session-implementation.ts +++ b/cvat-core/src/session-implementation.ts @@ -344,7 +344,16 @@ export function implementJob(Job) { }; Job.prototype.logger.log.implementation = async function (logType, payload, wait) { - const result = await loggerStorage.log(logType, { ...payload, task_id: this.taskId, job_id: this.id }, wait); + const result = await loggerStorage.log( + logType, + { + ...payload, + project_id: this.projectId, + task_id: this.taskId, + job_id: this.id, + }, + wait, + ); return result; }; @@ -795,7 +804,15 @@ export function implementTask(Task) { }; Task.prototype.logger.log.implementation = async function (logType, payload, wait) { - const result = await loggerStorage.log(logType, { ...payload, task_id: this.id }, wait); + const result = await loggerStorage.log( + logType, + { + ...payload, + project_id: this.projectId, + task_id: this.id, + }, + wait, + ); return result; }; diff --git a/cvat-ui/package.json b/cvat-ui/package.json index 55083ac82d9..40e6942de0f 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.48.1", + "version": "1.49.0", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index 3118999ad6f..2fbcb2b4c94 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -1110,7 +1110,6 @@ export function saveAnnotationsAsync(sessionInstance: any, afterSave?: () => voi await sessionInstance.frames.save(); await sessionInstance.annotations.save(); await saveJobEvent.close(); - await sessionInstance.logger.log(LogType.sendTaskInfo, await jobInfoGenerator(sessionInstance)); dispatch(saveLogsAsync()); const { frame } = receiveAnnotationsParameters(); diff --git a/cvat-ui/src/actions/import-actions.ts b/cvat-ui/src/actions/import-actions.ts index 9b8ce1c481b..92ca3df91f8 100644 --- a/cvat-ui/src/actions/import-actions.ts +++ b/cvat-ui/src/actions/import-actions.ts @@ -8,7 +8,7 @@ import { CombinedState } from 'reducers'; import { getCore, Storage } from 'cvat-core-wrapper'; import { LogType } from 'cvat-logger'; import { getProjectsAsync } from './projects-actions'; -import { jobInfoGenerator, receiveAnnotationsParameters, AnnotationActionTypes } from './annotation-actions'; +import { receiveAnnotationsParameters, AnnotationActionTypes } from './annotation-actions'; const core = getCore(); @@ -112,9 +112,7 @@ export const importDatasetAsync = ( const frame = state.annotation.player.frame.number; await instance.annotations.upload(format, useDefaultSettings, sourceStorage, file, { convMaskToPoly }); - await instance.logger.log(LogType.uploadAnnotations, { - ...(await jobInfoGenerator(instance)), - }); + await instance.logger.log(LogType.uploadAnnotations); await instance.annotations.clear(true); await instance.actions.clear(); diff --git a/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx b/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx index 655cf25125d..b1ca5a2b764 100644 --- a/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx +++ b/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx @@ -584,11 +584,6 @@ class CanvasWrapperComponent extends React.PureComponent { const { state, duration } = event.detail; const isDrawnFromScratch = !state.label; - if (isDrawnFromScratch) { - jobInstance.logger.log(LogType.drawObject, { count: 1, duration }); - } else { - jobInstance.logger.log(LogType.pasteObject, { count: 1, duration }); - } state.objectType = state.objectType || activeObjectType; state.label = state.label || jobInstance.labels.filter((label: any) => label.id === activeLabelID)[0]; @@ -608,6 +603,22 @@ class CanvasWrapperComponent extends React.PureComponent { }); } + const payload = { + object_type: state.objectType, + label: state.label.name, + frame: state.frame, + rotation: state.rotation, + occluded: state.occluded, + outside: state.outside, + shape_type: state.shapeType, + }; + + if (isDrawnFromScratch) { + jobInstance.logger.log(LogType.drawObject, { count: 1, duration, ...payload }); + } else { + jobInstance.logger.log(LogType.pasteObject, { count: 1, duration, ...payload }); + } + const objectState = new cvat.classes.ObjectState(state); onCreateAnnotations(jobInstance, frame, [objectState]); }; diff --git a/cvat-ui/src/components/cvat-app.tsx b/cvat-ui/src/components/cvat-app.tsx index 93c41e36fd1..e5c9db091e0 100644 --- a/cvat-ui/src/components/cvat-app.tsx +++ b/cvat-ui/src/components/cvat-app.tsx @@ -69,6 +69,7 @@ import showPlatformNotification, { } from 'utils/platform-checker'; import '../styles.scss'; import appConfig from 'config'; +import EventRecorder from 'utils/controls-logger'; import EmailConfirmationPage from './email-confirmation-pages/email-confirmed'; import EmailVerificationSentPage from './email-confirmation-pages/email-verification-sent'; import IncorrectEmailConfirmationPage from './email-confirmation-pages/incorrect-email-confirmation'; @@ -130,10 +131,12 @@ class CVATApplication extends React.PureComponent void)[] = []; - window.addEventListener('click', () => { + window.addEventListener('click', (event: MouseEvent) => { userActivityCallback.forEach((handler) => handler()); + EventRecorder.log(event); }); core.logger.configure(() => window.document.hasFocus, userActivityCallback); + EventRecorder.initSave(); customWaViewHit(location.pathname, location.search, location.hash); history.listen((_location) => { diff --git a/cvat-ui/src/components/global-error-boundary/global-error-boundary.tsx b/cvat-ui/src/components/global-error-boundary/global-error-boundary.tsx index 8b355f636a2..cc05777ab7c 100644 --- a/cvat-ui/src/components/global-error-boundary/global-error-boundary.tsx +++ b/cvat-ui/src/components/global-error-boundary/global-error-boundary.tsx @@ -97,9 +97,9 @@ class GlobalErrorBoundary extends React.PureComponent { }; if (job) { - job.logger.log(LogType.sendException, logPayload); + job.logger.log(LogType.exception, logPayload); } else { - logger.log(LogType.sendException, logPayload); + logger.log(LogType.exception, logPayload); } } diff --git a/cvat-ui/src/components/header/header.tsx b/cvat-ui/src/components/header/header.tsx index 7e5806ff0ae..5ba3b53a309 100644 --- a/cvat-ui/src/components/header/header.tsx +++ b/cvat-ui/src/components/header/header.tsx @@ -472,12 +472,12 @@ function HeaderContainer(props: Props): JSX.Element {