Prepro rework

.github/workflows/backend_e2e_tests.yml

@@ -42,6 +42,21 @@ jobs:
   run-end2end-tests:
     runs-on: ubuntu-latest
     steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: false
+
+          # all of these default to true, but feel free to set to
+          # "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: false
       - name: Set Swap Space to 10GB
         uses: pierotofy/set-swap-space@master
         with:
@@ -54,12 +69,9 @@ jobs:
           chmod -R a+rwx backend_repo/ models_cache/ spacy_models/ tika/
           python monkey_patch_docker_compose_for_backend_tests.py
           export GID=$(id -g)
-          export CELERY_TEXT_WORKER_CONCURRENCY=1
-          export CELERY_IMAGE_WORKER_CONCURRENCY=1
-          export CELERY_SIMSEARCH_WORKER_CONCURRENCY=1
-          export CELERY_ARCHIVE_WORKER_CONCURRENCY=1
           export API_PRODUCTION_WORKERS=0
-          docker compose -f compose-test.yml up -d --quiet-pull
+          export RAY_CONFIG="./config_test_no_gpu.yaml"
+          docker compose -f compose-test.yml up -d
           echo Waiting for containers to start...
           sleep 240
           cd ..
@@ -68,6 +80,7 @@ jobs:
           TESTDATA_PASSWORD: ${{ secrets.TESTDATA_PASSWORD }}
         run: |
           cd tools/importer
+          pip install -r requirements.txt
           wget -q http://ltdata1.informatik.uni-hamburg.de/dwts/totalitarismo.zip
           unzip -q -P "$TESTDATA_PASSWORD" totalitarismo.zip
           python dwts_importer.py --input_dir images --backend_url http://localhost:13120/ --project_name incel --tag_name totalitarisimo

.github/workflows/backend_unit_tests.yml

@@ -49,6 +49,21 @@ jobs:
   run-unit-tests:
     runs-on: ubuntu-latest
     steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: false
+
+          # all of these default to true, but feel free to set to
+          # "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: false
       - name: Set Swap Space to 10GB
         uses: pierotofy/set-swap-space@master
         with:
@@ -61,11 +76,8 @@ jobs:
           chmod -R a+rwx backend_repo/ models_cache/ spacy_models/ tika/
           python monkey_patch_docker_compose_for_backend_tests.py
           export GID=$(id -g)
-          export CELERY_TEXT_WORKER_CONCURRENCY=1
-          export CELERY_IMAGE_WORKER_CONCURRENCY=1
-          export CELERY_SIMSEARCH_WORKER_CONCURRENCY=1
-          export CELERY_ARCHIVE_WORKER_CONCURRENCY=1
           export API_PRODUCTION_WORKERS=0
+          export RAY_CONFIG="./config_test_no_gpu.yaml"
           docker compose -f compose-test.yml up -d --quiet-pull
           echo Waiting for containers to start...
           sleep 240

.pre-commit-config.yaml

@@ -1,3 +1,5 @@
+ci:
+  skip: [eslint]
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.3.0

README.md

@@ -1,5 +1,7 @@
 # D-WISE Tool Suite
 
+[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/uhh-lt/dwts/mwp_v1.svg)](https://results.pre-commit.ci/latest/github/uhh-lt/dwts/mwp_v1)
+
 This is the repository for the D-WISE Tool Suite (DWTS) - an outcome of
 the [D-WISE Project](https://www.dwise.uni-hamburg.de/)
 

backend/.env

@@ -8,12 +8,9 @@ INSTALL_JUPYTER=true
 API_PORT=5500
 API_PRODUCTION_MODE=0
 API_PRODUCTION_WORKERS=10
-CELERY_TEXT_WORKER_CONCURRENCY=10
-CELERY_IMAGE_WORKER_CONCURRENCY=1
-CELERY_AUDIO_WORKER_CONCURRENCY=1
-CELERY_VIDEO_WORKER_CONCURRENCY=1
 CELERY_SIMSEARCH_WORKER_CONCURRENCY=1
-CELERY_HEAVY_JOBS_WORKER_CONCURRENCY=1
+CELERY_BACKGROUND_JOBS_WORKER_CONCURRENCY=1
+CELERY_DEBUG_MODE=0
 
 REDIS_HOST=redis
 REDIS_PORT=6379
@@ -36,6 +33,10 @@ ES_HOST=elasticsearch
 ES_PORT=9200
 ES_MIN_HEALTH=50
 
+RAY_HOST=ray
+RAY_PORT=8000
+RAY_PROTOCOL=http
+
 API_EXPOSED=13120
 POSTGRES_EXPOSED=13121
 RABBIT1_EXPOSED=13123
@@ -44,16 +45,19 @@ RABBIT3_EXPOSED=13125
 RABBIT_EXPOSED=13126
 REDIS_EXPOSED=13127
 
-JUPYTER_TEXT_EXPOSED=13128
-JUPYTER_IMAGE_EXPOSED=13129
-JUPYTER_SIMSEARCH_EXPOSED=13136
-JUPYTER_HEAVY_JOBS_EXPOSED=13135
-JUPYTER_API_EXPOSED=13130
+KIBANA_EXPOSED=13128
+ELASTICSEARCH_EXPOSED=13129
+ELASTICSEARCH1_EXPOSED=13130
+CONTENT_SERVER_EXPOSED=13131
+
+RAY_API_EXPOSED=13132
+RAY_DASHBOARD_EXPOSED=13133
 
-KIBANA_EXPOSED=13131
-ELASTICSEARCH_EXPOSED=13132
-ELASTICSEARCH1_EXPOSED=13133
-CONTENT_SERVER_EXPOSED=13134
+JUPYTER_TEXT_EXPOSED=13134
+JUPYTER_IMAGE_EXPOSED=13135
+JUPYTER_API_EXPOSED=13136
+JUPYTER_SIMSEARCH_EXPOSED=13137
+JUPYTER_BACKGROUND_JOBS_EXPOSED=13138
 
 # MAIL SERVICE
 MAIL_ENABLED=True

backend/.env.dev

@@ -0,0 +1,31 @@
+DWISE_BACKEND_CONFIG=configs/default_localhost_dev.yaml
+LOG_LEVEL=debug
+JWT_TTL=10080
+JWT_SECRET=f5b73acd6d6776350bce963bbcd2853fb5de00a4a4a081280ce1123b4a69aea9
+
+API_PORT=33120
+API_PRODUCTION_MODE=0
+API_PRODUCTION_WORKERS=10
+
+CELERY_DEBUG_MODE=1
+
+REDIS_HOST=localhost
+REDIS_PORT=13124
+REDIS_PASSWORD=dwts123
+
+RABBITMQ_HOST=localhost
+RABBITMQ_PORT=13123
+RABBITMQ_USER=dwtsuser
+RABBITMQ_PASSWORD=dwts123
+
+POSTGRES_HOST=localhost
+POSTGRES_PORT=13122
+POSTGRES_DB=dwts
+POSTGRES_USER=dwtsuser
+POSTGRES_PASSWORD=dwts123
+
+FLOWER_BASIC_AUTH=dwtsuser:dwts123
+
+ES_HOST=localhost
+ES_PORT=13125
+ES_MIN_HEALTH=50

backend/.gitignore

@@ -1,3 +1,4 @@
+src/app/preprocessing/ray_model_worker/spec.yaml
 src/dev_notebooks
 backend_repo
 sample_data

backend/Dockerfile

@@ -28,6 +28,7 @@ RUN wget -q https://micro.mamba.pm/api/micromamba/linux-64/${MICROMAMBA_VERSION}
 # create the 'dwts' python environment with all dependencies
 ENV MAMBA_ROOT_PREFIX=/opt
 COPY environment.yml .
+COPY requirements.txt /tmp/requirements.txt
 RUN micromamba create -f environment.yml -q -y &&\
     micromamba clean -a -f -q -y &&\
     find /opt/ -follow -type f -name '*.a' -delete &&\

backend/environment.yml

@@ -1,20 +1,14 @@
 name: dwts
 channels:
-  - defaults
-  - huggingface
-  - pytorch
-  - fastai
   - conda-forge
+  - defaults
 dependencies:
   - python=3.10
-  - pytorch::pytorch=1.12
-  - conda-forge::cudatoolkit=11.6
-  - sentence-transformers=2.2
-  - huggingface::transformers=4.21
+  - conda-forge::pip=23.2.1
   - pydantic=1.8
-  - spacy=3.4
-  - cupy=11.2
   - fastapi=0.85
+  - srsly=2.4.8
+  - tqdm=4.66.1
   - sqlalchemy=1.4
   - psycopg2-binary=2.9
   - redis-py=4.3
@@ -32,9 +26,7 @@ dependencies:
   - frozendict=2.3
   - email_validator=1.3
   - sqlalchemy-utils=0.38
-  - timm=0.6
   - python-multipart=0.0.5
-  - spacy-transformers=1.1
   - ftfy=6.1
   - beautifulsoup4=4.11.1
   - pytest=7.2.0

backend/requirements.txt

@@ -1,6 +1,6 @@
-faiss-gpu==1.7.2
+weaviate-client==3.24.1
 fastapi-mail==1.2.5
-git+https://github.com/linto-ai/whisper-timestamped.git@d767f4fc3b401c78c20d55515b382838ca3c86aa
+multiprocess==0.70.15
 Scrapy==2.10.0
 scrapy-playwright==0.0.31
 cssselect==1.2.0

backend/src/api/dependencies.py

@@ -63,7 +63,7 @@ async def get_current_user(
         email: str = payload.get("sub")
         if email is None:
             raise credentials_exception
-    except (JWTError, ValidationError) as e:
+    except (JWTError, ValidationError):
         raise credentials_exception
 
     user = crud_user.read_by_email(db=db, email=email)

backend/src/api/endpoints/crawler.py

@@ -1,8 +1,8 @@
 from typing import List
 
+from app.celery.background_jobs import prepare_and_start_crawling_job_async
 from app.core.data.crawler.crawler_service import CrawlerService
 from app.core.data.dto.crawler_job import CrawlerJobParameters, CrawlerJobRead
-from app.docprepro.heavy_jobs import prepare_and_start_crawling_job_async
 from fastapi import APIRouter
 
 router = APIRouter(prefix="/crawler")

backend/src/api/endpoints/export.py

@@ -1,6 +1,6 @@
+from app.celery.background_jobs import prepare_and_start_export_job_async
 from app.core.data.dto.export_job import ExportJobParameters, ExportJobRead
 from app.core.data.export.export_service import ExportService
-from app.docprepro.heavy_jobs import prepare_and_start_export_job_async
 from fastapi import APIRouter
 
 router = APIRouter(prefix="/export")

backend/src/api/endpoints/feedback.py

@@ -78,8 +78,6 @@ async def reply_to(
 ) -> str:
     # todo: load_feedback should raise exception, if it does not exist!
     feedback: Optional[FeedbackRead] = RedisService().load_feedback(key=feedback_id)
-    if feedback is None:
-        return f"Feedback with id {feedback_id} not found."
 
     user = crud_user.read(db=db, id=feedback.user_id)