feat: add generative ai powered knowledge discovery (#1)

* initial oss release --------- Co-authored-by: Michael Sekamanya <michael.s@ametnes.com>
ametnes · Apr 5, 2024 · 49a6fb7 · 49a6fb7
1 parent 1f02078
commit 49a6fb7
Show file tree

Hide file tree

Showing 289 changed files with 79,892 additions and 1 deletion.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,18 @@
+**/local_data
+**/__pycache__
+**/tests
+**/Dockerfile
+**/*test*.txt
+**/*test*.yaml
+
+.venv
+**/.github
+**/.vscode
+**/.DS_Store
+**/.mypy_cache
+.ruff_cache
+terraform
+tests
+Dockerfile.*
+
+**/node_modules
diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
@@ -0,0 +1,86 @@
+name: Package Nesis
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Branch'
+        required: true
+        default: main
+      tag:
+        description: 'Tag'
+        required: true
+        default: latest
+
+jobs:
+  package_api:
+    name: API
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.inputs.branch }}
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: true
+          file: ./nesis/api/Dockerfile
+          tags: ametnes/nesis:${{ github.event.inputs.tag }}-api
+
+  package_frontend:
+    name: Frontend
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.inputs.dockerfile_branch }}
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+      - name: Build and push frontend Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: true
+          file: ./nesis/frontend/Dockerfile
+          tags: ametnes/nesis:${{ github.event.inputs.tag }}-frontend
+          build-args: |
+            PUBLIC_URL=/
+            PROFILE=PROD
+
+  package_rag:
+    name: RAG
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.inputs.dockerfile_branch }}
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+      - name: Build and push RAG docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: true
+          file: ./nesis/rag/Dockerfile
+          tags: ametnes/nesis:${{ github.event.inputs.tag }}-rag
diff --git a/.github/workflows/test_api.yml b/.github/workflows/test_api.yml
@@ -0,0 +1,69 @@
+name: Test API
+
+on:
+  push:
+    paths:
+      - "nesis/api/core/**"
+      - "nesis/api/tests/**"
+      - "nesis/api/core/requirements*"
+  pull_request:
+    paths:
+      - "nesis/api/core/**"
+      - "nesis/api/tests/**"
+      - "nesis/api/core/requirements*"
+
+jobs:
+  linter:
+    name: Black formatter
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: psf/black@stable
+        with:
+          options: "--check --verbose"
+          src: "./nesis/api/"
+          version: "24.3.0"
+  test:
+    runs-on: ubuntu-latest
+    name: Test API
+    services:
+      postgres:
+        image: ametnes/postgresql:16-debian-12
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: nesis
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd "pg_isready -h localhost -U $$POSTGRES_USER"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      memcached:
+        image: bitnami/memcached:1.6.19
+        ports:
+          - 11211:11211
+    steps:
+      - name: Check out source
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r nesis/api/requirements.txt -r nesis/api/requirements-test.txt
+      - name: Run unit tests
+        env:
+          NESIS_API_DATABASE_URL: "postgresql://postgres:password@localhost:5432/nesis"
+          NESIS_ADMIN_EMAIL: "some.email@domain.com"
+          NESIS_ADMIN_PASSWORD: "password"
+          NESIS_MEMCACHE_HOSTS: localhost:11211
+        run: |
+          pytest nesis/api/tests/
diff --git a/.github/workflows/test_frontend.yml b/.github/workflows/test_frontend.yml
@@ -0,0 +1,33 @@
+name: Test Frontend
+
+on:
+  push:
+    paths:
+      - "nesis/frontend/client/**"
+      - "nesis/frontend/server/**"
+  pull_request:
+    paths:
+      - "nesis/frontend/client/**"
+      - "nesis/frontend/server/**"
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    name: Lint Frontend
+    defaults:
+      run:
+        working-directory: ./nesis/frontend
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Install frontend dependencies
+        run: npm install --legacy-peer-deps --prefix client --only dev
+      - name: Install server dependencies
+        run: npm install --legacy-peer-deps
+      - run: npm run format:check
+        name: Check formatting
+      - name: Test API
+        run: npm run test:api
diff --git a/.github/workflows/test_rag.yml b/.github/workflows/test_rag.yml
@@ -0,0 +1,70 @@
+name: Test Rag Engine
+
+on:
+  push:
+    paths:
+      - "nesis/rag/core/**"
+      - "nesis/rag/tests/**"
+      - "nesis/rag/core/requirements*"
+  pull_request:
+    paths:
+      - "nesis/rag/core/**"
+      - "nesis/rag/tests/**"
+      - "nesis/rag/core/requirements*"
+
+jobs:
+  linter:
+    name: Black formatter
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: psf/black@stable
+        with:
+          options: "--check --verbose"
+          src: "./nesis/api/"
+          version: "24.3.0"
+
+  test:
+    runs-on: ubuntu-latest
+    name: Test RAG API
+    services:
+      postgres:
+        image: ametnes/postgresql:16-debian-12
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: nesis
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd "pg_isready -h localhost -U $$POSTGRES_USER"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      memcached:
+        image: bitnami/memcached:1.6.19
+        ports:
+          - 11211:11211
+    steps:
+      - name: Check out source
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r nesis/rag/requirements.txt -r nesis/rag/requirements-test.txt -r nesis/rag/requirements-huggingface.txt
+      - name: Run unit tests
+        env:
+          NESIS_MEMCACHE_HOSTS: localhost:11211
+          NESIS_RAG_PGVECTOR_URL: postgresql://postgres:password@localhost:5432/nesis
+          NESIS_RAG_EMBEDDING_MODE: local
+          OPENAI_API_KEY: sk_some_key
+        run: |
+          pytest nesis/rag/tests/