ci-uto.yaml

name: Build and test Universal Transfer Operator
on:
  push:
    branches: [ 'main', 'release-**' ]
    paths:
      - '**'

  pull_request:
    branches: [ 'main', 'release-**' ]
    paths:
      - '**'
  # Run on PRs from forks
  pull_request_target:
    branches: [ 'main' ]
    types: ['labeled']
  release:
    types: [ 'created' ]
defaults:
  run:
    working-directory: .

# This allows a subsequently queued workflow run to interrupt and cancel previous runs
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

# This list should only have non-sensitive env vars.
# Env vars with secrets should be in the specific jobs
env:
  GOOGLE_CLOUD_PROJECT: astronomer-dag-authoring
  POSTGRES_HOST: postgres
  POSTGRES_PORT: 5432
  SNOWFLAKE_ACCOUNT_NAME: ${{secrets.SNOWFLAKE_ACCOUNT_NAME}}
  SNOWFLAKE_SCHEMA: ASTROFLOW_CI
  SNOWFLAKE_DATABASE: SANDBOX
  SNOWFLAKE_WAREHOUSE: DEMO
  SNOWFLAKE_HOST: ${{secrets.SNOWFLAKE_HOST}}
  SNOWFLAKE_ACCOUNT: ${{secrets.SNOWFLAKE_ACCOUNT_NAME}}
  SNOWFLAKE_REGION: us-east-1
  SNOWFLAKE_ROLE: AIRFLOW_TEST_USER
  AIRFLOW__UNIVERSAL_TRANSFER_OPERATOR__SNOWFLAKE_DEFAULT_SCHEMA: ASTROFLOW_CI
  SFTP_HOSTNAME: ${{ secrets.SFTP_HOSTNAME }}
  SFTP_USERNAME: ${{ secrets.SFTP_USERNAME }}
  SFTP_PASSWORD: ${{ secrets.SFTP_PASSWORD }}
  FIVETRAN_KEY: ${{ secrets.FIVETRAN_KEY }}
  FIVETRAN_SECRET: ${{ secrets.FIVETRAN_SECRET }}
  SNOWFLAKE_FIVETRAN_ACCOUNT_NAME: ${{secrets.SNOWFLAKE_FIVETRAN_ACCOUNT_NAME}}
  SNOWFLAKE_FIVETRAN_SCHEMA: ${{secrets.SNOWFLAKE_FIVETRAN_SCHEMA}}
  SNOWFLAKE_FIVETRAN_DATABASE: ${{secrets.SNOWFLAKE_FIVETRAN_DATABASE}}
  SNOWFLAKE_FIVETRAN_WAREHOUSE: ${{secrets.SNOWFLAKE_FIVETRAN_WAREHOUSE}}
  SNOWFLAKE_FIVETRAN_HOST: ${{secrets.SNOWFLAKE_FIVETRAN_HOST}}
  SNOWFLAKE_FIVETRAN_ACCOUNT: ${{secrets.SNOWFLAKE_FIVETRAN_ACCOUNT}}
  SNOWFLAKE_FIVETRAN_REGION: ${{secrets.SNOWFLAKE_FIVETRAN_REGION}}
  SNOWFLAKE_FIVETRAN_ROLE: ${{secrets.SNOWFLAKE_FIVETRAN_ROLE}}

jobs:
  Type-Check:
    if: github.event.action != 'labeled'
    runs-on: ubuntu-latest
    env:
      MYPY_FORCE_COLOR: 1
      TERM: xterm-color
      SETUPTOOLS_USE_DISTUTILS: stdlib
    steps:
      - uses: actions/checkout@v3

      - uses: actions/setup-python@v3
        with:
          python-version: '3.8'
          architecture: 'x64'

      - uses: actions/cache@v3
        with:
           path: |
             ~/.cache/pip
             .nox
             .local
           key: universal_transfer_operator-type-check-os-${{ runner.os }}-deps-${{ hashFiles('pyproject.toml') }}

      - name: Install Nox
        run: |
          pipx install nox
          nox --version

      - name: Run type check
        run: nox -s type_check

  Build-Docs:
    if: github.event.action != 'labeled'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
          python-version: '3.9'
          architecture: 'x64'
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cache/pip
            .nox
          key: ${{ runner.os }}-${{ hashFiles('pyproject.toml') }}
      - run: pip3 install nox
      - run: nox -s build_docs

  Run-UTO-tests-unit-tests:
    if: >-
      github.event_name == 'push' ||
      (
        github.event_name == 'pull_request' &&
        github.event.pull_request.head.repo.fork == false
      ) ||
      (
        github.event_name == 'pull_request_target' &&
        contains(github.event.pull_request.labels.*.name, 'safe to test')
      )||
      (
        github.event_name == 'release'
      )
    strategy:
      fail-fast: false
      matrix:
        python: [ 3.8, 3.9 ]
        airflow: [ 2.2.5, 2.4, 2.5 ]
    runs-on: ubuntu-latest
    services:
      postgres:
        image: dimberman/pagila-test
        env:
          POSTGRES_PASSWORD: postgres
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
          --name postgres
        ports:
          - 5432:5432
      sftp:
        image: ghcr.io/astronomer/astro-sdk/sftp_docker
        ports:
          - 2222:22
    steps:
      - uses: actions/checkout@v3
        if: github.event_name != 'pull_request_target'

      - name: Checkout pull/${{ github.event.number }}
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}
        if: github.event_name == 'pull_request_target'

      - uses: actions/setup-python@v3
        with:
          python-version: '${{ matrix.python }}'
          architecture: 'x64'

      - uses: actions/cache@v3
        with:
          path: |
            ~/.cache/pip
            .nox
            .local/
          key: uto-os-${{ runner.os }}-python-${{ matrix.python }}-airflow-${{ matrix.airflow }}-deps-${{ hashFiles('pyproject.toml') }}

      - name: Copy the Connections
        run: cat .github/ci-test-connections.yaml > test-connections.yaml
      - name: Set GOOGLE_APPLICATION_CREDENTIALS environment variable
        run: python -c 'import os; print(os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip())' > ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}

      - name: Install Nox
        run: |
          pipx install nox
          nox --version

      - name: Run tests
        run: nox -s "test-${{ matrix.python }}(airflow='${{ matrix.airflow }}')" -- tests/ --cov=src --cov-report=xml --cov-branch

      - name: Upload coverage
        uses: actions/upload-artifact@v2
        with:
          name: coverage-${{ matrix.python }}-${{ matrix.airflow }}
          path: .coverage
    env:
      GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }}
      GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_credentials.json
      GOOGLE_CLOUD_KEY_PATH: /tmp/google_credentials.json
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      SNOWFLAKE_ACCOUNT_NAME: ${{secrets.SNOWFLAKE_ACCOUNT_NAME}}
      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
      SNOWFLAKE_FIVETRAN_ACCOUNT_NAME: ${{secrets.SNOWFLAKE_FIVETRAN_ACCOUNT_NAME}}
      SNOWFLAKE_FIVETRAN_PASSWORD: ${{ secrets.SNOWFLAKE_FIVETRAN_PASSWORD }}

  Run-UTO-tests-integrations:
    if: >-
      github.event_name == 'push' ||
      (
        github.event_name == 'pull_request' &&
        github.event.pull_request.head.repo.fork == false
      ) ||
      (
        github.event_name == 'pull_request_target' &&
        contains(github.event.pull_request.labels.*.name, 'safe to test')
      )||
      (
        github.event_name == 'release'
      )
    strategy:
      fail-fast: false
      matrix:
        python: [ 3.8, 3.9 ]
        airflow: [ 2.2.5, 2.4, 2.5 ]
    runs-on: ubuntu-latest
    services:
      postgres:
        image: dimberman/pagila-test
        env:
          POSTGRES_PASSWORD: postgres
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
          --name postgres
        ports:
          - 5432:5432
      sftp:
        image: ghcr.io/astronomer/astro-sdk/sftp_docker
        ports:
          - 2222:22
    steps:
      - uses: actions/checkout@v3
        if: github.event_name != 'pull_request_target'

      - name: Checkout pull/${{ github.event.number }}
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}
        if: github.event_name == 'pull_request_target'

      - uses: actions/setup-python@v3
        with:
          python-version: '${{ matrix.python }}'
          architecture: 'x64'

      - uses: actions/cache@v3
        with:
          path: |
            ~/.cache/pip
            .nox
            .local/
          key: uto-os-${{ runner.os }}-python-${{ matrix.python }}-airflow-${{ matrix.airflow }}-deps-${{ hashFiles('pyproject.toml') }}

      - name: Copy the Connections
        run: cat .github/ci-test-connections.yaml > test-connections.yaml
      - name: Set GOOGLE_APPLICATION_CREDENTIALS environment variable
        run: python -c 'import os; print(os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip())' > ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}

      - name: Install Nox
        run: |
          pipx install nox
          nox --version

      - name: Run tests
        run: nox -s "test-${{ matrix.python }}(airflow='${{ matrix.airflow }}')" -- tests_integration/ -k "not test_example_dags.py" --cov=src --cov-report=xml --cov-branch

      - name: Upload coverage
        uses: actions/upload-artifact@v2
        with:
          name: coverage-${{ matrix.python }}-${{ matrix.airflow }}
          path: .coverage
    env:
      GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }}
      GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_credentials.json
      GOOGLE_CLOUD_KEY_PATH: /tmp/google_credentials.json
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      SNOWFLAKE_ACCOUNT_NAME: ${{secrets.SNOWFLAKE_ACCOUNT_NAME}}
      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
      SNOWFLAKE_FIVETRAN_ACCOUNT_NAME: ${{secrets.SNOWFLAKE_FIVETRAN_ACCOUNT_NAME}}
      SNOWFLAKE_FIVETRAN_PASSWORD: ${{ secrets.SNOWFLAKE_FIVETRAN_PASSWORD }}


  Run-UTO-tests-example-dags-2_2_5:
    if: >-
      github.event_name == 'push' ||
      (
        github.event_name == 'pull_request' &&
        github.event.pull_request.head.repo.fork == false
      ) ||
      (
        github.event_name == 'pull_request_target' &&
        contains(github.event.pull_request.labels.*.name, 'safe to test')
      )||
      (
        github.event_name == 'release'
      )
    strategy:
      fail-fast: false
      matrix:
        python: [ 3.9 ]
        airflow: [ 2.2.5 ]
    runs-on: ubuntu-latest
    services:
      postgres:
        image: dimberman/pagila-test
        env:
          POSTGRES_PASSWORD: postgres
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
          --name postgres
        ports:
          - 5432:5432
      sftp:
        image: ghcr.io/astronomer/astro-sdk/sftp_docker
        ports:
          - 2222:22
    steps:
      - uses: actions/checkout@v3
        if: github.event_name != 'pull_request_target'

      - name: Checkout pull/${{ github.event.number }}
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}
        if: github.event_name == 'pull_request_target'

      - uses: actions/setup-python@v3
        with:
          python-version: '${{ matrix.python }}'
          architecture: 'x64'

      - uses: actions/cache@v3
        with:
          path: |
            ~/.cache/pip
            .nox
            .local/
          key: uto-os-${{ runner.os }}-python-${{ matrix.python }}-airflow-${{ matrix.airflow }}-deps-${{ hashFiles('pyproject.toml') }}

      - name: Copy the Connections
        run: cat .github/ci-test-connections.yaml > test-connections.yaml
      - name: Set GOOGLE_APPLICATION_CREDENTIALS environment variable
        run: python -c 'import os; print(os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip())' > ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}

      - name: Install Nox
        run: |
          pipx install nox
          nox --version

      - name: Run tests
        run: nox -s "test-${{ matrix.python }}(airflow='${{ matrix.airflow }}')" -- tests_integration/test_example_dags.py --cov=src --cov-report=xml --cov-branch

      - name: Upload coverage
        uses: actions/upload-artifact@v2
        with:
          name: coverage-${{ matrix.python }}-${{ matrix.airflow }}
          path: .coverage
    env:
      GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }}
      GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_credentials.json
      GOOGLE_CLOUD_KEY_PATH: /tmp/google_credentials.json
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_ACCOUNT_NAME }}
      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}

  Code-Coverage:
    if: github.event.action != 'labeled'
    needs:
      - Run-UTO-tests-unit-tests
      - Run-UTO-tests-integrations
      - Run-UTO-tests-example-dags-2_2_5
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python 3.8
        uses: actions/setup-python@v3
        with:
          python-version: 3.8
      - name: Install coverage
        run: |
          pip3 install coverage
      - name: Download all artifacts
        uses: actions/download-artifact@v2
        with:
          path: ./coverage
      - name: Run coverage
        run: |
          coverage combine ./coverage/coverage*/.coverage
          coverage report
          coverage xml
      - name: Upload coverage
        uses: codecov/codecov-action@v2
        with:
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}
          flags: UTO
          files: ./coverage.xml

  build-n-publish:
    if: github.event_name == 'release'
    name: Build and publish Python 🐍 distributions 📦 to PyPI
    needs:
      - Type-Check
      - Build-Docs
      - Run-UTO-tests-unit-tests
      - Run-UTO-tests-integrations
      - Run-UTO-tests-example-dags-2_2_5
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-python@v2
        with:
          python-version: '3.8'
          architecture: 'x64'
      - uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ hashFiles('pyproject.toml') }}
      - run: pip3 install nox
      - run: nox -s build
      - run: nox -s release -- dist/*
    env:
      TWINE_USERNAME: __token__
      TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}