Merge branch 'master' into feat-chart-list-description

.codecov.yml

@@ -3,6 +3,7 @@ codecov:
         after_n_builds: 4
 ignore:
   - "superset/migrations/versions/*.py"
+  - "superset-frontend/packages/superset-ui-demo/**/*"
   - "**/*.stories.tsx"
   - "**/*.stories.jsx"
 coverage:
@@ -14,6 +15,17 @@ coverage:
         # project coverage decrease:
         target: auto
         threshold: 0%
+      core-packages-ts:
+        target: 95%
+        paths:
+          - 'superset-frontend/packages'
+          - '!superset-frontend/packages/**/*.jsx'
+          - '!superset-frontend/packages/**/*.tsx'
+      core-packages-tsx:
+        target: 50%
+        paths:
+          - 'superset-frontend/packages/**/*.jsx'
+          - 'superset-frontend/packages/**/*.tsx'
     patch:
       default:
         informational: true

.github/dependabot.yml

@@ -21,3 +21,9 @@ updates:
     schedule:
       interval: "daily"
     open-pull-requests-limit: 0
+
+  - package-ecosystem: "npm"
+    directory: "/docs/"
+    schedule:
+      interval: "daily"
+    open-pull-requests-limit: 0

.github/workflows/superset-frontend.yml

@@ -43,11 +43,24 @@ jobs:
         run: |
           npm run lint
           npm run prettier-check
+      - name: Build plugins packages
+        if: steps.check.outcome == 'failure'
+        working-directory: ./superset-frontend
+        run: npm run plugins:build
+      - name: Build plugins Storybook
+        if: steps.check.outcome == 'failure'
+        working-directory: ./superset-frontend
+        run: npm run plugins:build-storybook
       - name: unit tests
         if: steps.check.outcome == 'failure'
         working-directory: ./superset-frontend
         run: |
           npm run test -- --coverage
+      # todo: remove this step when fix generator as a project in root jest.config.js
+      - name: generator-superset unit tests
+        if: steps.check.outcome == 'failure'
+        working-directory: ./superset-frontend/packages/generator-superset
+        run: npx jest
       - name: Upload code coverage
         if: steps.check.outcome == 'failure'
         working-directory: ./superset-frontend

.github/workflows/superset-python-integrationtest.yml

@@ -51,13 +51,16 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: 'requirements/testing.txt'
       - name: Install dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
             apt-get-install
             pip-upgrade
+            pip install wheel
             pip install -r requirements/testing.txt
             setup-mysql
       - name: Run celery
@@ -115,13 +118,16 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: 'requirements/testing.txt'
       - name: Install dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
             apt-get-install
             pip-upgrade
+            pip install wheel
             pip install -r requirements/testing.txt
             setup-postgres
       - name: Run celery
@@ -171,13 +177,16 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: 'requirements/testing.txt'
       - name: Install dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
             apt-get-install
             pip-upgrade
+            pip install wheel
             pip install -r requirements/testing.txt
             mkdir ${{ github.workspace }}/.temp
       - name: Run celery

.github/workflows/superset-python-misc.yml

@@ -33,13 +33,16 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: 'requirements/testing.txt'
       - name: Install dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
             apt-get-install
             pip-upgrade
+            pip install wheel
             pip install -r requirements/testing.txt
       - name: pylint
         if: steps.check.outcome == 'failure'
@@ -62,12 +65,15 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: 'requirements/integration.txt'
       - name: Install dependencies
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
             apt-get-install
             pip-upgrade
+            pip install wheel
             pip install -r requirements/integration.txt
       - name: pre-commit
         run: pre-commit run --all-files
@@ -88,12 +94,15 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: 'requirements/base.txt'
       - name: Install dependencies
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
             apt-get-install
             pip-upgrade
+            pip install wheel
             pip install -r requirements/base.txt
       - name: Test babel extraction
         run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct

.github/workflows/superset-python-presto-hive.yml

@@ -62,13 +62,16 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: 'requirements/testing.txt'
       - name: Install dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
             apt-get-install
             pip-upgrade
+            pip install wheel
             pip install -r requirements/testing.txt
             setup-postgres
       - name: Run celery
@@ -137,13 +140,16 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: 'requirements/testing.txt'
       - name: Install dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
             apt-get-install
             pip-upgrade
+            pip install wheel
             pip install -r requirements/testing.txt
             setup-postgres
       - name: Run celery

.github/workflows/superset-python-unittest.yml

@@ -35,6 +35,8 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: 'requirements/testing.txt'
 # TODO: separated requiermentes.txt file just for unit tests
       - name: Install dependencies
         if: steps.check.outcome == 'failure'
@@ -43,6 +45,7 @@ jobs:
           run: |
             apt-get-install
             pip-upgrade
+            pip install wheel
             pip install -r requirements/testing.txt
             mkdir ${{ github.workspace }}/.temp
       - name: Python unit tests

.gitignore

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-*.ipynb
 *.bak
 *.db
 *.pyc
@@ -61,14 +60,29 @@ tmp
 rat-results.txt
 superset/app/
 
-# Node.js, webpack artifacts
+# Node.js, webpack artifacts, storybook
 *.entry.js
 *.js.map
 node_modules
 npm-debug.log*
 superset/static/assets
 superset/static/version_info.json
+superset-frontend/**/esm/*
+superset-frontend/**/lib/*
+superset-frontend/**/storybook-static/*
 yarn-error.log
+*.map
+*.min.js
+test-changelog.md
+*.tsbuildinfo
+
+# Ignore package-lock in packages
+plugins/*/package-lock.json
+packages/*/package-lock.json
+
+# For country map geojson conversion script
+.ipynb_checkpoints/
+scripts/*.zip
 
 # IntelliJ
 *.iml

.pre-commit-config.yaml

@@ -33,6 +33,7 @@ repos:
     hooks:
       - id: check-docstring-first
       - id: check-added-large-files
+        exclude: \.(geojson)$
       - id: check-yaml
         exclude: ^helm/superset/templates/
       - id: debug-statements
@@ -45,7 +46,8 @@ repos:
       - id: black
         language_version: python3
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.2.1 # Use the sha or tag you want to point at
+    rev: v2.4.1 # Use the sha or tag you want to point at
     hooks:
       - id: prettier
+        args: ['--ignore-path=./superset-frontend/.prettierignore']
         files: 'superset-frontend'

.rat-excludes

@@ -48,3 +48,16 @@ vendor/*
 # github configuration
 .github/*
 .*mdx
+
+# skip license check in superset-ui
+tmp/*
+lib/*
+esm/*
+tsconfig.tsbuildinfo
+.*ipynb
+.*yml
+.*iml
+.esprintrc
+.prettierignore
+generator-superset/*
+temporary_superset_ui/*

CONTRIBUTING.md

@@ -420,7 +420,7 @@ For example, the image referenced above actually lives in `superset-frontend/src
 
 #### OS Dependencies
 
-Make sure your machine meets the [OS dependencies](https://superset.apache.org/docs/installation/installing-superset-from-scratch#os-dependencies) before following these steps.  
+Make sure your machine meets the [OS dependencies](https://superset.apache.org/docs/installation/installing-superset-from-scratch#os-dependencies) before following these steps.
 You also need to install MySQL or [MariaDB](https://mariadb.com/downloads).
 
 Ensure that you are using Python version 3.7 or 3.8, then proceed with:
@@ -436,12 +436,12 @@ pip install -r requirements/testing.txt
 # Install Superset in editable (development) mode
 pip install -e .
 
-# Create an admin user in your metadata database (use `admin` as username to be able to load the examples)
-superset fab create-admin
-
 # Initialize the database
 superset db upgrade
 
+# Create an admin user in your metadata database (use `admin` as username to be able to load the examples)
+superset fab create-admin
+
 # Create default roles and permissions
 superset init
 
@@ -523,6 +523,7 @@ Frontend assets (TypeScript, JavaScript, CSS, and images) must be compiled in or
 ##### nvm and node
 
 First, be sure you are using the following versions of Node.js and npm:
+
 - `Node.js`: Version 16
 - `npm`: Version 7
 
@@ -765,15 +766,21 @@ Note that the test environment uses a temporary directory for defining the
 SQLite databases which will be cleared each time before the group of test
 commands are invoked.
 
-There is also a utility script included in the Superset codebase to run python tests. The [readme can be
+There is also a utility script included in the Superset codebase to run python integration tests. The [readme can be
 found here](https://github.com/apache/superset/tree/master/scripts/tests)
 
-To run all tests for example, run this script from the root directory:
+To run all integration tests for example, run this script from the root directory:
 
 ```bash
 scripts/tests/run.sh
 ```
 
+You can run unit tests found in './tests/unit_tests' for example with pytest. It is a simple way to run an isolated test that doesn't need any database setup
+
+```bash
+pytest ./link_to_test.py
+```
+
 ### Frontend Testing
 
 We use [Jest](https://jestjs.io/) and [Enzyme](https://airbnb.io/enzyme/) to test TypeScript/JavaScript. Tests can be run with:
@@ -904,7 +911,7 @@ apt install -y net-tools
 pip install debugpy
 ```
 
-Find the PID for the Flask process. Make sure to use the first PID. The Flask app will re-spawn a sub-process everytime you change any of the python code. So it's important to use the first PID.
+Find the PID for the Flask process. Make sure to use the first PID. The Flask app will re-spawn a sub-process every time you change any of the python code. So it's important to use the first PID.
 
 ```bash
 ps -ef
@@ -975,7 +982,7 @@ Once the pod is running as root and has the SYS_PTRACE capability it will be abl
 
 You can follow the same instructions as in the docker-compose. Enter the pod and install the required library and packages; gdb, netstat and debugpy.
 
-Often in a kuernetes environment nodes are not addressable from ouside the cluster. VSCode will thus be unable to remotely connect to port 5678 on a kubernetes node. In order to do this you need to create a tunnel that port forwards 5678 to your local machine.
+Often in a Kubernetes environment nodes are not addressable from outside the cluster. VSCode will thus be unable to remotely connect to port 5678 on a Kubernetes node. In order to do this you need to create a tunnel that port forwards 5678 to your local machine.
 
 ```
 kubectl port-forward  pod/superset-<some random id> 5678:5678
@@ -1341,7 +1348,7 @@ Chart parameters are stored as a JSON encoded string the `slices.params` column
 
 The following tables provide a non-exhausive list of the various fields which can be present in the JSON object grouped by the Explorer pane sections. These values were obtained by extracting the distinct fields from a legacy deployment consisting of tens of thousands of charts and thus some fields may be missing whilst others may be deprecated.
 
-Note not all fields are correctly catagorized. The fields vary based on visualization type and may apprear in different sections depending on the type. Verified deprecated columns may indicate a missing migration and/or prior migrations which were unsucessful and thus future work may be required to clean up the form-data.
+Note not all fields are correctly categorized. The fields vary based on visualization type and may apprear in different sections depending on the type. Verified deprecated columns may indicate a missing migration and/or prior migrations which were unsuccessful and thus future work may be required to clean up the form-data.
 
 ### Datasource & Chart Type
 

Dockerfile

@@ -57,13 +57,11 @@ ENV BUILD_CMD=${NPM_BUILD_CMD}
 RUN mkdir -p /app/superset-frontend
 RUN mkdir -p /app/superset/assets
 COPY ./docker/frontend-mem-nag.sh /
-COPY ./superset-frontend/package* /app/superset-frontend/
+COPY ./superset-frontend /app/superset-frontend
 RUN /frontend-mem-nag.sh \
         && cd /app/superset-frontend \
         && npm ci
 
-# Next, copy in the rest and let webpack do its thing
-COPY ./superset-frontend /app/superset-frontend
 # This seems to be the most expensive step
 RUN cd /app/superset-frontend \
         && npm run ${BUILD_CMD} \
@@ -92,6 +90,7 @@ RUN mkdir -p ${PYTHONPATH} \
             default-libmysqlclient-dev \
             libsasl2-modules-gssapi-mit \
             libpq-dev \
+            libecpg-dev \
         && rm -rf /var/lib/apt/lists/*
 
 COPY --from=superset-py /usr/local/lib/python3.8/site-packages/ /usr/local/lib/python3.8/site-packages/