From 38a3fbdc33292204bcb28f3e433fda7561035fb6 Mon Sep 17 00:00:00 2001 From: EugeneTorap Date: Wed, 16 Nov 2022 21:53:15 +0300 Subject: [PATCH] feat: use a new official CH driver: clickhouse-connect (#22039) --- docs/docs/databases/clickhouse.mdx | 14 +++--- .../databases/installing-database-drivers.mdx | 2 +- setup.py | 2 +- .../db_engine_specs/clickhouse_tests.py | 47 ------------------- .../db_engine_specs/test_clickhouse.py | 46 ++++++++++++++++++ 5 files changed, 55 insertions(+), 56 deletions(-) delete mode 100644 tests/integration_tests/db_engine_specs/clickhouse_tests.py create mode 100644 tests/unit_tests/db_engine_specs/test_clickhouse.py diff --git a/docs/docs/databases/clickhouse.mdx b/docs/docs/databases/clickhouse.mdx index 7afcc50c15cd9..424e50da9352c 100644 --- a/docs/docs/databases/clickhouse.mdx +++ b/docs/docs/databases/clickhouse.mdx @@ -10,33 +10,33 @@ version: 1 To use ClickHouse with Superset, you will need to add the following Python library: ``` -clickhouse-sqlalchemy>=0.2.2 +clickhouse-connect>=0.4.1 ``` If running Superset using Docker Compose, add the following to your `./docker/requirements-local.txt` file: ``` -clickhouse-sqlalchemy>=0.2.2 +clickhouse-connect>=0.4.1 ``` The recommended connector library for ClickHouse is -[sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse). +[clickhouse-connect](https://github.com/ClickHouse/clickhouse-connect). The expected connection string is formatted as follows: ``` -clickhouse+native://:@:/[?options…]clickhouse://{username}:{password}@{hostname}:{port}/{database} +clickhousedb://:@:/[?options…]clickhouse://{username}:{password}@{hostname}:{port}/{database} ``` Here's a concrete example of a real connection string: ``` -clickhouse+native://demo:demo@github.demo.trial.altinity.cloud/default?secure=true +clickhousedb://demo:demo@github.demo.trial.altinity.cloud/default?secure=true ``` -If you're using Clickhouse locally on your computer, you can get away with using a native protocol URL that +If you're using Clickhouse locally on your computer, you can get away with using a http protocol URL that uses the default user without a password (and doesn't encrypt the connection): ``` -clickhouse+native://localhost/default +clickhousedb://localhost/default ``` diff --git a/docs/docs/databases/installing-database-drivers.mdx b/docs/docs/databases/installing-database-drivers.mdx index c2dc1159e6566..01c85bc0e12fc 100644 --- a/docs/docs/databases/installing-database-drivers.mdx +++ b/docs/docs/databases/installing-database-drivers.mdx @@ -35,7 +35,7 @@ A list of some of the recommended packages. | [Ascend.io](/docs/databases/ascend) | `pip install impyla` | `ascend://{username}:{password}@{hostname}:{port}/{database}?auth_mechanism=PLAIN;use_ssl=true` | | [Azure MS SQL](/docs/databases/sql-server) | `pip install pymssql` | `mssql+pymssql://UserName@presetSQL:TestPassword@presetSQL.database.windows.net:1433/TestSchema` | | [Big Query](/docs/databases/bigquery) | `pip install pybigquery` | `bigquery://{project_id}` | -| [ClickHouse](/docs/databases/clickhouse) | `pip install clickhouse-sqlalchemy` | `clickhouse+native://{username}:{password}@{hostname}:{port}/{database}` | +| [ClickHouse](/docs/databases/clickhouse) | `pip install clickhouse-connect` | `clickhousedb://{username}:{password}@{hostname}:{port}/{database}` | | [CockroachDB](/docs/databases/cockroachdb) | `pip install cockroachdb` | `cockroachdb://root@{hostname}:{port}/{database}?sslmode=disable` | | [Dremio](/docs/databases/dremio) | `pip install sqlalchemy_dremio` | `dremio://user:pwd@host:31010/` | | [Elasticsearch](/docs/databases/elasticsearch) | `pip install elasticsearch-dbapi` | `elasticsearch+http://{user}:{password}@{host}:9200/` | diff --git a/setup.py b/setup.py index fb05f2248c2f9..d2b950c6a4cd9 100644 --- a/setup.py +++ b/setup.py @@ -130,7 +130,7 @@ def get_git_sha() -> str: "pybigquery>=0.4.10", "google-cloud-bigquery>=2.4.0", ], - "clickhouse": ["clickhouse-sqlalchemy>=0.2.2, <0.3"], + "clickhouse": ["clickhouse-connect>=0.4.1, <0.5"], "cockroachdb": ["cockroachdb>=0.3.5, <0.4"], "cors": ["flask-cors>=2.0.0"], "crate": ["crate[sqlalchemy]>=0.26.0, <0.27"], diff --git a/tests/integration_tests/db_engine_specs/clickhouse_tests.py b/tests/integration_tests/db_engine_specs/clickhouse_tests.py deleted file mode 100644 index c8019a8758db7..0000000000000 --- a/tests/integration_tests/db_engine_specs/clickhouse_tests.py +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from unittest import mock - -import pytest - -from superset.db_engine_specs.clickhouse import ClickHouseEngineSpec -from superset.db_engine_specs.exceptions import SupersetDBAPIDatabaseError -from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec - - -class TestClickHouseDbEngineSpec(TestDbEngineSpec): - def test_convert_dttm(self): - dttm = self.get_dttm() - - self.assertEqual( - ClickHouseEngineSpec.convert_dttm("DATE", dttm), "toDate('2019-01-02')" - ) - - self.assertEqual( - ClickHouseEngineSpec.convert_dttm("DATETIME", dttm), - "toDateTime('2019-01-02 03:04:05')", - ) - - def test_execute_connection_error(self): - from urllib3.exceptions import NewConnectionError - - cursor = mock.Mock() - cursor.execute.side_effect = NewConnectionError( - "Dummypool", message="Exception with sensitive data" - ) - with pytest.raises(SupersetDBAPIDatabaseError) as ex: - ClickHouseEngineSpec.execute(cursor, "SELECT col1 from table1") diff --git a/tests/unit_tests/db_engine_specs/test_clickhouse.py b/tests/unit_tests/db_engine_specs/test_clickhouse.py new file mode 100644 index 0000000000000..ca01c304f6327 --- /dev/null +++ b/tests/unit_tests/db_engine_specs/test_clickhouse.py @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from datetime import datetime +from unittest import mock + +import pytest + +from tests.unit_tests.fixtures.common import dttm + + +def test_convert_dttm(dttm: datetime) -> None: + from superset.db_engine_specs.clickhouse import ClickHouseEngineSpec + + assert ClickHouseEngineSpec.convert_dttm("DATE", dttm) == "toDate('2019-01-02')" + assert ( + ClickHouseEngineSpec.convert_dttm("DATETIME", dttm) + == "toDateTime('2019-01-02 03:04:05')" + ) + + +def test_execute_connection_error() -> None: + from urllib3.exceptions import NewConnectionError + + from superset.db_engine_specs.clickhouse import ClickHouseEngineSpec + from superset.db_engine_specs.exceptions import SupersetDBAPIDatabaseError + + cursor = mock.Mock() + cursor.execute.side_effect = NewConnectionError( + "Dummypool", "Exception with sensitive data" + ) + with pytest.raises(SupersetDBAPIDatabaseError) as ex: + ClickHouseEngineSpec.execute(cursor, "SELECT col1 from table1")