New ClickHouse Driver for MetaBase

.gitignore

@@ -0,0 +1,9 @@
+\#*\#
+.\#*
+/target
+*.jar
+*.class
+/.lein-env
+/.lein-failures
+/.lein-plugins
+/.lein-repl-history

project.clj

@@ -0,0 +1,18 @@
+(defproject metabase/clickhouse-driver "1.0.0-SNAPSHOT-0.1.48"
+  :min-lein-version "2.5.0"
+
+  :dependencies
+  [[ru.yandex.clickhouse/clickhouse-jdbc "0.1.50"
+    :exclusions [com.fasterxml.jackson.core/jackson-core
+                 org.slf4j/slf4j-api]]]
+
+  :profiles
+  {:provided
+   {:dependencies [[metabase-core "1.0.0-SNAPSHOT"]]}
+
+   :uberjar
+   {:auto-clean    true
+    :aot           :all
+    :javac-options ["-target" "1.8", "-source" "1.8"]
+    :target-path   "target/%s"
+    :uberjar-name  "clickhouse.metabase-driver.jar"}})

resources/metabase-plugin.yaml

@@ -0,0 +1,29 @@
+info:
+  name: Metabase ClickHouse Driver
+  version: 1.0.0-SNAPSHOT-0.1.48
+  description: Allows Metabase to connect to ClickHouse databases.
+driver:
+  name: clickhouse
+  display-name: ClickHouse
+  lazy-load: true
+  parent: sql-jdbc
+  connection-properties:
+    - name: dbname
+      display-name: Database Name
+      placeholder: default
+    - host
+    - merge:
+        - port
+        - default: 8123
+    - user
+    - password
+    - ssl
+    - merge:
+        - additional-options
+        - placeholder: "max_rows_to_group_by=42"
+  connection-properties-include-tunnel-config: true
+init:
+  - step: load-namespace
+    namespace: metabase.driver.clickhouse
+  - step: register-jdbc-driver
+    class: ru.yandex.clickhouse.ClickHouseDriver

src/metabase/driver/clickhouse.clj

@@ -0,0 +1,263 @@
+(ns metabase.driver.clickhouse
+  "Driver for ClickHouse databases"
+  (:require [clojure.java.jdbc :as jdbc]
+            [clojure.string :as string]
+            [honeysql.core :as hsql]
+            [metabase
+             [config :as config]
+             [driver :as driver]
+             [util :as u]]
+            [metabase.driver
+             [common :as driver.common]
+             [sql :as sql]]
+            [metabase.driver.sql-jdbc
+             [common :as sql-jdbc.common]
+             [connection :as sql-jdbc.conn]
+             [execute :as sql-jdbc.execute]
+             [sync :as sql-jdbc.sync]]
+            [metabase.driver.sql.query-processor :as sql.qp]
+            [metabase.driver.sql.util.unprepare :as unprepare]
+            [metabase.util
+             [date :as du]
+             [honeysql-extensions :as hx]]
+            [schema.core :as sc])
+  (:import [java.sql DatabaseMetaData Time]
+           java.util.Date))
+
+(driver/register! :clickhouse, :parent :sql-jdbc)
+
+(def ^:private database-type->base-type
+  (sql-jdbc.sync/pattern-based-database-type->base-type
+   [
+    [#"Array"       :type/*]
+    [#"DateTime"    :type/DateTime]
+    [#"Date"        :type/Date]
+    [#"Decimal"     :type/Decimal]
+    [#"Enum8"       :type/*]
+    [#"Enum16"      :type/*]
+    [#"FixedString" :type/Text]
+    [#"Float32"     :type/Float]
+    [#"Float64"     :type/Float]
+    [#"Int8"        :type/Integer]
+    [#"Int16"       :type/Integer]
+    [#"Int32"       :type/Integer]
+    [#"Int64"       :type/BigInteger]
+    [#"String"      :type/Text]
+    [#"Tuple"       :type/*]
+    [#"UInt8"       :type/Integer]
+    [#"UInt16"      :type/Integer]
+    [#"UInt32"      :type/Integer]
+    [#"UInt64"      :type/BigInteger]
+    [#"UUID"        :type/UUID]]))
+
+(defmethod sql-jdbc.sync/database-type->base-type :clickhouse [_ database-type]
+  (database-type->base-type
+   (string/replace (name database-type) #"(?:Nullable|LowCardinality)\((\S+)\)" "$1")))
+
+(defmethod sql-jdbc.sync/excluded-schemas :clickhouse [_]
+  #{"system"})
+
+(defmethod sql-jdbc.conn/connection-details->spec :clickhouse
+  [_ {:keys [user password dbname host port ssl]
+      :or   {user "default", password "", dbname "default", host "localhost", port "8123"}
+      :as   details}]
+  (-> {:classname                      "ru.yandex.clickhouse.ClickHouseDriver"
+       :subprotocol                    "clickhouse"
+       :subname                        (str "//" host ":" port "/" dbname)
+       :password                       password
+       :user                           user
+       :ssl                            (boolean ssl)
+       :use_server_time_zone_for_dates true}
+      (sql-jdbc.common/handle-additional-options details, :seperator-style :url)))
+
+(defn- modulo [a b]
+  (hsql/call :modulo a b))
+
+(defn- to-relative-day-num [expr]
+  (hsql/call :toRelativeDayNum (hsql/call :toDateTime expr)))
+
+(defn- to-relative-month-num [expr]
+  (hsql/call :toRelativeMonthNum (hsql/call :toDateTime expr)))
+
+(defn- to-start-of-year [expr]
+  (hsql/call :toStartOfYear (hsql/call :toDateTime expr)))
+
+(defn- to-day-of-year [expr]
+  (hx/+
+   (hx/- (to-relative-day-num expr)
+          (to-relative-day-num (to-start-of-year expr)))
+   1))
+
+(defn- to-week-of-year [expr]
+  (hsql/call :toUInt8 (hsql/call :formatDateTime (hx/+ (hsql/call :toDate expr) 1) "%V")))
+
+(defn- to-month-of-year [expr]
+  (hx/+
+   (hx/- (to-relative-month-num expr)
+          (to-relative-month-num (to-start-of-year expr)))
+   1))
+
+(defn- to-quarter-of-year [expr]
+  (hsql/call :ceil (hx//
+                    (hx/+
+                     (hx/- (to-relative-month-num expr)
+                           (to-relative-month-num (to-start-of-year expr)))
+                     1)
+                    3)))
+
+(defn- to-start-of-week [expr]
+  ;; ClickHouse weeks start on Monday
+  (hx/- (hsql/call :toMonday (hx/+ (hsql/call :toDate expr) 1)) 1))
+
+(defn- to-start-of-minute [expr]
+  (hsql/call :toStartOfMinute (hsql/call :toDateTime expr)))
+
+(defn- to-start-of-hour [expr]
+  (hsql/call :toStartOfHour (hsql/call :toDateTime expr)))
+
+(defn- to-hour [expr]
+  (hsql/call :toHour (hsql/call :toDateTime expr)))
+
+(defn- to-minute [expr]
+  (hsql/call :toMinute (hsql/call :toDateTime expr)))
+
+(defn- to-year [expr]
+  (hsql/call :toYear (hsql/call :toDateTime expr)))
+
+(defn- to-day [expr]
+  (hsql/call :toDate expr))
+
+(defn- to-day-of-week [expr]
+  ;; ClickHouse weeks start on Monday
+  (hx/+ (modulo (hsql/call :toDayOfWeek (hsql/call :toDateTime expr)) 7) 1))
+
+(defn- to-day-of-month [expr]
+  (hsql/call :toDayOfMonth (hsql/call :toDateTime expr)))
+
+(defn- to-start-of-month [expr]
+  (hsql/call :toStartOfMonth (hsql/call :toDateTime expr)))
+
+(defn- to-start-of-quarter [expr]
+  (hsql/call :toStartOfQuarter (hsql/call :toDateTime expr)))
+
+(defmethod sql.qp/date [:clickhouse :default]         [_ _ expr] expr)
+(defmethod sql.qp/date [:clickhouse :minute]          [_ _ expr] (to-start-of-minute expr))
+(defmethod sql.qp/date [:clickhouse :minute-of-hour]  [_ _ expr] (to-minute expr))
+(defmethod sql.qp/date [:clickhouse :hour]            [_ _ expr] (to-start-of-hour expr))
+(defmethod sql.qp/date [:clickhouse :hour-of-day]     [_ _ expr] (to-hour expr))
+(defmethod sql.qp/date [:clickhouse :day-of-week]     [_ _ expr] (to-day-of-week expr))
+(defmethod sql.qp/date [:clickhouse :day-of-month]    [_ _ expr] (to-day-of-month expr))
+(defmethod sql.qp/date [:clickhouse :day-of-year]     [_ _ expr] (to-day-of-year expr))
+(defmethod sql.qp/date [:clickhouse :week-of-year]    [_ _ expr] (to-week-of-year expr))
+(defmethod sql.qp/date [:clickhouse :month]           [_ _ expr] (to-start-of-month expr))
+(defmethod sql.qp/date [:clickhouse :month-of-year]   [_ _ expr] (to-month-of-year expr))
+(defmethod sql.qp/date [:clickhouse :quarter-of-year] [_ _ expr] (to-quarter-of-year expr))
+(defmethod sql.qp/date [:clickhouse :year]            [_ _ expr] (to-year expr))
+
+(defmethod sql.qp/date [:clickhouse :day]             [_ _ expr] (to-day expr))
+(defmethod sql.qp/date [:clickhouse :week]            [_ _ expr] (to-start-of-week expr))
+(defmethod sql.qp/date [:clickhouse :quarter]         [_ _ expr] (to-start-of-quarter expr))
+
+(defmethod sql.qp/unix-timestamp->timestamp [:clickhouse :seconds] [_ _ expr]
+  (hsql/call :toDateTime expr))
+
+(defmethod unprepare/unprepare-value [:clickhouse Date] [_ value]
+  (format "parseDateTimeBestEffort('%s')" (du/date->iso-8601 value)))
+
+(prefer-method unprepare/unprepare-value [:clickhouse Date] [:sql Time])
+
+;; Parameter values for date ranges are set via TimeStamp. This confuses the ClickHouse
+;; server, so we override the default formatter
+(sc/defmethod sql/->prepared-substitution [:clickhouse java.util.Date] :- sql/PreparedStatementSubstitution
+  [_ date]
+  (sql/make-stmt-subs "?" [(du/format-date "yyyy-MM-dd" date)]))
+
+;; ClickHouse doesn't support `TRUE`/`FALSE`; it uses `1`/`0`, respectively;
+;; convert these booleans to numbers.
+(defmethod sql.qp/->honeysql [:clickhouse Boolean]
+  [_ bool]
+  (if bool 1 0))
+
+(defmethod sql.qp/->honeysql [:clickhouse :stddev]
+  [driver [_ field]]
+  (hsql/call :stddevSamp (sql.qp/->honeysql driver field)))
+
+(defmethod sql.qp/->honeysql [:clickhouse :/]
+  [driver args]
+  (let [args (for [arg args]
+               (hsql/call :toFloat64 (sql.qp/->honeysql driver arg)))]
+    ((get-method sql.qp/->honeysql [:sql :/]) driver args)))
+
+(defmethod sql.qp/quote-style :clickhouse [_] :mysql)
+
+
+;; ClickHouse aliases are globally usable. Once an alias is introduced, we
+;; can not refer to the same field by qualified name again, unless we mean
+;; it. See https://clickhouse.yandex/docs/en/query_language/syntax/#peculiarities-of-use
+;; We add a suffix to make the reference in the query unique.
+(defmethod sql.qp/field->alias :clickhouse [_ field]
+  (str (:name field) "_mb_alias"))
+
+;; See above. We are removing the artificial alias suffix
+(defmethod driver/execute-query :clickhouse [driver query]
+  (update-in
+   (sql-jdbc.execute/execute-query :sql-jdbc query)
+   [:columns]
+   (fn [columns]
+     (mapv (fn [column]
+             (if (string/ends-with? column "_mb_alias")
+               (subs column 0 (string/last-index-of column "_mb_alias"))
+               column))
+           columns))))
+
+(defn- get-tables
+  "Fetch a JDBC Metadata ResultSet of tables in the DB, optionally limited to ones belonging to a given schema."
+  [^DatabaseMetaData metadata, ^String schema-or-nil, ^String db-name-or-nil]
+  (vec
+   (jdbc/metadata-result
+    (.getTables metadata db-name-or-nil schema-or-nil "%" ; tablePattern "%" = match all tables
+                (into-array String ["TABLE", "VIEW", "FOREIGN TABLE", "MATERIALIZED VIEW"])))))
+
+(defn- post-filtered-active-tables
+  [driver, ^DatabaseMetaData metadata, & [db-name-or-nil]]
+  (set (for [table   (filter #(not (contains? (sql-jdbc.sync/excluded-schemas driver) (:table_schem %)))
+                             (get-tables metadata db-name-or-nil nil))]
+         (let [remarks (:remarks table)]
+           {:name        (:table_name  table)
+            :schema      (:table_schem table)
+            :description (when-not (string/blank? remarks)
+                           remarks)}))))
+
+(defn- ->spec [db-or-id-or-spec]
+  (if (u/id db-or-id-or-spec)
+    (sql-jdbc.conn/db->pooled-connection-spec db-or-id-or-spec)
+    db-or-id-or-spec))
+
+;; ClickHouse exposes databases as schemas, but MetaBase sees
+;; schemas as sub-entities of a database, at least the fast-active-tables
+;; implementation would lead to duplicate tables because it iterates
+;; over all schemas of the current dbs and then retrieves all
+;; tables of a schema
+(defmethod driver/describe-database :clickhouse
+  [driver db-or-id-or-spec]
+  (jdbc/with-db-metadata [metadata (->spec db-or-id-or-spec)]
+    {:tables (post-filtered-active-tables
+              ;; TODO: this only covers the db case, not id or spec
+              driver metadata (get-in db-or-id-or-spec [:details :db]))}))
+
+(defmethod driver.common/current-db-time-date-formatters :clickhouse [_]
+  (driver.common/create-db-time-formatters "yyyy-MM-dd HH:mm.ss"))
+
+(defmethod driver.common/current-db-time-native-query :clickhouse [_]
+  "SELECT formatDateTime(NOW(), '%F %R.%S')")
+
+(defmethod driver/current-db-time :clickhouse [& args]
+  (apply driver.common/current-db-time args))
+
+(defmethod driver/display-name :clickhouse [_] "ClickHouse")
+
+(defmethod driver/supports? [:clickhouse :foreign-keys] [_ _] false)
+
+;; TODO: Nested queries are actually supported, but I do not know how
+;; to make the driver use correct aliases per sub-query
+(defmethod driver/supports? [:clickhouse :nested-queries] [_ _] false)

test/metabase/driver/clickhouse_test.clj

@@ -0,0 +1,45 @@
+(ns metabase.driver.clickhouse-test
+  "Tests for specific behavior of the ClickHouse driver."
+  (:require [expectations :refer :all]
+            [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+            [metabase.util :as u]
+            [metabase.query-processor-test :refer :all]
+            [metabase.test.data :as data]
+            [metabase.test.data
+             [datasets :as datasets]
+             [interface :as tx]]
+            [metabase.test.util :as tu]))
+
+(datasets/expect-with-driver :clickhouse
+  "UTC"
+  (tu/db-timezone-id))
+
+(datasets/expect-with-driver :clickhouse
+  21.0
+  (-> (data/with-temp-db
+    [_
+     (tx/create-database-definition "ClickHouse with Decimal Field"
+       ["test-data"
+        [{:field-name "my_money", :base-type {:native "Decimal(12,3)"}}]
+        [[1.0] [23.0] [42.0] [42.0]]])]
+    (data/run-mbql-query test-data
+                         {:expressions {:divided [:/ $my_money 2]}
+                          :filter      [:> [:expression :divided] 1.0]
+                          :breakout    [[:expression :divided]]
+                          :order-by    [[:desc [:expression :divided]]]
+                          :limit       1}))
+      first-row last float))
+
+(expect
+  {:classname                      "ru.yandex.clickhouse.ClickHouseDriver"
+   :subprotocol                    "clickhouse"
+   :subname                        "//localhost:8123/foo?sessionTimeout=42"
+   :user                           "default"
+   :password                       ""
+   :ssl                            false
+   :use_server_time_zone_for_dates true}
+  (sql-jdbc.conn/connection-details->spec :clickhouse
+    {:host               "localhost"
+     :port               "8123"
+     :dbname             "foo"
+     :additional-options "sessionTimeout=42"}))