Merge pull request #736 from zhicwu/new-client

New Java client and gRPC support
ClickHouse · Oct 7, 2021 · ed43dfe · ed43dfe
2 parents 0c615ba + 9686cd4
commit ed43dfe
Show file tree

Hide file tree

Showing 273 changed files with 35,353 additions and 1,945 deletions.
diff --git a/.github/toolchains.xml b/.github/toolchains.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<toolchains>
+    <!--
+    <toolchain>
+        <type>jdk</type>
+        <provides>
+            <version>1.8</version>
+        </provides>
+        <configuration>
+            <jdkHome>/path/to/jdk/1.8</jdkHome>
+        </configuration>
+    </toolchain>
+    <toolchain>
+        <type>jdk</type>
+        <provides>
+            <version>9</version>
+        </provides>
+        <configuration>
+            <jdkHome>/path/to/jdk/9</jdkHome>
+        </configuration>
+    </toolchain>
+    -->
+    <toolchain>
+        <type>jdk</type>
+        <provides>
+            <version>11</version>
+        </provides>
+        <configuration>
+            <jdkHome>${env.JAVA_HOME}</jdkHome>
+        </configuration>
+    </toolchain>
+</toolchains>
diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml
@@ -73,5 +73,6 @@ jobs:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: |
           find . -type f -name "log4j.*" -exec rm -fv '{}' \;
-          mvn -q --batch-mode -Panalysis verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
+          mvn -q --batch-mode --global-toolchains .github/toolchains.xml -Drelease \
+            -Panalysis verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
         continue-on-error: true
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -61,7 +61,6 @@ jobs:
         run: |
           mvn --batch-mode --update-snapshots -q -DskipTests install
           cd clickhouse-benchmark
-          mvn --batch-mode --update-snapshots install
           java -DclickhouseVersion="21.8" -jar target/benchmarks.jar -rf text -p client=clickhouse-jdbc Basic
           echo "BENCHMARK_REPORT<<EOF" >> $GITHUB_ENV
           cat jmh-result.text >> $GITHUB_ENV

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -57,6 +57,12 @@ jobs:
           key: ${{ runner.os }}-build-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-build-
-      - name: Build with Maven
+      - name: Build
         run: |
-          mvn --batch-mode --update-snapshots -Drelease -DclickhouseVersion=${{ matrix.clickhouse }} verify
+          mvn --batch-mode --update-snapshots -DclickhouseVersion=${{ matrix.clickhouse }} verify
+        if: matrix.java == '8'
+      - name: Build in release mode
+        run: |
+          mvn --batch-mode --update-snapshots --global-toolchains .github/toolchains.xml \
+            -Drelease -DclickhouseVersion=${{ matrix.clickhouse }} verify
+        if: matrix.java != '8'
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -19,7 +19,7 @@ jobs:
       - name: Install Java and Maven
         uses: actions/setup-java@v1
         with:
-          java-version: 8
+          java-version: 11
       - name: Update pom files and reduce logs
         run: |
           find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ github.event.inputs.version }}|g' \

diff --git a/.github/workflows/timezone.yml b/.github/workflows/timezone.yml
@@ -43,10 +43,10 @@ jobs:
           git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \
             origin pull/${{ github.event.inputs.pr }}/merge:merged-pr && git checkout merged-pr
         if: github.event.inputs.pr != ''
-      - name: Set up JDK 11
+      - name: Set up JDK 8
         uses: actions/setup-java@v1
         with:
-          java-version: 11
+          java-version: 8
       - name: Cache maven dependencies
         uses: actions/cache@v2
         with:

diff --git a/LICENSE b/LICENSE
@@ -1,4 +1,3 @@
-Copyright 2016-2021 Yandex LLC
 
                                  Apache License
                            Version 2.0, January 2004
@@ -188,7 +187,7 @@ Copyright 2016-2021 Yandex LLC
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2016-2021 Yandex LLC
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

diff --git a/README.md b/README.md
@@ -1,27 +1,83 @@
-ClickHouse JDBC driver
-===============
+# ClickHouse Java Client & JDBC Driver
+
 [![clickhouse-jdbc](https://maven-badges.herokuapp.com/maven-central/ru.yandex.clickhouse/clickhouse-jdbc/badge.svg)](https://maven-badges.herokuapp.com/maven-central/ru.yandex.clickhouse/clickhouse-jdbc) ![Build Status(/~https://github.com/ClickHouse/clickhouse-jdbc/workflows/Build/badge.svg)](/~https://github.com/ClickHouse/clickhouse-jdbc/workflows/Build/badge.svg) [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=ClickHouse_clickhouse-jdbc&metric=coverage)](https://sonarcloud.io/dashboard?id=ClickHouse_clickhouse-jdbc)
 
-This is a basic and restricted implementation of jdbc driver for ClickHouse.
-It has support of a minimal subset of features to be usable.
+Java client and JDBC driver for ClickHouse.
+
+## Usage
+
+### Java Client
+
+Use Java client when you prefer async and more "direct" way to communicate with ClickHouse. JDBC driver is actually a thin wrapper of the Java client.
+
+```xml
+<dependency>
+    <groupId>com.clickhouse</groupId>
+    <!-- you'll be able to use clickhouse-http-client and clickhouse-tcp-client as well in the near future -->
+    <artifactId>clickhouse-grpc-client</artifactId>
+    <version>0.3.2</version>
+</dependency>
+```
+
+Example:
+
+```Java
+// declare a server to connect to
+ClickHouseNode server = ClickHouseNode.of("server.domain", ClickHouseProtocol.GRPC, 9100, "my_db");
+
+// run multiple queries in one go and wait until it's finished
+ClickHouseClient.send(server,
+    "create database if not exists test",
+    "use test", // change current database from my_db to test
+    "create table if not exists test_table(s String) engine=Memory",
+    "insert into test_table values('1')('2')('3')",
+    "select * from test_table limit 1",
+    "truncate table test_table",
+    "drop table if exists test_table").get();
+
+// query with named parameters
+try (ClickHouseClient client = ClickHouseClient.newInstance(ClickHouseProtocol.GRPC);
+    ClickHouseResponse resp = client.connect(server)
+        .format(ClickHouseFormat.RowBinaryWithNamesAndTypes).set("send_logs_level", "trace")
+        .query("select id, name from some_table where id in :ids and name like :name").params(Arrays.asList(1,2,3), "%key%").execute().get()) {
+    // you can also use resp.recordStream() as well
+    for (ClickHouseRecord record : resp.records()) {
+        int id = record.getValue(0).asInteger();
+        String name = record.getValue(1).asString();
+    }
+
+    ClickHouseResponseSummary summary = resp.getSummary();
+    long totalRows = summary.getRows();
+}
+
+// load data with custom writer
+ClickHouseClient.load(server, "target_table", ClickHouseFormat.TabSeparated,
+    ClickHouseCompression.NONE, new ClickHouseWriter() {
+        @Override
+        public void write(OutputStream output) throws IOException {
+            output.write("1\t\\N\n".getBytes());
+            output.write("2\t123".getBytes());
+        }
+    }).get();
+```
 
+### JDBC Driver
 
-### Usage
 ```xml
 <dependency>
+    <!-- groupId and package name will be changed to com.clickhouse starting from 0.4.0 -->
     <groupId>ru.yandex.clickhouse</groupId>
     <artifactId>clickhouse-jdbc</artifactId>
     <version>0.3.2</version>
 </dependency>
 ```
 
-URL syntax: 
-`jdbc:clickhouse://<host>:<port>[/<database>]`, e.g. `jdbc:clickhouse://localhost:8123/test`
+URL syntax: `jdbc:clickhouse://<host>:<port>[/<database>[?param1=value1&param2=value2]]`, e.g. `jdbc:clickhouse://localhost:8123/test?socket_timeout=120000`
 
-JDBC Driver Class:
-`ru.yandex.clickhouse.ClickHouseDriver`
+JDBC Driver Class: `ru.yandex.clickhouse.ClickHouseDriver` (will be changed to `com.clickhouse.jdbc.ClickHouseDriver` starting from 0.4.0)
 
 For example:
+
 ```java
 String url = "jdbc:clickhouse://localhost:8123/test";
 ClickHouseProperties properties = new ClickHouseProperties();
@@ -47,37 +103,40 @@ try (ClickHouseConnection conn = dataSource.getConnection();
 
 Additionally, if you have a few instances, you can use `BalancedClickhouseDataSource`.
 
-
 ### Extended API
+
 In order to provide non-JDBC complaint data manipulation functionality, proprietary API exists.
 Entry point for API is `ClickHouseStatement#write()` method.
 
 #### Importing file into table
+
 ```java
 import ru.yandex.clickhouse.ClickHouseStatement;
 ClickHouseStatement sth = connection.createStatement();
 sth
     .write() // Write API entrypoint
     .table("default.my_table") // where to write data
     .option("format_csv_delimiter", ";") // specific param
-    .data(new File("/path/to/file.csv.gz"), ClickHouseFormat.CSV, ClickHouseCompression.gzip) // specify input     
+    .data(new File("/path/to/file.csv.gz"), ClickHouseFormat.CSV, ClickHouseCompression.gzip) // specify input
     .send();
 ```
 
 #### Configurable send
+
 ```java
 import ru.yandex.clickhouse.ClickHouseStatement;
 ClickHouseStatement sth = connection.createStatement();
 sth
     .write()
     .sql("INSERT INTO default.my_table (a,b,c)")
     .data(new MyCustomInputStream(), ClickHouseFormat.JSONEachRow)
-    .dataCompression(ClickHouseCompression.brotli)    
+    .dataCompression(ClickHouseCompression.brotli)
     .addDbParam(ClickHouseQueryParam.MAX_PARALLEL_REPLICAS, 2)
     .send();
 ```
 
 #### Send data in binary formatted with custom user callback
+
 ```java
 import ru.yandex.clickhouse.ClickHouseStatement;
 ClickHouseStatement sth = connection.createStatement();
@@ -93,19 +152,55 @@ sth.write().send("INSERT INTO test.writer", new ClickHouseStreamCallback() {
 ClickHouseFormat.RowBinary); // RowBinary or Native are supported
 ```
 
+## Compatibility
+
+Java 8 or higher is required in order to use Java client and/or JDBC driver.
 
-### Supported Server Versions
-All [active releases](../ClickHouse/pulls?q=is%3Aopen+is%3Apr+label%3Arelease) are supported. You can still use the driver for older versions like 18.14 or 19.16 but please keep in mind that they're no longer supported.
+### Data Format
 
+`RowBinary` is preferred format in Java client, while JDBC driver uses `TabSeparated`.
 
-### Compiling with maven
-The driver is built with maven.
-`mvn package -DskipTests=true`
+### Data Type
+
+| Data Type(s)       | Java Client | JDBC Driver                | Remark                                                                |
+| ------------------ | ----------- | -------------------------- | --------------------------------------------------------------------- |
+| Date\*             | Y           | Y                          |                                                                       |
+| DateTime\*         | Y           | Y                          |                                                                       |
+| Decimal\*          | Y           | Y                          | `SET output_format_decimal_trailing_zeros=1` in 21.9+ for consistency |
+| Enum\*             | Y           | Treated as integer         |
+| Int*, UInt*        | Y           | UInt64 is mapped to `long` |
+| Geo Types          | Y           | N                          |                                                                       |
+| AggregatedFunction | N           | N                          | Partially supported                                                   |
+| Array              | Y           | N                          |                                                                       |
+| Map                | Y           | Y                          |                                                                       |
+| Nested             | Y           | N                          |                                                                       |
+| Tuple              | Y           | N                          |                                                                       |
+
+### Server Version
+
+All [active releases](../ClickHouse/pulls?q=is%3Aopen+is%3Apr+label%3Arelease) are supported. You can still use the JDBC driver for older versions like 18.14 or 19.16, but please keep in mind that they're no longer supported.
+
+## Build with Maven
+
+Use `mvn clean verify` to compile, test and generate packages if you're using JDK 8.
+
+If you want to make a multi-release jar file(see [JEP-238](https://openjdk.java.net/jeps/238)), you'd better use JDK 11 or higher version like 17 with below command line:
+
+```bash
+mvn --global-toolchains .github/toolchains.xml -Drelease clean verify
+```
 
-To build a jar with dependencies use
+## Testing
 
-`mvn package assembly:single -DskipTests=true`
+By default, docker container will be created automatically during integration test. You can pass system property like `-DclickhouseVersion=21.8` to test against specific version of ClickHouse.
 
+In the case you prefer to test against an existing server, please follow instructions below:
 
-### Build requirements
-In order to build the jdbc client one need to have jdk 1.8 or higher.
+-   make sure the server can be accessed using default account(`default` user without password), which has both DDL and DML privileges
+-   add below two configuration files to the existing server and expose all ports for external access
+    -   [ports.xml](./tree/master/clickhouse-client/src/test/resources/containers/clickhouse-server/config.d/ports.xml) - enable all ports
+    -   and [users.xml](./tree/master/clickhouse-client/src/test/resources/containers/clickhouse-server/users.d/users.xml) - accounts used for integration test
+-   put `test.properties` under either `test/resources` or `~/.m2/clickhouse` with content like below:
+    ```properties
+    clickhouseServer=127.0.0.1
+    ```