diff --git a/clickhouse-cli-client/README.md b/clickhouse-cli-client/README.md index e6e388b92..564b332ca 100644 --- a/clickhouse-cli-client/README.md +++ b/clickhouse-cli-client/README.md @@ -6,11 +6,11 @@ This is a thin wrapper of ClickHouse native command-line client. It provides an - native CLI client instead of pure Java implementation - an example of implementing SPI defined in `clickhouse-client` module -Either [clickhouse-client](https://clickhouse.com/docs/en/interfaces/cli/) or [docker](https://docs.docker.com/get-docker/) must be installed prior to use. And it's important to understand that this module uses sub-process(in addition to threads) and file-based streaming, meaning 1) it's not as fast as native CLI client or pure Java implementation, although it's close in the case of dumping and loading data; and 2) it's not suitable for scenarios like dealing with many queries in short period of time. +Either [clickhouse](https://clickhouse.com/docs/en/interfaces/cli/) or [docker](https://docs.docker.com/get-docker/) must be installed prior to use. And it's important to understand that this module uses sub-process(in addition to threads) and file-based streaming, meaning 1) it's not as fast as native CLI client or pure Java implementation, although it's close in the case of dumping and loading data; and 2) it's not suitable for scenarios like dealing with many queries in short period of time. ## Limitations and Known Issues -- Only `max_result_rows` and `result_overflow_mode` two settings are currently supported +- Only `max_result_rows`, `result_overflow_mode` and `readonly` 3 settings are currently supported - ClickHouseResponseSummary is always empty - see ClickHouse/ClickHouse#37241 - Session is not supported - see ClickHouse/ClickHouse#37308 @@ -28,10 +28,10 @@ Either [clickhouse-client](https://clickhouse.com/docs/en/interfaces/cli/) or [d ## Examples ```java -// make sure 'clickhouse-client' or 'docker' is in PATH before you start the program +// make sure 'clickhouse' or 'docker' is in PATH before you start the program // alternatively, configure CLI path in either Java system property or environment variable, for examples: -// CHC_CLICKHOUSE_CLI_PATH=/path/to/clickhouse-client CHC_DOCKER_CLI_PATH=/path/to/docker java MyProgram -// java -Dchc_clickhouse_cli_path=/path/to/clickhouse-client -Dchc_docker_cli_path=/path/to/docker MyProgram +// CHC_CLICKHOUSE_CLI_PATH=/path/to/clickhouse CHC_DOCKER_CLI_PATH=/path/to/docker java MyProgram +// java -Dchc_clickhouse_cli_path=/path/to/clickhouse -Dchc_docker_cli_path=/path/to/docker MyProgram // clickhouse-cli-client uses TCP protocol ClickHouseProtocol preferredProtocol = ClickHouseProtocol.TCP; diff --git a/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/ClickHouseCommandLine.java b/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/ClickHouseCommandLine.java index f11c21eeb..4595065f5 100644 --- a/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/ClickHouseCommandLine.java +++ b/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/ClickHouseCommandLine.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.io.OutputStream; import java.io.UncheckedIOException; +import java.net.ConnectException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -93,7 +94,7 @@ static void dockerCommand(ClickHouseConfig config, String hostDir, String contai cli = DEFAULT_DOCKER_CLI_PATH; } if (!check(timeout, cli, DEFAULT_CLI_ARG_VERSION)) { - throw new IllegalStateException("Docker command-line is not available: " + cli); + throw new UncheckedIOException(new ConnectException("Docker command-line is not available: " + cli)); } else { commands.add(cli); } @@ -111,7 +112,7 @@ static void dockerCommand(ClickHouseConfig config, String hostDir, String contai DEFAULT_CLI_ARG_VERSION) && !check(timeout, cli, "run", "--rm", "--name", str, "-v", hostDir + ':' + containerDir, "-d", img, "tail", "-f", "/dev/null")) { - throw new IllegalStateException("Failed to start new container: " + str); + throw new UncheckedIOException(new ConnectException("Failed to start new container: " + str)); } } } @@ -122,7 +123,7 @@ static void dockerCommand(ClickHouseConfig config, String hostDir, String contai } else { // create new container for each query if (!check(timeout, cli, "run", "--rm", img, DEFAULT_CLICKHOUSE_CLI_PATH, DEFAULT_CLIENT_OPTION, DEFAULT_CLI_ARG_VERSION)) { - throw new IllegalStateException("Invalid ClickHouse docker image: " + img); + throw new UncheckedIOException(new ConnectException("Invalid ClickHouse docker image: " + img)); } commands.add("run"); commands.add("--rm"); @@ -235,6 +236,10 @@ static Process startProcess(ClickHouseNode server, ClickHouseRequest request) if (value != null) { commands.add("--result_overflow_mode=".concat(value.toString())); } + value = settings.get("readonly"); + if (value != null) { + commands.add("--readonly=".concat(value.toString())); + } if ((boolean) config.getOption(ClickHouseCommandLineOption.USE_PROFILE_EVENTS)) { commands.add("--print-profile-events"); commands.add("--profile-events-delay-ms=-1"); diff --git a/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/config/ClickHouseCommandLineOption.java b/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/config/ClickHouseCommandLineOption.java index f824325d1..b8ef108a3 100644 --- a/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/config/ClickHouseCommandLineOption.java +++ b/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/config/ClickHouseCommandLineOption.java @@ -8,10 +8,10 @@ public enum ClickHouseCommandLineOption implements ClickHouseOption { /** * ClickHouse native command-line client path. Empty value is treated as - * 'clickhouse-client'. + * 'clickhouse'. */ CLICKHOUSE_CLI_PATH("clickhouse_cli_path", "", - "ClickHouse native command-line client path, empty value is treated as 'clickhouse-client'"), + "ClickHouse native command-line client path, empty value is treated as 'clickhouse'"), /** * ClickHouse docker image. Empty value is treated as * 'clickhouse/clickhouse-server'. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java index fbbd17951..a608e7689 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java @@ -151,7 +151,7 @@ static ClickHouseInputStream getResponseInputStream(ClickHouseConfig config, Inp /** * Gets piped input stream for reading data from response asynchronously. When - * {@code config} is null or {@code config.isAsync()} is faluse, this method is + * {@code config} is null or {@code config.isAsync()} is false, this method is * same as * {@link #getResponseInputStream(ClickHouseConfig, InputStream, Runnable)}. * diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClientBuilder.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClientBuilder.java index 77eda2f64..8e678e38e 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClientBuilder.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClientBuilder.java @@ -1,6 +1,9 @@ package com.clickhouse.client; import java.io.Serializable; +import java.io.UncheckedIOException; +import java.net.ConnectException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -27,25 +30,36 @@ */ public class ClickHouseClientBuilder { /** - * Dummy client which is only used {@link Agent}. + * Dummy client which is only used by {@link Agent}. */ static class DummyClient implements ClickHouseClient { - static final ClickHouseConfig CONFIG = new ClickHouseConfig(); - static final DummyClient INSTANCE = new DummyClient(); + static final ClickHouseConfig DEFAULT_CONFIG = new ClickHouseConfig(); + + private final ClickHouseConfig config; + + DummyClient() { + this(null); + } + + DummyClient(ClickHouseConfig config) { + this.config = config != null ? config : DEFAULT_CONFIG; + } @Override public boolean accept(ClickHouseProtocol protocol) { - return true; + return false; } @Override public CompletableFuture execute(ClickHouseRequest request) { - return CompletableFuture.completedFuture(ClickHouseResponse.EMPTY); + CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(new ConnectException("No client available")); + return future; } @Override public ClickHouseConfig getConfig() { - return CONFIG; + return config; } @Override @@ -55,7 +69,7 @@ public void close() { @Override public boolean ping(ClickHouseNode server, int timeout) { - return true; + return false; } } @@ -68,8 +82,8 @@ static final class Agent implements ClickHouseClient { private final AtomicReference client; - Agent(ClickHouseClient client) { - this.client = new AtomicReference<>(client != null ? client : DummyClient.INSTANCE); + Agent(ClickHouseClient client, ClickHouseConfig config) { + this.client = new AtomicReference<>(client != null ? client : new DummyClient(config)); } ClickHouseClient getClient() { @@ -90,25 +104,27 @@ boolean changeClient(ClickHouseClient currentClient, ClickHouseClient newClient) return changed; } - ClickHouseResponse failover(ClickHouseRequest sealedRequest, Throwable cause, int times) { + ClickHouseResponse failover(ClickHouseRequest sealedRequest, ClickHouseException exception, int times) { for (int i = 1; i <= times; i++) { - log.debug("Failover %d of %d due to: %s", i, times, cause.getMessage()); + log.debug("Failover %d of %d due to: %s", i, times, exception.getCause(), null); ClickHouseNode current = sealedRequest.getServer(); ClickHouseNodeManager manager = current.manager.get(); if (manager == null) { break; } - ClickHouseNode next = manager.suggestNode(current, cause); + ClickHouseNode next = manager.suggestNode(current, exception); if (next == current) { + log.debug("Cancel failover for same node returned from %s", manager.getPolicy()); break; } current.update(Status.FAULTY); next = sealedRequest.changeServer(current, next); if (next == current) { + log.debug("Cancel failover for no alternative of %s", current); break; } - log.info("Switching node from %s to %s due to: %s", current, next, cause.getMessage()); + log.info("Switching node from %s to %s due to: %s", current, next, exception.getCause(), null); final ClickHouseProtocol protocol = next.getProtocol(); final ClickHouseClient currentClient = client.get(); if (!currentClient.accept(protocol)) { @@ -118,64 +134,70 @@ ClickHouseResponse failover(ClickHouseRequest sealedRequest, Throwable cause, .config(new ClickHouseConfig(currentClient.getConfig(), next.config)) .nodeSelector(ClickHouseNodeSelector.of(protocol)).build(); } catch (Exception e) { - cause = e; - continue; + exception = ClickHouseException.of(new ConnectException("No client available for " + next), + sealedRequest.getServer()); } finally { if (newClient != null) { boolean changed = changeClient(currentClient, newClient); - log.debug("Switching client from %s to %s: %s", currentClient, newClient, changed); + log.info("Switching client from %s to %s: %s", currentClient, newClient, changed); if (changed) { sealedRequest.resetCache(); } } } + + if (newClient == null) { + continue; + } } try { return sendOnce(sealedRequest); } catch (Exception exp) { - cause = exp.getCause(); - if (cause == null) { - cause = exp; - } + exception = ClickHouseException.of(exp.getCause() != null ? exp.getCause() : exp, + sealedRequest.getServer()); } } - throw new CompletionException(cause); + throw new CompletionException(exception); } - ClickHouseResponse retry(ClickHouseRequest sealedRequest, Throwable cause, int times) { + ClickHouseResponse retry(ClickHouseRequest sealedRequest, ClickHouseException exception, int times) { for (int i = 1; i <= times; i++) { - log.debug("Retry %d of %d due to: %s", i, times, cause.getMessage()); + log.debug("Retry %d of %d due to: %s", i, times, exception.getMessage()); // TODO retry idempotent query - if (cause instanceof ClickHouseException - && ((ClickHouseException) cause).getErrorCode() == ClickHouseException.ERROR_NETWORK) { + if (exception.getErrorCode() == ClickHouseException.ERROR_NETWORK) { log.info("Retry request on %s due to connection issue", sealedRequest.getServer()); try { return sendOnce(sealedRequest); } catch (Exception exp) { - cause = exp.getCause(); - if (cause == null) { - cause = exp; - } + exception = ClickHouseException.of(exp.getCause() != null ? exp.getCause() : exp, + sealedRequest.getServer()); } } } - throw new CompletionException(cause); + throw new CompletionException(exception); } ClickHouseResponse handle(ClickHouseRequest sealedRequest, Throwable cause) { + // in case there's any recoverable exception wrapped by UncheckedIOException + if (cause instanceof UncheckedIOException && cause.getCause() != null) { + cause = ((UncheckedIOException) cause).getCause(); + } + + log.debug("Handling %s(failover=%d, retry=%d)", cause, sealedRequest.getConfig().getFailover(), + sealedRequest.getConfig().getRetry()); try { int times = sealedRequest.getConfig().getFailover(); if (times > 0) { - return failover(sealedRequest, cause, times); + return failover(sealedRequest, ClickHouseException.of(cause, sealedRequest.getServer()), times); } // different from failover: 1) retry on the same node; 2) never retry on timeout times = sealedRequest.getConfig().getRetry(); if (times > 0) { - return retry(sealedRequest, cause, times); + return retry(sealedRequest, ClickHouseException.of(cause, sealedRequest.getServer()), times); } throw new CompletionException(cause); @@ -200,8 +222,8 @@ ClickHouseResponse sendOnce(ClickHouseRequest sealedRequest) { ClickHouseResponse send(ClickHouseRequest sealedRequest) { try { return sendOnce(sealedRequest); - } catch (CompletionException e) { - return handle(sealedRequest, e.getCause()); + } catch (Exception e) { + return handle(sealedRequest, e.getCause() != null ? e.getCause() : e); } } @@ -228,9 +250,32 @@ public boolean ping(ClickHouseNode server, int timeout) { @Override public CompletableFuture execute(ClickHouseRequest request) { final ClickHouseRequest sealedRequest = request.seal(); + final ClickHouseNode server = sealedRequest.getServer(); + final ClickHouseProtocol protocol = server.getProtocol(); + final ClickHouseClient currentClient = client.get(); + if (!currentClient.accept(protocol)) { + ClickHouseClient newClient = null; + try { + newClient = ClickHouseClient.builder().agent(false) + .config(new ClickHouseConfig(currentClient.getConfig(), server.config)) + .nodeSelector(ClickHouseNodeSelector.of(protocol)).build(); + } catch (IllegalStateException e) { + // let it fail on execution phase + log.debug("Failed to find client for %s", server); + } finally { + if (newClient != null) { + boolean changed = changeClient(currentClient, newClient); + log.debug("Switching client from %s to %s: %s", currentClient, newClient, changed); + if (changed) { + sealedRequest.resetCache(); + } + } + } + } return sealedRequest.getConfig().isAsync() ? getClient().execute(sealedRequest) - .handle((r, t) -> t == null ? r : handle(sealedRequest, t.getCause())) + .handle((r, t) -> t == null ? r + : handle(sealedRequest, t.getCause() != null ? t.getCause() : t)) : CompletableFuture.completedFuture(send(sealedRequest)); } @@ -339,26 +384,28 @@ public ClickHouseConfig getConfig() { public ClickHouseClient build() { ClickHouseClient client = null; - boolean noSelector = nodeSelector == null || nodeSelector == ClickHouseNodeSelector.EMPTY; - int counter = 0; ClickHouseConfig conf = getConfig(); - for (ClickHouseClient c : loadClients()) { - c.init(conf); + int counter = 0; + if (nodeSelector != null) { + for (ClickHouseClient c : loadClients()) { + c.init(conf); - counter++; - if (noSelector || nodeSelector.match(c)) { - client = c; - break; + counter++; + if (nodeSelector == ClickHouseNodeSelector.EMPTY || nodeSelector.match(c)) { + client = c; + break; + } } } - if (client == null) { + if (agent) { + return new Agent(client, conf); + } else if (client == null) { throw new IllegalStateException( ClickHouseUtils.format("No suitable ClickHouse client(out of %d) found in classpath for %s.", counter, nodeSelector)); } - - return agent ? new Agent(client) : client; + return client; } /** @@ -475,7 +522,11 @@ public ClickHouseClientBuilder defaultCredentials(ClickHouseCredentials credenti */ public ClickHouseClientBuilder nodeSelector(ClickHouseNodeSelector nodeSelector) { if (!ClickHouseChecker.nonNull(nodeSelector, "nodeSelector").equals(this.nodeSelector)) { - this.nodeSelector = nodeSelector; + this.nodeSelector = (nodeSelector.getPreferredProtocols().isEmpty() || nodeSelector.getPreferredProtocols() + .equals(Collections.singletonList(ClickHouseProtocol.ANY))) + && nodeSelector.getPreferredTags().isEmpty() + ? ClickHouseNodeSelector.EMPTY + : nodeSelector; resetConfig(); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCluster.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCluster.java index 13451563a..98b76a373 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCluster.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCluster.java @@ -136,6 +136,6 @@ public String toString() { .append(checking.get()).append(", index=").append(index.get()).append(", lock=r") .append(lock.getReadHoldCount()).append('w').append(lock.getWriteHoldCount()).append(", nodes=") .append(nodes.size()).append(", faulty=").append(faultyNodes.size()).append(", policy=") - .append(policy.getClass().getSimpleName()).append(']').toString(); + .append(policy.getClass().getSimpleName()).append("]@").append(hashCode()).toString(); } } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java index 5a0bf54e5..3eb2559bc 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java @@ -482,6 +482,65 @@ public ClickHouseNode build() { public static final String SCHEME_DELIMITER = "://"; + static int extract(String scheme, int port, ClickHouseProtocol protocol, Map params) { + if (port < MIN_PORT_NUM || port > MAX_PORT_NUM) { + port = MIN_PORT_NUM; + } + if (protocol != ClickHouseProtocol.POSTGRESQL && scheme.charAt(scheme.length() - 1) == 's') { + params.putIfAbsent(ClickHouseClientOption.SSL.getKey(), Boolean.TRUE.toString()); + params.putIfAbsent(ClickHouseClientOption.SSL_MODE.getKey(), ClickHouseSslMode.STRICT.name()); + } + + if (protocol != ClickHouseProtocol.ANY && port == MIN_PORT_NUM) { + if (Boolean.TRUE.toString().equals(params.get(ClickHouseClientOption.SSL.getKey()))) { + port = protocol.getDefaultSecurePort(); + } else { + port = protocol.getDefaultPort(); + } + } + return port; + } + + static ClickHouseCredentials extract(String rawUserInfo, Map params, + ClickHouseCredentials defaultCredentials) { + ClickHouseCredentials credentials = defaultCredentials; + String user = ""; + String passwd = ""; + if (credentials != null && !credentials.useAccessToken()) { + user = credentials.getUserName(); + passwd = credentials.getPassword(); + } + + if (!ClickHouseChecker.isNullOrEmpty(rawUserInfo)) { + int index = rawUserInfo.indexOf(':'); + if (index < 0) { + user = ClickHouseUtils.decode(rawUserInfo); + } else { + String str = ClickHouseUtils.decode(rawUserInfo.substring(0, index)); + if (!ClickHouseChecker.isNullOrEmpty(str)) { + user = str; + } + passwd = ClickHouseUtils.decode(rawUserInfo.substring(index + 1)); + } + } + + String str = params.remove(ClickHouseDefaults.USER.getKey()); + if (!ClickHouseChecker.isNullOrEmpty(str)) { + user = str; + } + str = params.remove(ClickHouseDefaults.PASSWORD.getKey()); + if (str != null) { + passwd = str; + } + if (!ClickHouseChecker.isNullOrEmpty(user)) { + credentials = ClickHouseCredentials.fromUserAndPassword(user, passwd); + } else if (!ClickHouseChecker.isNullOrEmpty(passwd)) { + credentials = ClickHouseCredentials + .fromUserAndPassword((String) ClickHouseDefaults.USER.getEffectiveDefaultValue(), passwd); + } + return credentials; + } + static URI normalize(String uri, ClickHouseProtocol defaultProtocol) { int index = ClickHouseChecker.nonEmpty(uri, "URI").indexOf(SCHEME_DELIMITER); String normalized; @@ -503,6 +562,12 @@ static URI normalize(String uri, ClickHouseProtocol defaultProtocol) { }); } + static void parseDatabase(String path, Map params) { + if (!ClickHouseChecker.isNullOrEmpty(path) && path.length() > 1) { + params.put(ClickHouseClientOption.DATABASE.getKey(), path.substring(1)); + } + } + static void parseOptions(String query, Map params) { if (ClickHouseChecker.isNullOrEmpty(query)) { return; @@ -539,7 +604,9 @@ static void parseOptions(String query, Map params) { } // any multi-value option? cluster? - params.put(key, value); + if (!ClickHouseChecker.isNullOrEmpty(value)) { + params.put(key, value); + } } } @@ -735,33 +802,34 @@ public static ClickHouseNode of(String uri) { */ public static ClickHouseNode of(String uri, Map options) { URI normalizedUri = normalize(uri, null); - ClickHouseNode template = DEFAULT; + + Map params = new LinkedHashMap<>(); + parseDatabase(normalizedUri.getPath(), params); + + parseOptions(normalizedUri.getRawQuery(), params); + + Set tags = new LinkedHashSet<>(); + parseTags(normalizedUri.getRawFragment(), tags); + if (options != null && !options.isEmpty()) { - Builder builder = builder(DEFAULT); for (Entry entry : options.entrySet()) { if (entry.getKey() != null) { - builder.addOption(entry.getKey().toString(), - entry.getValue() != null ? entry.getValue().toString() : null); + if (entry.getValue() != null) { + params.put(entry.getKey().toString(), entry.getValue().toString()); + } else { + params.remove(entry.getKey().toString()); + } } } - String user = builder.options.remove(ClickHouseDefaults.USER.getKey()); - String passwd = builder.options.remove(ClickHouseDefaults.PASSWORD.getKey()); - if (!ClickHouseChecker.isNullOrEmpty(user)) { - builder.credentials(ClickHouseCredentials.fromUserAndPassword(user, passwd == null ? "" : passwd)); - } - String db = builder.options.get(ClickHouseClientOption.DATABASE.getKey()); - if (!ClickHouseChecker.isNullOrEmpty(db)) { - try { - normalizedUri = new URI(normalizedUri.getScheme(), normalizedUri.getUserInfo(), - normalizedUri.getHost(), normalizedUri.getPort(), "/" + db, normalizedUri.getQuery(), - normalizedUri.getFragment()); - } catch (URISyntaxException e) { // should not happen - throw new IllegalArgumentException("Failed to update database in given URI", e); - } - } - template = builder.build(); } - return of(normalizedUri, template); + + String scheme = normalizedUri.getScheme(); + ClickHouseProtocol protocol = ClickHouseProtocol.fromUriScheme(scheme); + int port = extract(scheme, normalizedUri.getPort(), protocol, params); + + ClickHouseCredentials credentials = extract(normalizedUri.getRawUserInfo(), params, null); + + return new ClickHouseNode(normalizedUri.getHost(), protocol, port, credentials, params, tags); } /** @@ -793,67 +861,15 @@ public static ClickHouseNode of(URI uri, ClickHouseNode template) { host = template.getHost(); } - int port = uri.getPort(); Map params = new LinkedHashMap<>(template.options); - ClickHouseProtocol protocol = ClickHouseProtocol.fromUriScheme(scheme); - if (port < MIN_PORT_NUM || port > MAX_PORT_NUM) { - port = MIN_PORT_NUM; - } - if (protocol != ClickHouseProtocol.POSTGRESQL && scheme.charAt(scheme.length() - 1) == 's') { - params.put(ClickHouseClientOption.SSL.getKey(), Boolean.TRUE.toString()); - params.put(ClickHouseClientOption.SSL_MODE.getKey(), ClickHouseSslMode.STRICT.name()); - } - - ClickHouseCredentials credentials = template.credentials; - String user = ""; - String passwd = ""; - if (credentials != null && !credentials.useAccessToken()) { - user = credentials.getUserName(); - passwd = credentials.getPassword(); - } - String auth = uri.getRawUserInfo(); - if (!ClickHouseChecker.isNullOrEmpty(auth)) { - int index = auth.indexOf(':'); - if (index < 0) { - user = ClickHouseUtils.decode(auth); - } else { - String str = ClickHouseUtils.decode(auth.substring(0, index)); - if (!ClickHouseChecker.isNullOrEmpty(str)) { - user = str; - } - passwd = ClickHouseUtils.decode(auth.substring(index + 1)); - } - } - - String db = uri.getPath(); - if (!ClickHouseChecker.isNullOrEmpty(db) && db.length() > 1) { - params.put(ClickHouseClientOption.DATABASE.getKey(), db.substring(1)); - } + parseDatabase(uri.getPath(), params); parseOptions(uri.getRawQuery(), params); - String str = params.remove(ClickHouseDefaults.USER.getKey()); - if (!ClickHouseChecker.isNullOrEmpty(str)) { - user = str; - } - str = params.remove(ClickHouseDefaults.PASSWORD.getKey()); - if (str != null) { - passwd = str; - } - if (!ClickHouseChecker.isNullOrEmpty(user)) { - credentials = ClickHouseCredentials.fromUserAndPassword(user, passwd); - } else if (!ClickHouseChecker.isNullOrEmpty(passwd)) { - credentials = ClickHouseCredentials - .fromUserAndPassword((String) ClickHouseDefaults.USER.getEffectiveDefaultValue(), passwd); - } + ClickHouseProtocol protocol = ClickHouseProtocol.fromUriScheme(scheme); + int port = extract(scheme, uri.getPort(), protocol, params); - if (protocol != ClickHouseProtocol.ANY && port == MIN_PORT_NUM) { - if (Boolean.TRUE.toString().equals(params.get(ClickHouseClientOption.SSL.getKey()))) { - port = protocol.getDefaultSecurePort(); - } else { - port = protocol.getDefaultPort(); - } - } + ClickHouseCredentials credentials = extract(uri.getRawUserInfo(), params, template.credentials); Set tags = new LinkedHashSet<>(template.tags); parseTags(uri.getRawFragment(), tags); @@ -1265,7 +1281,7 @@ public String toString() { if (!tags.isEmpty()) { builder.append(", tags=").append(tags); } - return builder.append(']').toString(); + return builder.append("]@").append(hashCode()).toString(); } /** diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodes.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodes.java index 0b4192fd6..a49bf7ce4 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodes.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodes.java @@ -41,58 +41,20 @@ public class ClickHouseNodes implements ClickHouseNodeManager { private static final Map cache = Collections.synchronizedMap(new WeakHashMap<>()); private static final char[] separators = new char[] { '/', '?', '#' }; - /** - * Build unique key for the given base URI and options. - * - * @param uri non-null URI - * @param options options - * @return non-null unique key for caching - */ - static String buildKey(String uri, Map options) { - if (uri == null) { - throw new IllegalArgumentException("Non-null URI required"); - } else if ((uri = uri.trim()).isEmpty()) { - throw new IllegalArgumentException("Non-blank URI required"); - } - if (options == null || options.isEmpty()) { - return uri; - } - - SortedMap sorted; - if (options instanceof SortedMap) { - sorted = (SortedMap) options; - } else { - sorted = new TreeMap<>(); - for (Entry entry : options.entrySet()) { - if (entry.getKey() != null) { - sorted.put(entry.getKey(), entry.getValue()); - } - } - } - - StringBuilder builder = new StringBuilder(uri).append('|'); - for (Entry entry : sorted.entrySet()) { - if (entry.getKey() != null) { - builder.append(entry.getKey()).append('=').append(entry.getValue()).append(','); - } - } - return builder.toString(); - } - /** * Creates list of managed {@link ClickHouseNode} for load balancing and * fail-over. * - * @param enpoints non-empty URIs separated by comma + * @param endpoints non-empty URIs separated by comma * @param defaultOptions default options * @return non-null list of nodes */ - static ClickHouseNodes create(String enpoints, Map defaultOptions) { - int index = enpoints.indexOf(ClickHouseNode.SCHEME_DELIMITER); + static ClickHouseNodes create(String endpoints, Map defaultOptions) { + int index = endpoints.indexOf(ClickHouseNode.SCHEME_DELIMITER); String defaultProtocol = ((ClickHouseProtocol) ClickHouseDefaults.PROTOCOL .getEffectiveDefaultValue()).name(); if (index > 0) { - defaultProtocol = enpoints.substring(0, index); + defaultProtocol = endpoints.substring(0, index); if (ClickHouseProtocol.fromUriScheme(defaultProtocol) == ClickHouseProtocol.ANY) { defaultProtocol = ClickHouseProtocol.ANY.name(); index = 0; @@ -106,13 +68,13 @@ static ClickHouseNodes create(String enpoints, Map defaultOptions) { String defaultParams = ""; Set list = new LinkedHashSet<>(); char stopChar = ','; - for (int i = index, len = enpoints.length(); i < len; i++) { - char ch = enpoints.charAt(i); + for (int i = index, len = endpoints.length(); i < len; i++) { + char ch = endpoints.charAt(i); if (ch == ',' || Character.isWhitespace(ch)) { index++; continue; } else if (ch == '/' || ch == '?' || ch == '#') { - defaultParams = enpoints.substring(i); + defaultParams = endpoints.substring(i); break; } switch (ch) { @@ -130,19 +92,19 @@ static ClickHouseNodes create(String enpoints, Map defaultOptions) { int endIndex = i; for (int j = i + 1; j < len; j++) { - ch = enpoints.charAt(j); + ch = endpoints.charAt(j); if (ch == stopChar || Character.isWhitespace(ch)) { endIndex = j; break; } } if (endIndex > i) { - list.add(enpoints.substring(index, endIndex).trim()); + list.add(endpoints.substring(index, endIndex).trim()); i = endIndex; index = endIndex + 1; stopChar = ','; } else { - String last = enpoints.substring(index); + String last = endpoints.substring(index); int sepIndex = last.indexOf(ClickHouseNode.SCHEME_DELIMITER); int startIndex = sepIndex < 0 ? 0 : sepIndex + 3; for (char spec : separators) { @@ -167,9 +129,9 @@ static ClickHouseNodes create(String enpoints, Map defaultOptions) { } if (list.size() == 1 && defaultParams.isEmpty()) { - enpoints = new StringBuilder().append(defaultProtocol).append(ClickHouseNode.SCHEME_DELIMITER) + endpoints = new StringBuilder().append(defaultProtocol).append(ClickHouseNode.SCHEME_DELIMITER) .append(list.iterator().next()).toString(); - return new ClickHouseNodes(Collections.singletonList(ClickHouseNode.of(enpoints, defaultOptions))); + return new ClickHouseNodes(Collections.singletonList(ClickHouseNode.of(endpoints, defaultOptions))); } ClickHouseNode defaultNode = ClickHouseNode.of(defaultProtocol + "://localhost" + defaultParams, @@ -247,31 +209,88 @@ static void pickNodes(Collection source, ClickHouseNodeSelector } } + /** + * Build unique key according to the given base URI and options for caching. + * + * @param uri non-null URI + * @param options options + * @return non-empty unique key for caching + */ + public static String buildCacheKey(String uri, Map options) { + if (uri == null) { + throw new IllegalArgumentException("Non-null URI required"); + } else if ((uri = uri.trim()).isEmpty()) { + throw new IllegalArgumentException("Non-blank URI required"); + } + if (options == null || options.isEmpty()) { + return uri; + } + + SortedMap sorted; + if (options instanceof SortedMap) { + sorted = (SortedMap) options; + } else { + sorted = new TreeMap<>(); + for (Entry entry : options.entrySet()) { + if (entry.getKey() != null) { + sorted.put(entry.getKey(), entry.getValue()); + } + } + } + + StringBuilder builder = new StringBuilder(uri).append('|'); + for (Entry entry : sorted.entrySet()) { + if (entry.getKey() != null) { + builder.append(entry.getKey()).append('=').append(entry.getValue()).append(','); + } + } + return builder.toString(); + } + /** * Gets or creates list of managed {@link ClickHouseNode} for load balancing * and fail-over. * - * @param enpoints non-empty URIs separated by comma + * @param endpoints non-empty URIs separated by comma * @return non-null list of nodes */ - public static ClickHouseNodes of(String enpoints) { - return of(enpoints, Collections.emptyMap()); + public static ClickHouseNodes of(String endpoints) { + return of(endpoints, Collections.emptyMap()); } /** * Gets or creates list of managed {@link ClickHouseNode} for load balancing * and fail-over. * - * @param enpoints non-empty URIs separated by comma - * @param options default options + * @param endpoints non-empty URIs separated by comma + * @param options default options * @return non-null list of nodes */ - public static ClickHouseNodes of(String enpoints, Map options) { + public static ClickHouseNodes of(String endpoints, Map options) { + return cache.computeIfAbsent(buildCacheKey(ClickHouseChecker.nonEmpty(endpoints, "Endpoints"), options), + k -> create(endpoints, options)); + } + + /** + * Gets or creates list of managed {@link ClickHouseNode} for load balancing + * and fail-over. Since the list will be cached in a {@link WeakHashMap}, as + * long as you hold strong reference to the {@code cacheKey}, same combination + * of {@code endpoints} and {@code options} will be always mapped to the exact + * same list. + * + * @param cacheKey non-empty cache key + * @param endpoints non-empty URIs separated by comma + * @param options default options + * @return non-null list of nodes + */ + public static ClickHouseNodes of(String cacheKey, String endpoints, Map options) { // TODO discover endpoints from a URL or custom service, for examples: // discover://(smb://fs1/ch-list.txt),(smb://fs1/ch-dc.json) // discover:com.mycompany.integration.clickhouse.Endpoints - return cache.computeIfAbsent(buildKey(ClickHouseChecker.nonEmpty(enpoints, "Endpoints"), options), - k -> create(enpoints, options)); + if (ClickHouseChecker.isNullOrEmpty(cacheKey) || ClickHouseChecker.isNullOrEmpty(endpoints)) { + throw new IllegalArgumentException("Non-empty cache key and endpoints are required"); + } + return cache.computeIfAbsent(cacheKey, k -> create(endpoints, options)); } /** @@ -314,6 +333,10 @@ public static ClickHouseNodes of(String enpoints, Map options) { * Load balancing tags for filtering out nodes. */ protected final ClickHouseNodeSelector selector; + /** + * Flag indicating whether it's single node or not. + */ + protected final boolean singleNode; /** * Template node. */ @@ -358,8 +381,11 @@ protected ClickHouseNodes(Collection nodes, ClickHouseNode templ n.setManager(this); } if (autoDiscovery) { + this.singleNode = false; this.discoveryFuture.getAndUpdate(current -> policy.schedule(current, ClickHouseNodes.this::discover, (int) template.config.getOption(ClickHouseClientOption.NODE_DISCOVERY_INTERVAL))); + } else { + this.singleNode = nodes.size() == 1; } this.healthCheckFuture.getAndUpdate(current -> policy.schedule(current, ClickHouseNodes.this::check, (int) template.config.getOption(ClickHouseClientOption.HEALTH_CHECK_INTERVAL))); @@ -472,6 +498,15 @@ protected ClickHouseNode get() { return apply(selector); } + /** + * Checks whether it's single node or not. + * + * @return true if it's single node; false otherwise + */ + public boolean isSingleNode() { + return singleNode; + } + @Override public ClickHouseNode apply(ClickHouseNodeSelector t) { lock.readLock().lock(); @@ -747,6 +782,6 @@ public String toString() { .append(index.get()).append(", lock=r").append(lock.getReadHoldCount()).append('w') .append(lock.getWriteHoldCount()).append(", nodes=").append(nodes.size()).append(", faulty=") .append(faultyNodes.size()).append(", policy=").append(policy.getClass().getSimpleName()) - .append(", tags=").append(selector.getPreferredTags()).append(']').toString(); + .append(", tags=").append(selector.getPreferredTags()).append("]@").append(hashCode()).toString(); } } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/logging/LogMessage.java b/clickhouse-client/src/main/java/com/clickhouse/client/logging/LogMessage.java index 36057fd0f..649ad5d74 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/logging/LogMessage.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/logging/LogMessage.java @@ -23,11 +23,6 @@ public static LogMessage of(Object format, Object... arguments) { Object lastArg = arguments[len - 1]; if (lastArg instanceof Throwable) { t = (Throwable) lastArg; - if (--len > 0) { - Object[] args = new Object[len]; - System.arraycopy(arguments, 0, args, 0, len); - arguments = args; - } } if (len > 0) { diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClientBuilderTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClientBuilderTest.java index 0e4481467..ecb73d76e 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClientBuilderTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClientBuilderTest.java @@ -12,9 +12,15 @@ public void testBuildClient() { ClickHouseClientBuilder builder = new ClickHouseClientBuilder(); ClickHouseClient client = builder.build(); Assert.assertTrue(client instanceof Agent); - Assert.assertTrue(((Agent) client).getClient() instanceof ClickHouseTestClient); + Assert.assertTrue(((Agent) client).getClient() instanceof ClickHouseClientBuilder.DummyClient); Assert.assertNotEquals(builder.build(), client); + Assert.assertTrue(client.getConfig() == builder.getConfig()); + builder.nodeSelector(ClickHouseNodeSelector.of(ClickHouseProtocol.ANY)); + client = builder.build(); + Assert.assertTrue(client instanceof Agent); + Assert.assertTrue(((Agent) client).getClient() instanceof ClickHouseTestClient); + Assert.assertNotEquals(builder.build(), client); Assert.assertTrue(client.getConfig() == builder.getConfig()); } diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java index ac72a0b87..a0eb4601b 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java @@ -379,7 +379,7 @@ public void testQueryWithSlash() throws Exception { Assert.assertEquals(server.toUri(), new URI("http://localhost:1234?/a/b/c=d")); Assert.assertEquals(ClickHouseNode.of("https://myserver/db/1/2/3?a%20=%201&b=/root/my.crt").toUri(), - new URI("http://myserver:8443/db/1/2/3?ssl=true&sslmode=STRICT&a%20=%201&b=/root/my.crt")); + new URI("http://myserver:8443/db/1/2/3?a%20=%201&b=/root/my.crt&ssl=true&sslmode=STRICT")); } @Test(groups = { "integration" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodesTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodesTest.java index 7357adbac..20cb76796 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodesTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodesTest.java @@ -29,33 +29,35 @@ public void testNullOrEmptyList() { } @Test(groups = { "unit" }) - public void testBuildKey() { + public void testBuildCacheKey() { String baseUri = "localhost"; - Assert.assertEquals(ClickHouseNodes.buildKey(baseUri, null), baseUri); - Assert.assertEquals(ClickHouseNodes.buildKey(baseUri, new TreeMap()), baseUri); - Assert.assertEquals(ClickHouseNodes.buildKey(baseUri, new Properties()), baseUri); + Assert.assertEquals(ClickHouseNodes.buildCacheKey(baseUri, null), baseUri); + Assert.assertEquals(ClickHouseNodes.buildCacheKey(baseUri, new TreeMap()), baseUri); + Assert.assertEquals(ClickHouseNodes.buildCacheKey(baseUri, new Properties()), baseUri); Map defaultOptions = new HashMap<>(); - Assert.assertEquals(ClickHouseNodes.buildKey(baseUri, defaultOptions), baseUri); + Assert.assertEquals(ClickHouseNodes.buildCacheKey(baseUri, defaultOptions), baseUri); defaultOptions.put("b", " "); - Assert.assertEquals(ClickHouseNodes.buildKey(baseUri, defaultOptions), baseUri + "|b= ,"); + Assert.assertEquals(ClickHouseNodes.buildCacheKey(baseUri, defaultOptions), baseUri + "|b= ,"); defaultOptions.put("a", 1); - Assert.assertEquals(ClickHouseNodes.buildKey(baseUri, defaultOptions), baseUri + "|a=1,b= ,"); + Assert.assertEquals(ClickHouseNodes.buildCacheKey(baseUri, defaultOptions), baseUri + "|a=1,b= ,"); defaultOptions.put(" ", false); - Assert.assertEquals(ClickHouseNodes.buildKey(baseUri, defaultOptions), baseUri + "| =false,a=1,b= ,"); + Assert.assertEquals(ClickHouseNodes.buildCacheKey(baseUri, defaultOptions), + baseUri + "| =false,a=1,b= ,"); defaultOptions.put(null, "null-key"); - Assert.assertEquals(ClickHouseNodes.buildKey(baseUri, defaultOptions), baseUri + "| =false,a=1,b= ,"); + Assert.assertEquals(ClickHouseNodes.buildCacheKey(baseUri, defaultOptions), + baseUri + "| =false,a=1,b= ,"); defaultOptions.put("null-value", null); - Assert.assertEquals(ClickHouseNodes.buildKey(baseUri, defaultOptions), + Assert.assertEquals(ClickHouseNodes.buildCacheKey(baseUri, defaultOptions), baseUri + "| =false,a=1,b= ,null-value=null,"); defaultOptions.put(null, null); - Assert.assertEquals(ClickHouseNodes.buildKey(baseUri, defaultOptions), + Assert.assertEquals(ClickHouseNodes.buildCacheKey(baseUri, defaultOptions), baseUri + "| =false,a=1,b= ,null-value=null,"); defaultOptions.put(ClickHouseDefaults.USER.getKey(), "hello "); - Assert.assertEquals(ClickHouseNodes.buildKey(baseUri, defaultOptions), + Assert.assertEquals(ClickHouseNodes.buildCacheKey(baseUri, defaultOptions), baseUri + "| =false,a=1,b= ,null-value=null,user=hello ,"); defaultOptions.put(ClickHouseDefaults.PASSWORD.getKey(), " /?&#"); - Assert.assertEquals(ClickHouseNodes.buildKey(baseUri, defaultOptions), + Assert.assertEquals(ClickHouseNodes.buildCacheKey(baseUri, defaultOptions), baseUri + "| =false,a=1,b= ,null-value=null,password= /?&#,user=hello ,"); Assert.assertTrue( ClickHouseNodes.of(baseUri, defaultOptions) == ClickHouseNodes.of(baseUri, @@ -103,7 +105,8 @@ public void testCredentials() { Map options = new HashMap<>(); options.put(ClickHouseDefaults.USER.getKey(), ""); options.put(ClickHouseDefaults.PASSWORD.getKey(), ""); - Assert.assertEquals(ClickHouseNodes.of("https://dba:managed@node1,(node2),(tcp://aaa:bbb@node3)/test", options) + Assert.assertEquals(ClickHouseNodes + .of("https://dba:managed@node1,(node2),(tcp://aaa:bbb@node3)/test", options) .getTemplate().getCredentials().orElse(null), null); options.put(ClickHouseDefaults.USER.getKey(), "/u:s?e#r"); options.put(ClickHouseDefaults.PASSWORD.getKey(), ""); @@ -127,10 +130,34 @@ public void testCredentials() { Assert.assertEquals( ClickHouseNodes.of("https://[::1]:3218/db1?password=ppp").nodes.get(0) .getCredentials().orElse(null), - ClickHouseCredentials.fromUserAndPassword((String) ClickHouseDefaults.USER.getEffectiveDefaultValue(), + ClickHouseCredentials.fromUserAndPassword( + (String) ClickHouseDefaults.USER.getEffectiveDefaultValue(), "ppp")); } + @Test(groups = { "unit" }) + public void testFactoryMethods() { + Properties props = new Properties(); + props.setProperty("database", "cc"); + props.setProperty("socket_timeout", "12345"); + props.setProperty("failover", "7"); + props.setProperty("load_balancing_policy", "roundRobin"); + for (ClickHouseNodes nodes : new ClickHouseNodes[] { + ClickHouseNodes.of( + "http://host1,host2,host3/bb?database=cc&socket_timeout=12345&failover=7&load_balancing_policy=roundRobin"), + ClickHouseNodes.of( + "http://host1,host2,host3?database=aa&socket_timeout=54321&failover=3&load_balancing_policy=random", + props), + ClickHouseNodes.of("http://host1,host2,host3/bb", props) + }) { + Assert.assertEquals(nodes.template.config.getDatabase(), "cc"); + Assert.assertEquals(nodes.template.config.getSocketTimeout(), 12345); + Assert.assertEquals(nodes.template.config.getFailover(), 7); + Assert.assertEquals(nodes.template.config.getOption(ClickHouseClientOption.LOAD_BALANCING_POLICY), + ClickHouseLoadBalancingPolicy.ROUND_ROBIN); + } + } + @Test(groups = { "unit" }) public void testGetNodes() { // without selector @@ -187,7 +214,8 @@ public void testNodeGrouping() throws Exception { @Test(groups = { "unit" }) public void testQueryWithSlash() throws Exception { - ClickHouseNodes servers = ClickHouseNodes.of("https://node1?a=/b/c/d,node2/db2?/a/b/c=d,node3/db1?a=/d/c.b"); + ClickHouseNodes servers = ClickHouseNodes + .of("https://node1?a=/b/c/d,node2/db2?/a/b/c=d,node3/db1?a=/d/c.b"); Assert.assertEquals(servers.nodes.get(0).getDatabase().orElse(null), "db1"); Assert.assertEquals(servers.nodes.get(0).getOptions().get("a"), "/b/c/d"); Assert.assertEquals(servers.nodes.get(1).getDatabase().orElse(null), "db2"); diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java index 7c227abb5..a801e36ba 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java @@ -177,9 +177,10 @@ public void testInitialization() throws Exception { Assert.assertNotEquals(getProtocol(), ClickHouseProtocol.ANY, "The client should support a specific protocol instead of ANY"); - try (ClickHouseClient client1 = ClickHouseClient.builder().build(); + try (ClickHouseClient client1 = ClickHouseClient.builder() + .nodeSelector(ClickHouseNodeSelector.of(getProtocol())).build(); ClickHouseClient client2 = ClickHouseClient.builder().option(ClickHouseClientOption.ASYNC, false) - .build(); + .nodeSelector(ClickHouseNodeSelector.of(ClickHouseProtocol.ANY)).build(); ClickHouseClient client3 = ClickHouseClient.newInstance(); ClickHouseClient client4 = ClickHouseClient.newInstance(getProtocol()); ClickHouseClient client5 = getClient()) { @@ -419,7 +420,8 @@ public void testQuery() throws Exception { public void testQueryInSameThread() throws Exception { ClickHouseNode server = getServer(); - try (ClickHouseClient client = ClickHouseClient.builder().option(ClickHouseClientOption.ASYNC, false).build()) { + try (ClickHouseClient client = ClickHouseClient.builder().nodeSelector(ClickHouseNodeSelector.EMPTY) + .option(ClickHouseClientOption.ASYNC, false).build()) { CompletableFuture future = client.connect(server) .format(ClickHouseFormat.TabSeparatedWithNamesAndTypes).query("select 1,2").execute(); // Assert.assertTrue(future instanceof ClickHouseImmediateFuture); diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/logging/LogMessageTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/logging/LogMessageTest.java index 0245e52b1..54bb903e0 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/logging/LogMessageTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/logging/LogMessageTest.java @@ -37,5 +37,13 @@ public void testMessageWithThrowable() { msg = LogMessage.of("test %s", 1, t); Assert.assertEquals("test 1", msg.getMessage()); Assert.assertEquals(t, msg.getThrowable()); + + msg = LogMessage.of("test %d %s", 1, t); + Assert.assertEquals("test 1 java.lang.Exception", msg.getMessage()); + Assert.assertEquals(t, msg.getThrowable()); + + msg = LogMessage.of("test %d %s", 1, t, null); + Assert.assertEquals("test 1 java.lang.Exception", msg.getMessage()); + Assert.assertEquals(msg.getThrowable(), null); } } diff --git a/clickhouse-http-client/pom.xml b/clickhouse-http-client/pom.xml index 184246f1c..4edd2ecae 100644 --- a/clickhouse-http-client/pom.xml +++ b/clickhouse-http-client/pom.xml @@ -19,7 +19,6 @@ ${project.parent.groupId} clickhouse-client ${revision} - compile com.google.code.gson diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImpl.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImpl.java index be75aa33f..4e6af77b5 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImpl.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImpl.java @@ -27,11 +27,13 @@ import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseClient; +import com.clickhouse.client.ClickHouseClientBuilder; import com.clickhouse.client.ClickHouseColumn; import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseFormat; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseNodeSelector; +import com.clickhouse.client.ClickHouseNodes; import com.clickhouse.client.ClickHouseParameterizedQuery; import com.clickhouse.client.ClickHouseRecord; import com.clickhouse.client.ClickHouseRequest; @@ -218,16 +220,33 @@ public ClickHouseConnectionImpl(ConnectionInfo connInfo) throws SQLException { jdbcConf = connInfo.getJdbcConfig(); autoCommit = !jdbcConf.isJdbcCompliant() || jdbcConf.isAutoCommit(); - - ClickHouseNode node = connInfo.getServer(); - log.debug("Connecting to: %s", node); - jvmTimeZone = TimeZone.getDefault(); - client = ClickHouseClient.builder().options(ClickHouseDriver.toClientOptions(connInfo.getProperties())) - .defaultCredentials(connInfo.getDefaultCredentials()) - .nodeSelector(ClickHouseNodeSelector.of(node.getProtocol())).build(); - clientRequest = client.connect(node); + ClickHouseClientBuilder clientBuilder = ClickHouseClient.builder() + .options(ClickHouseDriver.toClientOptions(connInfo.getProperties())) + .defaultCredentials(connInfo.getDefaultCredentials()); + ClickHouseNodes nodes = connInfo.getNodes(); + final ClickHouseNode node; + if (nodes.isSingleNode()) { + try { + node = nodes.apply(nodes.getNodeSelector()); + } catch (Exception e) { + throw SqlExceptionUtils.clientError("Failed to get single-node", e); + } + client = clientBuilder.nodeSelector(ClickHouseNodeSelector.of(node.getProtocol())).build(); + clientRequest = client.connect(node); + } else { + log.debug("Selecting node from: %s", nodes); + client = clientBuilder.build(); // use dummy client + clientRequest = client.connect(nodes); + try { + node = clientRequest.getServer(); + } catch (Exception e) { + throw SqlExceptionUtils.clientError("No healthy node available", e); + } + } + + log.warn("Connecting to: %s", node); ClickHouseConfig config = clientRequest.getConfig(); String currentUser = null; TimeZone timeZone = null; diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParser.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParser.java index 9e016cba0..897248649 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParser.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParser.java @@ -12,21 +12,19 @@ import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseDefaults; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; import com.clickhouse.jdbc.JdbcConfig; import com.clickhouse.jdbc.SqlExceptionUtils; public class ClickHouseJdbcUrlParser { - private static final Logger log = LoggerFactory.getLogger(ClickHouseJdbcUrlParser.class); - public static class ConnectionInfo { + private final String cacheKey; private final ClickHouseCredentials credentials; private final ClickHouseNodes nodes; private final JdbcConfig jdbcConf; private final Properties props; - protected ConnectionInfo(ClickHouseNodes nodes, Properties props) { + protected ConnectionInfo(String cacheKey, ClickHouseNodes nodes, Properties props) { + this.cacheKey = cacheKey; this.nodes = nodes; this.jdbcConf = new JdbcConfig(props); this.props = props; @@ -46,6 +44,14 @@ public ClickHouseCredentials getDefaultCredentials() { return this.credentials; } + /** + * Gets selected server. + * + * @return non-null selected server + * @deprecated will be removed in v0.3.3, please use {@link #getNodes()} + * instead + */ + @Deprecated public ClickHouseNode getServer() { return nodes.apply(nodes.getNodeSelector()); } @@ -54,6 +60,15 @@ public JdbcConfig getJdbcConfig() { return jdbcConf; } + /** + * Gets nodes defined in connection string. + * + * @return non-null nodes + */ + public ClickHouseNodes getNodes() { + return nodes; + } + public Properties getProperties() { return props; } @@ -100,10 +115,12 @@ public static ConnectionInfo parse(String jdbcUrl, Properties defaults) throws S } try { - ClickHouseNodes nodes = ClickHouseNodes.of(jdbcUrl, defaults); + String cacheKey = ClickHouseNodes.buildCacheKey(jdbcUrl, defaults); + ClickHouseNodes nodes = ClickHouseNodes.of(cacheKey, jdbcUrl, defaults); Properties props = newProperties(); props.putAll(nodes.getTemplate().getOptions()); - return new ConnectionInfo(nodes, props); + props.putAll(defaults); + return new ConnectionInfo(cacheKey, nodes, props); } catch (IllegalArgumentException e) { throw SqlExceptionUtils.clientError(e); } diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseDataSourceTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseDataSourceTest.java index ba76d0a28..47404d670 100644 --- a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseDataSourceTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseDataSourceTest.java @@ -1,5 +1,6 @@ package com.clickhouse.jdbc; +import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; @@ -8,10 +9,53 @@ import org.testng.Assert; import org.testng.annotations.Test; +import com.clickhouse.client.ClickHouseLoadBalancingPolicy; +import com.clickhouse.client.ClickHouseProtocol; +import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseDefaults; public class ClickHouseDataSourceTest extends JdbcIntegrationTest { + @Test(groups = "integration") + public void testHighAvailabilityConfig() throws SQLException { + String httpEndpoint = "http://" + getServerAddress(ClickHouseProtocol.HTTP) + "/"; + String grpcEndpoint = "grpc://" + getServerAddress(ClickHouseProtocol.GRPC) + "/"; + String tcpEndpoint = "tcp://" + getServerAddress(ClickHouseProtocol.TCP) + "/"; + + String url = "jdbc:ch://(" + httpEndpoint + "),(" + grpcEndpoint + "),(" + tcpEndpoint + ")/system"; + Properties props = new Properties(); + props.setProperty("failover", "21"); + props.setProperty("load_balancing_policy", "roundRobin"); + try (Connection conn = DriverManager.getConnection(url, props)) { + Assert.assertEquals(conn.unwrap(ClickHouseRequest.class).getConfig().getFailover(), 21); + Assert.assertEquals(conn.unwrap(ClickHouseRequest.class).getConfig().getOption( + ClickHouseClientOption.LOAD_BALANCING_POLICY), ClickHouseLoadBalancingPolicy.ROUND_ROBIN); + } + } + + @Test // (groups = "integration") + public void testMultiEndpoints() throws SQLException { + String httpEndpoint = "http://" + getServerAddress(ClickHouseProtocol.HTTP) + "/"; + String grpcEndpoint = "grpc://" + getServerAddress(ClickHouseProtocol.GRPC) + "/"; + String tcpEndpoint = "tcp://" + getServerAddress(ClickHouseProtocol.TCP) + "/"; + + String url = "jdbc:ch://(" + httpEndpoint + "),(" + grpcEndpoint + "),(" + tcpEndpoint + + ")/system?load_balancing_policy=roundRobin"; + Properties props = new Properties(); + props.setProperty("user", "default"); + props.setProperty("password", ""); + ClickHouseDataSource ds = new ClickHouseDataSource(url, props); + for (int i = 0; i < 10; i++) { + try (Connection httpConn = ds.getConnection(); + Connection grpcConn = ds.getConnection("default", ""); + Connection tcpConn = DriverManager.getConnection(url, props)) { + Assert.assertEquals(httpConn.unwrap(ClickHouseRequest.class).getServer().getBaseUri(), httpEndpoint); + Assert.assertEquals(grpcConn.unwrap(ClickHouseRequest.class).getServer().getBaseUri(), grpcEndpoint); + Assert.assertEquals(tcpConn.unwrap(ClickHouseRequest.class).getServer().getBaseUri(), tcpEndpoint); + } + } + } + @Test(groups = "integration") public void testGetConnection() throws SQLException { String url = "jdbc:ch:" + DEFAULT_PROTOCOL.name() + "://" + getServerAddress(DEFAULT_PROTOCOL);