Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[client-v2] Proxy support #1748

Merged
merged 7 commits into from
Jul 29, 2024
Merged

[client-v2] Proxy support #1748

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
568a906
added proxy authentication
chernser Jul 25, 2024
80a2373
added socks proxy to socket config
chernser Jul 25, 2024
9380c5d
modified test to have local instance of client and proxy
chernser Jul 26, 2024
82cbb6a
fix NPE in proxy configuration
chernser Jul 26, 2024
eec0fe5
added cookies test. fixed handling Bad Gateway status from server
chernser Jul 26, 2024
64569f6
added option to disable cookies
chernser Jul 26, 2024
bcc0f63
add reading configuration for some parameters
chernser Jul 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion client-v2/src/main/java/com/clickhouse/client/api/Client.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,12 @@ public Builder addProxy(ProxyType type, String host, int port) {
return this;
}

public Builder setProxyCredentials(String user, String pass) {
this.configuration.put("proxy_user", user);
this.configuration.put("proxy_password", pass);
return this;
}

/**
* Sets the maximum time for operation to complete. By default, it is set to 3 hours.
* @param timeout
Expand All @@ -444,6 +450,12 @@ public Builder useNewImplementation(boolean useNewImplementation) {
return this;
}

public Builder setHttpCookiesEnabled(boolean enabled) {
//TODO: extract to settings string constants
this.configuration.put("client.http.cookies_enabled", String.valueOf(enabled));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is that used for?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  • sticky sessions
  • some enterprise proxies like F5 that store routing information in cookies

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

actually there may be opposite case when cookies should not be preserved to avoid sticky session when proxy behavior may not be changed.

return this;
}

public Client build() {
// check if endpoint are empty. so can not initiate client
if (this.endpoints.isEmpty()) {
Expand Down Expand Up @@ -963,6 +975,8 @@ public CompletableFuture<QueryResponse> query(String sqlQuery, Map<String, Objec
metrics.operationComplete();

return new QueryResponse(httpResponse, finalSettings, metrics);
} catch (ClientException e) {
throw e;
} catch (Exception e) {
throw new ClientException("Failed to execute query", e);
}
Expand Down Expand Up @@ -1145,7 +1159,7 @@ public CompletableFuture<CommandResponse> execute(String sql) {
} catch (Exception e) {
throw new ClientException("Failed to get command response", e);
}
});
}, sharedOperationExecutor);
}

private String startOperation() {
Expand Down
14 changes: 14 additions & 0 deletions client-v2/src/main/java/com/clickhouse/client/api/ClientMisconfigurationException.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package com.clickhouse.client.api;

/**
* Represents errors caused by a client misconfiguration.
*/
public class ClientMisconfigurationException extends ClientException {
public ClientMisconfigurationException(String message) {
super(message);
}

public ClientMisconfigurationException(String message, Throwable cause) {
super(message, cause);
}
}
66 changes: 53 additions & 13 deletions client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,26 @@
import com.clickhouse.client.ClickHouseNode;
import com.clickhouse.client.api.Client;
import com.clickhouse.client.api.ClientException;
import com.clickhouse.client.api.ClientMisconfigurationException;
import com.clickhouse.client.api.ServerException;
import com.clickhouse.client.api.enums.ProxyType;
import com.clickhouse.client.config.ClickHouseClientOption;
import com.clickhouse.client.http.ClickHouseHttpProto;
import com.clickhouse.client.http.config.ClickHouseHttpOption;
import org.apache.hc.client5.http.SchemePortResolver;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.auth.CredentialsProviderBuilder;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
import org.apache.hc.client5.http.impl.routing.DefaultRoutePlanner;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.ClassicHttpResponse;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.Header;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.NoHttpResponseException;
import org.apache.hc.core5.http.io.SocketConfig;
import org.apache.hc.core5.http.io.entity.EntityTemplate;
import org.apache.hc.core5.io.IOCallback;
import org.apache.hc.core5.net.URIBuilder;
Expand All @@ -29,12 +32,14 @@
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Serializable;
import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.net.NoRouteToHostException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.Base64;
import java.util.EnumSet;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
Expand All @@ -50,25 +55,48 @@ public class HttpAPIClientHelper {

private RequestConfig baseRequestConfig;

private String proxyAuthHeaderValue;

public HttpAPIClientHelper(Map<String, String> configuration) {
this.chConfiguration = configuration;
this.httpClient = createHttpClient(configuration, null);
this.httpClient = createHttpClient();
this.baseRequestConfig = RequestConfig.custom()
.setConnectionRequestTimeout(1000, TimeUnit.MILLISECONDS)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should it be configurable?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. Added more configuration parameters. Planning to add more later.

.build();
}

public CloseableHttpClient createHttpClient(Map<String, String> chConfig, Map<String, Serializable> requestConfig) {
HttpClientBuilder httpclient = HttpClientBuilder.create();
public CloseableHttpClient createHttpClient() {
HttpClientBuilder clientBuilder = HttpClientBuilder.create();
CredentialsProviderBuilder credProviderBuilder = CredentialsProviderBuilder.create();
SocketConfig.Builder soCfgBuilder = SocketConfig.custom();


String proxyHost = chConfig.get(ClickHouseClientOption.PROXY_HOST.getKey());
String proxyPort = chConfig.get(ClickHouseClientOption.PROXY_PORT.getKey());
String proxyHost = chConfiguration.get(ClickHouseClientOption.PROXY_HOST.getKey());
String proxyPort = chConfiguration.get(ClickHouseClientOption.PROXY_PORT.getKey());
HttpHost proxy = null;
if (proxyHost != null && proxyPort != null) {
HttpHost proxy = new HttpHost(proxyHost, Integer.parseInt(proxyPort));
httpclient.setProxy(proxy);
proxy = new HttpHost(proxyHost, Integer.parseInt(proxyPort));
}


String proxyTypeVal = chConfiguration.get(ClickHouseClientOption.PROXY_TYPE.getKey());
ProxyType proxyType = proxyTypeVal == null ? null : ProxyType.valueOf(proxyTypeVal);
if (proxyType == ProxyType.HTTP) {
clientBuilder.setProxy(proxy);
if (chConfiguration.containsKey("proxy_password") && chConfiguration.containsKey("proxy_user")) {
proxyAuthHeaderValue = "Basic " + Base64.getEncoder().encodeToString(
(chConfiguration.get("proxy_user") + ":" + chConfiguration.get("proxy_password")).getBytes());
}
} else if (proxyType == ProxyType.SOCKS) {
soCfgBuilder.setSocksProxyAddress(new InetSocketAddress(proxyHost, Integer.parseInt(proxyPort)));
}

return httpclient.build();
if (chConfiguration.getOrDefault("client.http.cookies_enabled", "true")
.equalsIgnoreCase("false")) {
clientBuilder.disableCookieManagement();
}
clientBuilder.setDefaultCredentialsProvider(credProviderBuilder.build());
return clientBuilder.build();
}

/**
Expand Down Expand Up @@ -113,13 +141,19 @@ public ClassicHttpResponse executeRequest(ClickHouseNode server, Map<String, Obj

try {
ClassicHttpResponse httpResponse = httpClient.executeOpen(target, req, context);
if (httpResponse.getCode() >= 400 && httpResponse.getCode() < 500) {
if (httpResponse.getCode() == HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED) {
throw new ClientMisconfigurationException("Proxy authentication required. Please check your proxy settings.");
} else if (httpResponse.getCode() >= HttpStatus.SC_BAD_REQUEST &&
httpResponse.getCode() < HttpStatus.SC_SERVER_ERROR) {
try {
throw readError(httpResponse);
} finally {
httpResponse.close();
}
} else if (httpResponse.getCode() >= 500) {
} else if (httpResponse.getCode() == HttpStatus.SC_BAD_GATEWAY) {
httpResponse.close();
throw new ClientException("Server returned '502 Bad gateway'. Check network and proxy settings.");
} else if (httpResponse.getCode() >= HttpStatus.SC_INTERNAL_SERVER_ERROR) {
httpResponse.close();
return httpResponse;
}
Expand All @@ -133,6 +167,8 @@ public ClassicHttpResponse executeRequest(ClickHouseNode server, Map<String, Obj
throw e;
} catch (NoHttpResponseException e) {
throw e;
} catch (ClientException e) {
throw e;
} catch (Exception e) {
throw new ClientException("Failed to execute request", e);
}
Expand All @@ -150,6 +186,10 @@ private void addHeaders(HttpPost req, Map<String, String> chConfig, Map<String,
}
}
req.addHeader(ClickHouseHttpProto.HEADER_DATABASE, chConfig.get(ClickHouseClientOption.DATABASE.getKey()));

if (proxyAuthHeaderValue != null) {
req.addHeader(HttpHeaders.PROXY_AUTHORIZATION, proxyAuthHeaderValue);
}
}
private void addQueryParams(URIBuilder req, Map<String, String> chConfig, Map<String, Object> requestConfig) {
if (requestConfig != null) {
Expand Down
Loading
Loading