Repository: kylin Updated Branches: refs/heads/master 65afee782 -> b6cdc0d1b
KYLIN-2043 rollback httpclient to 4.2.5 to align with Hadoop 2.6/2.7 Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b6cdc0d1 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b6cdc0d1 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b6cdc0d1 Branch: refs/heads/master Commit: b6cdc0d1b20683feb7255a71dfa8d5b98c4e636e Parents: 65afee7 Author: Yang Li <liy...@apache.org> Authored: Sat Sep 24 01:31:22 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Sat Sep 24 01:31:22 2016 +0800 ---------------------------------------------------------------------- .../kylin/common/restclient/RestClient.java | 35 +++++++-------- .../engine/mr/common/HadoopStatusGetter.java | 27 ++++++------ .../java/org/apache/kylin/jdbc/KylinClient.java | 46 ++++++++++---------- pom.xml | 2 +- 4 files changed, 52 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/b6cdc0d1/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java index 46a9e9b..363cc54 100644 --- a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java +++ b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java @@ -23,16 +23,15 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; +import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.util.EntityUtils; import org.apache.kylin.common.util.JsonUtil; /** @@ -45,7 +44,7 @@ public class RestClient { protected String baseUrl; protected String userName; protected String password; - protected CloseableHttpClient client; + protected DefaultHttpClient client; protected static Pattern fullRestPattern = Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?"); @@ -79,13 +78,13 @@ public class RestClient { this.password = password; this.baseUrl = "http://"; + host + ":" + port + "/kylin/api"; - client = HttpClients.createDefault(); + client = new DefaultHttpClient(); if (userName != null && password != null) { CredentialsProvider provider = new BasicCredentialsProvider(); UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(userName, password); - provider.setCredentials(AuthScope.ANY, credentials); - client = HttpClients.custom().setDefaultCredentialsProvider(provider).build(); + provider.setCredentials(AuthScope.ANY, credentials); + client.setCredentialsProvider(provider); } } @@ -94,12 +93,11 @@ public class RestClient { HttpPut request = new HttpPut(url); try { - CloseableHttpResponse response = client.execute(request); + HttpResponse response = client.execute(request); String msg = EntityUtils.toString(response.getEntity()); if (response.getStatusLine().getStatusCode() != 200) - throw new IOException("Invalid response " + response.getStatusLine().getStatusCode() + " with cache wipe url " + url + "\n" + msg); - response.close(); + throw new IOException("Invalid response " + response.getStatusLine().getStatusCode() + " with cache wipe url " + url + "\n" + msg); } catch (Exception ex) { throw new IOException(ex); } finally { @@ -111,14 +109,13 @@ public class RestClient { String url = baseUrl + "/admin/config"; HttpGet request = new HttpGet(url); try { - CloseableHttpResponse response = client.execute(request); + HttpResponse response = client.execute(request); String msg = EntityUtils.toString(response.getEntity()); Map<String, String> map = JsonUtil.readValueAsMap(msg); msg = map.get("config"); if (response.getStatusLine().getStatusCode() != 200) throw new IOException("Invalid response " + response.getStatusLine().getStatusCode() + " with cache wipe url " + url + "\n" + msg); - response.close(); return msg; } finally { request.releaseConnection(); http://git-wip-us.apache.org/repos/asf/kylin/blob/b6cdc0d1/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java index 14f7235..619de90 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java @@ -34,19 +34,15 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; -import org.apache.http.auth.AuthSchemeProvider; +import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthSchemeRegistry; import org.apache.http.auth.AuthScope; import org.apache.http.auth.Credentials; -import org.apache.http.client.config.AuthSchemes; -import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.config.Lookup; -import org.apache.http.config.RegistryBuilder; +import org.apache.http.client.params.AuthPolicy; import org.apache.http.impl.auth.SPNegoSchemeFactory; import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.client.DefaultHttpClient; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import org.slf4j.Logger; @@ -88,9 +84,12 @@ public class HadoopStatusGetter { System.setProperty("java.security.krb5.conf", krb5ConfigPath); System.setProperty("sun.security.krb5.debug", "true"); System.setProperty("javax.security.auth.useSubjectCredsOnly", "false"); - Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider> create().register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(skipPortAtKerberosDatabaseLookup)).build(); - CloseableHttpClient client = HttpClients.custom().setDefaultAuthSchemeRegistry(authSchemeRegistry).build(); - HttpClientContext context = HttpClientContext.create(); + + DefaultHttpClient client = new DefaultHttpClient(); + AuthSchemeRegistry authSchemeRegistry = new AuthSchemeRegistry(); + authSchemeRegistry.register(AuthPolicy.SPNEGO, new SPNegoSchemeFactory(skipPortAtKerberosDatabaseLookup)); + client.setAuthSchemes(authSchemeRegistry); + BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); Credentials useJaasCreds = new Credentials() { public String getPassword() { @@ -101,9 +100,9 @@ public class HadoopStatusGetter { return null; } }; - credentialsProvider.setCredentials(new AuthScope(null, -1, null), useJaasCreds); - context.setCredentialsProvider(credentialsProvider); + client.setCredentialsProvider(credentialsProvider); + String response = null; while (response == null) { if (url.startsWith("https://";)) { @@ -116,7 +115,7 @@ public class HadoopStatusGetter { HttpGet httpget = new HttpGet(url); httpget.addHeader("accept", "application/json"); try { - CloseableHttpResponse httpResponse = client.execute(httpget, context); + HttpResponse httpResponse = client.execute(httpget); String redirect = null; org.apache.http.Header h = httpResponse.getFirstHeader("Location"); if (h != null) { http://git-wip-us.apache.org/repos/asf/kylin/blob/b6cdc0d1/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java ---------------------------------------------------------------------- diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java index 8890575..2d06a92 100644 --- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java +++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java @@ -20,6 +20,7 @@ package org.apache.kylin.jdbc; import java.io.IOException; import java.math.BigDecimal; +import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.sql.Date; import java.sql.Time; @@ -31,7 +32,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; -import javax.net.ssl.SSLContext; import javax.xml.bind.DatatypeConverter; import org.apache.calcite.avatica.AvaticaParameter; @@ -39,17 +39,15 @@ import org.apache.calcite.avatica.ColumnMetaData; import org.apache.calcite.avatica.ColumnMetaData.Rep; import org.apache.calcite.avatica.ColumnMetaData.ScalarType; import org.apache.http.HttpResponse; -import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.scheme.Scheme; +import org.apache.http.conn.ssl.SSLSocketFactory; import org.apache.http.conn.ssl.TrustStrategy; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.ssl.SSLContexts; +import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.util.EntityUtils; import org.apache.kylin.jdbc.KylinMeta.KMetaCatalog; import org.apache.kylin.jdbc.KylinMeta.KMetaColumn; @@ -74,25 +72,25 @@ public class KylinClient implements IRemoteClient { private final KylinConnection conn; private final Properties connProps; - private CloseableHttpClient httpClient; + private DefaultHttpClient httpClient; private final ObjectMapper jsonMapper; public KylinClient(KylinConnection conn) { this.conn = conn; this.connProps = conn.getConnectionProperties(); - this.httpClient = HttpClients.createDefault(); + this.httpClient = new DefaultHttpClient(); this.jsonMapper = new ObjectMapper(); // trust all certificates if (isSSL()) { try { - TrustStrategy acceptingTrustStrategy = new TrustStrategy() { - public boolean isTrusted(X509Certificate[] certificate, String type) { + SSLSocketFactory sslsf = new SSLSocketFactory(new TrustStrategy() { + public boolean isTrusted(final X509Certificate[] chain, String authType) throws CertificateException { + // Oh, I am easy... return true; } - }; - SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, acceptingTrustStrategy).build(); - httpClient = HttpClients.custom().setSSLHostnameVerifier(new NoopHostnameVerifier()).setSSLContext(sslContext).build(); + }); + httpClient.getConnectionManager().getSchemeRegistry().register(new Scheme("https", 443, sslsf)); } catch (Exception e) { throw new RuntimeException("Initialize HTTPS client failed", e); } @@ -227,12 +225,15 @@ public class KylinClient implements IRemoteClient { StringEntity requestEntity = new StringEntity("{}", ContentType.create("application/json", "UTF-8")); post.setEntity(requestEntity); - CloseableHttpResponse response = httpClient.execute(post); + try { + HttpResponse response = httpClient.execute(post); - if (response.getStatusLine().getStatusCode() != 200 && response.getStatusLine().getStatusCode() != 201) { - throw asIOException(post, response); + if (response.getStatusLine().getStatusCode() != 200 && response.getStatusLine().getStatusCode() != 201) { + throw asIOException(post, response); + } + } finally { + post.releaseConnection(); } - response.close(); } @Override @@ -243,7 +244,7 @@ public class KylinClient implements IRemoteClient { HttpGet get = new HttpGet(url); addHttpHeaders(get); - CloseableHttpResponse response = httpClient.execute(get); + HttpResponse response = httpClient.execute(get); if (response.getStatusLine().getStatusCode() != 200 && response.getStatusLine().getStatusCode() != 201) { throw asIOException(get, response); @@ -255,7 +256,7 @@ public class KylinClient implements IRemoteClient { List<KMetaTable> tables = convertMetaTables(tableMetaStubs); List<KMetaSchema> schemas = convertMetaSchemas(tables); List<KMetaCatalog> catalogs = convertMetaCatalogs(schemas); - response.close(); + get.releaseConnection(); return new KMetaProject(project, catalogs); } @@ -368,14 +369,14 @@ public class KylinClient implements IRemoteClient { StringEntity requestEntity = new StringEntity(postBody, ContentType.create("application/json", "UTF-8")); post.setEntity(requestEntity); - CloseableHttpResponse response = httpClient.execute(post); + HttpResponse response = httpClient.execute(post); if (response.getStatusLine().getStatusCode() != 200 && response.getStatusLine().getStatusCode() != 201) { throw asIOException(post, response); } SQLResponseStub stub = jsonMapper.readValue(response.getEntity().getContent(), SQLResponseStub.class); - response.close(); + post.releaseConnection(); return stub; } @@ -416,8 +417,5 @@ public class KylinClient implements IRemoteClient { @Override public void close() throws IOException { - if (httpClient != null) { - httpClient.close(); - } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/b6cdc0d1/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 1abc4eb..ecb2724 100644 --- a/pom.xml +++ b/pom.xml @@ -96,7 +96,7 @@ <extendedset.version>1.3.4</extendedset.version> <kryo.version>4.0.0</kryo.version> <ehcache.version>2.10.2.2.21</ehcache.version> - <apache-httpclient.version>4.5.2</apache-httpclient.version> + <apache-httpclient.version>4.2.5</apache-httpclient.version> <roaring.version>0.6.18</roaring.version> <cglib.version>3.2.4</cglib.version> <supercsv.version>2.4.0</supercsv.version>
