KYLIN-2748 Calcite code generation can not gc cause OOM
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8898213f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8898213f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8898213f Branch: refs/heads/2.1.x Commit: 8898213f9d8229382b805574784e566fb841c7eb Parents: 057bca4 Author: yiming.xu <100650...@qq.com> Authored: Thu Jul 20 18:12:44 2017 +0800 Committer: Hongbin Ma <m...@kyligence.io> Committed: Mon Jul 24 17:56:04 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/query/QueryConnection.java | 42 ++++++++++++++++++++ .../apache/kylin/rest/service/CacheService.java | 1 + .../apache/kylin/rest/service/QueryService.java | 10 ++--- .../kylin/rest/service/CubeServiceTest.java | 6 ++- .../kylin/rest/service/JobServiceTest.java | 6 ++- .../kylin/rest/service/QueryServiceTest.java | 3 +- 6 files changed, 57 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/query/src/main/java/org/apache/kylin/query/QueryConnection.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/QueryConnection.java b/query/src/main/java/org/apache/kylin/query/QueryConnection.java new file mode 100644 index 0000000..bf8d970 --- /dev/null +++ b/query/src/main/java/org/apache/kylin/query/QueryConnection.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.query; + +import java.io.File; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.query.schema.OLAPSchemaFactory; +import org.apache.log4j.Logger; + +public class QueryConnection { + private static final Logger logger = Logger.getLogger(QueryConnection.class); + + public static Connection getConnection(String project) throws SQLException { + File olapTmp = OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.getNormalizedProjectName(project), + KylinConfig.getInstanceFromEnv()); + Properties info = new Properties(); + info.put("model", olapTmp.getAbsolutePath()); + return DriverManager.getConnection("jdbc:calcite:", info); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java index 742efcf..7758987 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java @@ -147,6 +147,7 @@ public class CacheService extends BasicService { queryDataSource.clearCache(); } + @Deprecated public DataSource getOLAPDataSource(String project) { project = ProjectInstance.getNormalizedProjectName(project); http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java index 4061c82..75c16bd 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java @@ -44,7 +44,6 @@ import java.util.Map; import java.util.Set; import javax.annotation.PostConstruct; -import javax.sql.DataSource; import org.apache.calcite.avatica.ColumnMetaData.Rep; import org.apache.calcite.config.CalciteConnectionConfig; @@ -81,6 +80,7 @@ import org.apache.kylin.metadata.querymeta.SelectedColumnMeta; import org.apache.kylin.metadata.querymeta.TableMeta; import org.apache.kylin.metadata.querymeta.TableMetaWithType; import org.apache.kylin.metadata.realization.RealizationType; +import org.apache.kylin.query.QueryConnection; import org.apache.kylin.query.relnode.OLAPContext; import org.apache.kylin.query.util.QueryUtil; import org.apache.kylin.rest.constant.Constant; @@ -517,8 +517,7 @@ public class QueryService extends BasicService { } ResultSet JDBCTableMeta = null; try { - DataSource dataSource = cacheService.getOLAPDataSource(project); - conn = dataSource.getConnection(); + conn = QueryConnection.getConnection(project); DatabaseMetaData metaData = conn.getMetaData(); JDBCTableMeta = metaData.getTables(null, null, null, null); @@ -593,8 +592,7 @@ public class QueryService extends BasicService { } ResultSet JDBCTableMeta = null; try { - DataSource dataSource = cacheService.getOLAPDataSource(project); - conn = dataSource.getConnection(); + conn = QueryConnection.getConnection(project); DatabaseMetaData metaData = conn.getMetaData(); JDBCTableMeta = metaData.getTables(null, null, null, null); @@ -757,7 +755,7 @@ public class QueryService extends BasicService { List<SelectedColumnMeta> columnMetas = Lists.newArrayList(); try { - conn = cacheService.getOLAPDataSource(sqlRequest.getProject()).getConnection(); + conn = QueryConnection.getConnection(sqlRequest.getProject()); // special case for prepare query. if (BackdoorToggles.getPrepareOnly()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java index 96146ef..769d9d1 100644 --- a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java +++ b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java @@ -19,11 +19,13 @@ package org.apache.kylin.rest.service; import java.net.UnknownHostException; +import java.sql.SQLException; import java.util.List; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.job.exception.JobException; import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.query.QueryConnection; import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -45,11 +47,11 @@ public class CubeServiceTest extends ServiceTestBase { private CacheService cacheService; @Test - public void testBasics() throws JsonProcessingException, JobException, UnknownHostException { + public void testBasics() throws JsonProcessingException, JobException, UnknownHostException, SQLException { Assert.assertNotNull(cubeService.getConfig()); Assert.assertNotNull(cubeService.getConfig()); Assert.assertNotNull(cubeService.getMetadataManager()); - Assert.assertNotNull(cacheService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME)); + Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME)); List<CubeInstance> cubes = cubeService.listAllCubes(null, null, null, true); Assert.assertNotNull(cubes); http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java index 0932748..b47f05f 100644 --- a/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java +++ b/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java @@ -19,10 +19,12 @@ package org.apache.kylin.rest.service; import java.io.IOException; +import java.sql.SQLException; import org.apache.kylin.job.constant.JobTimeFilterEnum; import org.apache.kylin.job.exception.JobException; import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.query.QueryConnection; import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -42,11 +44,11 @@ public class JobServiceTest extends ServiceTestBase { private CacheService cacheService; @Test - public void testBasics() throws JobException, IOException { + public void testBasics() throws JobException, IOException, SQLException { Assert.assertNotNull(jobService.getConfig()); Assert.assertNotNull(jobService.getConfig()); Assert.assertNotNull(jobService.getMetadataManager()); - Assert.assertNotNull(cacheService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME)); + Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME)); Assert.assertNull(jobService.getJobInstance("job_not_exist")); Assert.assertNotNull(jobService.searchJobs(null, null, null, 0, 0, JobTimeFilterEnum.ALL)); } http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java index 1e5a7a5..c980959 100644 --- a/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java +++ b/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java @@ -23,6 +23,7 @@ import java.sql.SQLException; import org.apache.kylin.job.exception.JobException; import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.query.QueryConnection; import org.apache.kylin.rest.request.SQLRequest; import org.apache.kylin.rest.response.SQLResponse; import org.junit.Assert; @@ -48,7 +49,7 @@ public class QueryServiceTest extends ServiceTestBase { Assert.assertNotNull(queryService.getConfig()); Assert.assertNotNull(queryService.getConfig()); Assert.assertNotNull(queryService.getMetadataManager()); - Assert.assertNotNull(cacheService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME)); + Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME)); // Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0); //
