This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1a00e9ac57de03d3436e153dbc535a5ee5809cd2 Author: qianhao.zhou <z.qian...@gmail.com> AuthorDate: Sat Nov 12 20:41:02 2022 +0800 KYLIN-5399 refactor core-common * fix async query cannot run * fix newten-job.jar cannot run * add missing dependency Co-authored-by: qhzhou <qianhao.z...@kyligence.io> --- pom.xml | 18 ++ src/assembly/pom.xml | 4 + src/common-service/pom.xml | 4 + .../apache/kylin/rest/config/MetricsConfig.java | 4 +- .../kylin/rest/controller/NBasicController.java | 17 +- .../interceptor/ReloadAuthoritiesInterceptor.java | 5 +- .../rest/security/NUnauthorisedEntryPoint.java | 3 +- .../org/apache/kylin/rest/util/SparkUIUtil.java | 3 +- src/core-common/pom.xml | 222 ++++----------------- .../org/apache/kylin/common/KylinConfigBase.java | 4 +- .../AsyncProfilerConstants.java} | 23 +-- .../common/persistence/metadata/jdbc/JdbcUtil.java | 2 +- .../common/persistence/transaction/UnitOfWork.java | 22 +- .../java/org/apache/kylin/common/util/Unsafe.java | 8 +- src/core-job/pom.xml | 16 ++ .../kylin/common/util/SystemInfoCollector.java | 0 .../apache/kylin/metrics/HdfsCapacityMetrics.java | 0 .../kylin/metrics/HdfsCapacityMetricsTest.java | 0 src/core-metadata/pom.xml | 26 +-- .../kylin/metadata/model/ComputedColumnDesc.java | 9 - .../apache/kylin/metadata/model/FunctionDesc.java | 5 +- .../metadata/model/schema/ModelEdgeCollector.java | 12 +- .../org/apache/kylin/source/jdbc/H2Database.java | 2 +- src/{core-job => core-metrics}/pom.xml | 128 ++++++------ .../kylin/common/metrics/MetricsCategory.java | 0 .../apache/kylin/common/metrics/MetricsConfig.java | 0 .../kylin/common/metrics/MetricsController.java | 0 .../apache/kylin/common/metrics/MetricsGroup.java | 0 .../common/metrics/MetricsInfluxdbReporter.java | 0 .../apache/kylin/common/metrics/MetricsName.java | 0 .../apache/kylin/common/metrics/MetricsObject.java | 0 .../kylin/common/metrics/MetricsObjectType.java | 0 .../kylin/common/metrics/MetricsReporter.java | 0 .../apache/kylin/common/metrics/MetricsTag.java | 0 .../common/metrics/context/ClusterContext.java | 0 .../kylin/common/metrics/context/ClusterInfo.java | 0 .../common/metrics/gauges/QueryRatioGauge.java | 0 .../metrics/prometheus/PrometheusMetrics.java | 0 .../common/metrics/reporter/InfluxdbReporter.java | 2 +- .../metrics/reporter/ServerModeMetricFilter.java | 0 .../common/metrics/service/InfluxDBInstance.java | 4 +- .../metrics/service/JobStatusMonitorMetric.java | 0 .../kylin/common/metrics/service/MonitorDao.java | 0 .../common/metrics/service/MonitorMetric.java | 0 .../metrics/service/MonitorMetricOperation.java | 0 .../common/metrics/service/QueryMonitorMetric.java | 0 .../apache/kylin/common/util/InfluxDBUtils.java | 0 .../kylin/common/metric/InfluxDBInstanceTest.java | 0 .../kylin/common/metric/InfluxMetricsTest.java | 0 .../kylin/common/metric/MetricsControllerTest.java | 16 +- .../apache/kylin/common/metric/MonitorDaoTest.java | 0 .../kylin/rest/controller/BaseController.java | 19 +- .../kylin/rest/filter/SegmentsRequestFilter.java | 3 +- src/datasource-sdk/pom.xml | 27 +-- src/datasource-service/pom.xml | 4 + src/{core-job => distributed-lock-ext}/pom.xml | 107 +++++----- .../lock/curator/CuratorDistributedLock.java | 0 .../curator/CuratorDistributedLockFactory.java | 0 .../lock/jdbc/JdbcDistributedLockFactory.java | 0 .../common/lock/jdbc/JdbcDistributedLockUtil.java | 24 ++- .../java/org/apache/kylin/common/util/ZKUtil.java | 8 +- .../common/lock/DistributedLockFactoryTest.java | 11 +- .../curator/CuratorDistributedLockFactoryTest.java | 43 ++-- .../lock/jdbc/JdbcDistributedLockFactoryTest.java | 13 +- src/kylin-it/pom.xml | 5 + .../kylin/streaming/StreamingMergeEntryTest.java | 9 + .../kylin/common/util/AddTableNameSqlVisitor.java | 0 .../common/util/ModifyTableNameSqlVisitor.java | 0 .../kylin/rest/service/ModelSemanticHelper.java | 9 +- src/query/pom.xml | 4 + src/server/pom.xml | 24 +++ .../org/apache/kylin/rest/QueryNodeFilter.java | 7 +- src/spark-project/engine-build-sdk/pom.xml | 1 - src/spark-project/engine-spark/pom.xml | 4 + .../kylin/engine/spark/job/JobManagerTest.java | 29 +-- src/spark-project/spark-common/pom.xml | 5 + .../kylin/common/asyncprofiler/AsyncArchUtil.java | 0 .../kylin/common/asyncprofiler/AsyncProfiler.java | 12 -- .../common/asyncprofiler/AsyncProfilerUtils.java | 0 .../org/apache/kylin/source/dfs/DFSFileTable.java | 0 .../kylin/source/dfs/DFSFileTableReader.java | 4 +- .../libasyncProfiler-linux-arm64.so | Bin 0 -> 298256 bytes .../libasyncProfiler-linux-x64.so | Bin 0 -> 303903 bytes .../async-profiler-lib/libasyncProfiler-mac.so | Bin 0 -> 634272 bytes .../async-profiler-lib/linux64/libasyncProfiler.so | Bin 0 -> 314098 bytes .../async-profiler-lib/macOS/libasyncProfiler.so | Bin 0 -> 239460 bytes .../AsyncProfilerExecutorPlugin.scala | 0 .../common/asyncprofiler/AsyncProfilerTool.scala | 2 +- .../kylin/common/asyncprofiler/Message.scala | 4 +- .../common/asyncprofiler/AsyncArchUtilTest.java | 0 .../common/asyncprofiler/AsyncProfilerTest.java | 18 -- .../asyncprofiler/AsyncProfilerToolTest.java | 0 .../asyncprofiler/AsyncProfilerUtilsTest.java | 6 +- .../kylin/common/asyncprofiler/MessageTest.java | 3 +- src/systools/pom.xml | 22 +- 95 files changed, 425 insertions(+), 561 deletions(-) diff --git a/pom.xml b/pom.xml index ad78eecd6d..40eb6b25f9 100644 --- a/pom.xml +++ b/pom.xml @@ -317,6 +317,8 @@ <module>src/query</module> <module>src/core-common</module> <module>src/core-metadata</module> + <module>src/distributed-lock-ext</module> + <module>src/core-metrics</module> <module>src/source-hive</module> <module>src/server</module> <module>src/spark-project/kylin-soft-affinity-cache</module> @@ -523,6 +525,22 @@ <artifactId>kylin-integration-service</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>distributed-lock-ext</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-core-metrics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-streaming</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + </dependency> <!-- arthas --> <dependency> diff --git a/src/assembly/pom.xml b/src/assembly/pom.xml index 2a56110bfc..5436fda9bd 100644 --- a/src/assembly/pom.xml +++ b/src/assembly/pom.xml @@ -88,6 +88,10 @@ <groupId>org.apache.kylin</groupId> <artifactId>kylin-query</artifactId> </dependency> + <dependency> + <groupId>org.postgresql</groupId> + <artifactId>postgresql</artifactId> + </dependency> <!-- Env --> <dependency> <groupId>org.apache.hadoop</groupId> diff --git a/src/common-service/pom.xml b/src/common-service/pom.xml index 6226d6ea78..d6fcd55cc1 100644 --- a/src/common-service/pom.xml +++ b/src/common-service/pom.xml @@ -59,6 +59,10 @@ <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> </dependency> + <dependency> + <groupId>commons-fileupload</groupId> + <artifactId>commons-fileupload</artifactId> + </dependency> <dependency> <groupId>net.sf.ehcache</groupId> diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/MetricsConfig.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/MetricsConfig.java index a0126eee3e..0289709b62 100644 --- a/src/common-service/src/main/java/org/apache/kylin/rest/config/MetricsConfig.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/MetricsConfig.java @@ -28,11 +28,11 @@ import java.util.stream.Collectors; import org.apache.kylin.common.KapConfig; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.NamedThreadFactory; -import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.common.metrics.MetricsController; import org.apache.kylin.common.metrics.MetricsGroup; +import org.apache.kylin.common.util.NamedThreadFactory; import org.apache.kylin.metadata.project.NProjectManager; +import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.rest.cluster.ClusterManager; import org.apache.kylin.rest.config.initialize.MetricsRegistry; import org.springframework.beans.factory.annotation.Autowired; diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java b/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java index 4ffe05c013..884a807128 100644 --- a/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java @@ -83,7 +83,6 @@ import org.apache.kylin.common.msg.MsgPicker; import org.apache.kylin.common.persistence.transaction.TransactionException; import org.apache.kylin.common.util.DateFormat; import org.apache.kylin.common.util.JsonUtil; -import org.apache.kylin.common.util.Unsafe; import org.apache.kylin.job.constant.JobStatusEnum; import org.apache.kylin.job.dao.ExecutablePO; import org.apache.kylin.job.execution.JobTypeEnum; @@ -188,7 +187,7 @@ public class NBasicController { if (kylinException != null) { cause = kylinException; } - return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), cause); + return new ErrorResponse(req.getRequestURL().toString(), cause); } @ResponseStatus(HttpStatus.FORBIDDEN) @@ -196,7 +195,7 @@ public class NBasicController { @ResponseBody ErrorResponse handleForbidden(HttpServletRequest req, Exception ex) { getLogger().error("", ex); - return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex); + return new ErrorResponse(req.getRequestURL().toString(), ex); } @ResponseStatus(HttpStatus.NOT_FOUND) @@ -204,7 +203,7 @@ public class NBasicController { @ResponseBody ErrorResponse handleNotFound(HttpServletRequest req, Exception ex) { getLogger().error("", ex); - return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex); + return new ErrorResponse(req.getRequestURL().toString(), ex); } @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) @@ -228,7 +227,7 @@ public class NBasicController { ErrorResponse handleAccessDenied(HttpServletRequest req, Throwable ex) { getLogger().error("", ex); KylinException e = new KylinException(ACCESS_DENIED, MsgPicker.getMsg().getAccessDeny()); - return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), e); + return new ErrorResponse(req.getRequestURL().toString(), e); } @ResponseStatus(HttpStatus.BAD_REQUEST) @@ -238,7 +237,7 @@ public class NBasicController { ErrorResponse handleInvalidRequestParam(HttpServletRequest req, Throwable ex) { KylinException e = new KylinException(INVALID_PARAMETER, ex); getLogger().error("", e); - return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), e); + return new ErrorResponse(req.getRequestURL().toString(), e); } @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) @@ -247,14 +246,14 @@ public class NBasicController { ErrorResponse handleErrorCode(HttpServletRequest req, Throwable ex) { getLogger().error("", ex); KylinException cause = (KylinException) ex; - return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), cause); + return new ErrorResponse(req.getRequestURL().toString(), cause); } @ResponseStatus(HttpStatus.BAD_REQUEST) @ExceptionHandler(MethodArgumentNotValidException.class) @ResponseBody ErrorResponse handleInvalidArgument(HttpServletRequest request, MethodArgumentNotValidException ex) { - val response = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(request), ex); + val response = new ErrorResponse(request.getRequestURL().toString(), ex); val target = ex.getBindingResult().getTarget(); if (target instanceof Validation) { response.setMsg(((Validation) target).getErrorMessage(ex.getBindingResult().getFieldErrors())); @@ -272,7 +271,7 @@ public class NBasicController { ErrorResponse handleUnauthorized(HttpServletRequest req, Throwable ex) { KylinException e = new KylinException(USER_UNAUTHORIZED, ex); getLogger().error("", e); - return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex); + return new ErrorResponse(req.getRequestURL().toString(), ex); } protected void checkRequiredArg(String fieldName, Object fieldValue) { diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/ReloadAuthoritiesInterceptor.java b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/ReloadAuthoritiesInterceptor.java index 0d8714647a..1d48d51d66 100644 --- a/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/ReloadAuthoritiesInterceptor.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/ReloadAuthoritiesInterceptor.java @@ -31,10 +31,9 @@ import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.exception.KylinException; import org.apache.kylin.common.msg.MsgPicker; import org.apache.kylin.common.util.JsonUtil; +import org.apache.kylin.metadata.user.ManagedUser; import org.apache.kylin.rest.response.ErrorResponse; import org.apache.kylin.rest.service.UserService; -import org.apache.kylin.common.util.Unsafe; -import org.apache.kylin.metadata.user.ManagedUser; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.annotation.Order; @@ -84,7 +83,7 @@ public class ReloadAuthoritiesInterceptor extends HandlerInterceptorAdapter { if (present) { response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); response.setContentType(MediaType.APPLICATION_JSON_VALUE); - ErrorResponse errorResponse = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(request), + ErrorResponse errorResponse = new ErrorResponse(request.getRequestURL().toString(), new KylinException(USER_DATA_SOURCE_CONNECTION_FAILED, MsgPicker.getMsg().getlDapUserDataSourceConnectionFailed())); response.setCharacterEncoding("UTF-8"); diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/security/NUnauthorisedEntryPoint.java b/src/common-service/src/main/java/org/apache/kylin/rest/security/NUnauthorisedEntryPoint.java index 6c376b81f7..3aceb2ad5c 100644 --- a/src/common-service/src/main/java/org/apache/kylin/rest/security/NUnauthorisedEntryPoint.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/security/NUnauthorisedEntryPoint.java @@ -37,7 +37,6 @@ import org.apache.kylin.common.exception.KylinException; import org.apache.kylin.common.msg.MsgPicker; import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.rest.response.ErrorResponse; -import org.apache.kylin.common.util.Unsafe; import org.springframework.http.MediaType; import org.springframework.ldap.CommunicationException; import org.springframework.security.authentication.DisabledException; @@ -94,7 +93,7 @@ public class NUnauthorisedEntryPoint implements AuthenticationEntryPoint { throws IOException { response.setStatus(statusCode); response.setContentType(MediaType.APPLICATION_JSON_VALUE); - ErrorResponse errorResponse = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(request), ex); + ErrorResponse errorResponse = new ErrorResponse(request.getRequestURL().toString(), ex); String errorStr = JsonUtil.writeValueAsIndentString(errorResponse); response.setCharacterEncoding("UTF-8"); PrintWriter writer = response.getWriter(); diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/util/SparkUIUtil.java b/src/common-service/src/main/java/org/apache/kylin/rest/util/SparkUIUtil.java index a814b725d0..51a01d6c1e 100644 --- a/src/common-service/src/main/java/org/apache/kylin/rest/util/SparkUIUtil.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/util/SparkUIUtil.java @@ -28,7 +28,6 @@ import javax.servlet.http.HttpServletResponse; import org.apache.commons.io.IOUtils; import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.kylin.common.util.Unsafe; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpHeaders; @@ -59,7 +58,7 @@ public class SparkUIUtil { final HttpMethod method = HttpMethod.resolve(servletRequest.getMethod()); try (ClientHttpResponse response = execute(target, method, proxyLocationBase)) { - rewrite(response, servletResponse, method, Unsafe.getUrlFromHttpServletRequest(servletRequest), + rewrite(response, servletResponse, method, servletRequest.getRequestURL().toString(), REDIRECT_THRESHOLD, proxyLocationBase); } } diff --git a/src/core-common/pom.xml b/src/core-common/pom.xml index 6c1211efe8..47bd12746c 100644 --- a/src/core-common/pom.xml +++ b/src/core-common/pom.xml @@ -37,9 +37,8 @@ <dependencies> <dependency> <groupId>io.kyligence.ke</groupId> - <artifactId>kap-external-curator</artifactId> + <artifactId>kap-external-guava20</artifactId> </dependency> - <dependency> <groupId>io.kyligence</groupId> <artifactId>kyligence-config-external</artifactId> @@ -50,67 +49,25 @@ </exclusion> </exclusions> </dependency> - - <!-- provided --> - <dependency> - <groupId>io.netty</groupId> - <artifactId>netty</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdfs</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.tomcat.embed</groupId> - <artifactId>tomcat-embed-core</artifactId> - </dependency> - - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - </dependency> <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-email</artifactId> </dependency> - <dependency> - <groupId>commons-fileupload</groupId> - <artifactId>commons-fileupload</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </dependency> <dependency> <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> </dependency> - <dependency> - <groupId>com.lmax</groupId> - <artifactId>disruptor</artifactId> - </dependency> <dependency> <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-api</artifactId> </dependency> <dependency> <groupId>org.apache.logging.log4j</groupId> - <artifactId>log4j-core</artifactId> + <artifactId>log4j-web</artifactId> </dependency> <dependency> <groupId>org.apache.logging.log4j</groupId> - <artifactId>log4j-web</artifactId> + <artifactId>log4j-core</artifactId> </dependency> <dependency> <groupId>org.apache.logging.log4j</groupId> @@ -121,69 +78,42 @@ <artifactId>log4j-slf4j-impl</artifactId> </dependency> <dependency> - <groupId>org.mybatis</groupId> - <artifactId>mybatis</artifactId> + <groupId>javax.validation</groupId> + <artifactId>validation-api</artifactId> </dependency> <dependency> - <groupId>org.mybatis.dynamic-sql</groupId> - <artifactId>mybatis-dynamic-sql</artifactId> - </dependency> - - <!-- dropwizard --> - <dependency> - <groupId>io.dropwizard.metrics</groupId> - <artifactId>metrics-core</artifactId> - </dependency> - <dependency> - <groupId>io.dropwizard.metrics</groupId> - <artifactId>metrics-jvm</artifactId> - </dependency> - <dependency> - <groupId>io.dropwizard.metrics</groupId> - <artifactId>metrics-json</artifactId> - </dependency> - <dependency> - <groupId>io.dropwizard.metrics</groupId> - <artifactId>metrics-jmx</artifactId> - </dependency> - - <!-- micrometer --> - <dependency> - <groupId>io.micrometer</groupId> - <artifactId>micrometer-registry-prometheus</artifactId> + <groupId>com.alibaba</groupId> + <artifactId>transmittable-thread-local</artifactId> </dependency> - <dependency> - <groupId>org.latencyutils</groupId> - <artifactId>LatencyUtils</artifactId> + <groupId>org.springframework</groupId> + <artifactId>spring-jdbc</artifactId> + <exclusions> + <exclusion> + <groupId>org.springframework</groupId> + <artifactId>spring-jcl</artifactId> + </exclusion> + </exclusions> </dependency> - + <!-- provided --> <dependency> - <groupId>io.kyligence.ke</groupId> - <artifactId>kap-external-guava20</artifactId> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-core</artifactId> + <scope>provided</scope> </dependency> <dependency> - <groupId>io.kyligence.ke</groupId> - <artifactId>kap-external-influxdb</artifactId> + <groupId>org.apache.commons</groupId> + <artifactId>commons-dbcp2</artifactId> + <scope>provided</scope> </dependency> + <!-- test --> <dependency> <groupId>joda-time</groupId> <artifactId>joda-time</artifactId> <scope>test</scope> <version>2.10.2</version> </dependency> - - <dependency> - <groupId>com.github.oshi</groupId> - <artifactId>oshi-core</artifactId> - </dependency> - - <dependency> - <groupId>org.javassist</groupId> - <artifactId>javassist</artifactId> - </dependency> - <!--Env & Test--> <dependency> <groupId>com.github.sakserv</groupId> <artifactId>hadoop-mini-clusters-hdfs</artifactId> @@ -191,25 +121,8 @@ <version>0.1.16</version> </dependency> <dependency> - <groupId>mysql</groupId> - <artifactId>mysql-connector-java</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.postgresql</groupId> - <artifactId>postgresql</artifactId> - </dependency> - - <!-- postgresql 42.2.25 not dependency jcl-over-slf4j, ut testFindContainingJar will fail --> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>jcl-over-slf4j</artifactId> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.apache.curator</groupId> - <artifactId>curator-test</artifactId> + <groupId>org.assertj</groupId> + <artifactId>assertj-core</artifactId> <scope>test</scope> </dependency> <dependency> @@ -217,43 +130,6 @@ <artifactId>spring-test</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.kafka</groupId> - <artifactId>kafka_2.12</artifactId> - <scope>provided</scope> - </dependency> - - <!-- jdbc persistence --> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-jdbc</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-context-support</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.springframework.security</groupId> - <artifactId>spring-security-core</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-dbcp2</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.assertj</groupId> - <artifactId>assertj-core</artifactId> - </dependency> - <dependency> <groupId>org.awaitility</groupId> <artifactId>awaitility</artifactId> @@ -274,11 +150,6 @@ <artifactId>junit-pioneer</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>com.h2database</groupId> - <artifactId>h2</artifactId> - <scope>test</scope> - </dependency> <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> @@ -289,7 +160,6 @@ <artifactId>log4j-core</artifactId> </exclusion> </exclusions> - <version>${mockito.version}</version> </dependency> <dependency> <groupId>org.mockito</groupId> @@ -301,7 +171,6 @@ <artifactId>log4j-core</artifactId> </exclusion> </exclusions> - <version>${mockito.version}</version> </dependency> <dependency> <groupId>org.powermock</groupId> @@ -314,32 +183,15 @@ <scope>test</scope> </dependency> <dependency> - <groupId>commons-codec</groupId> - <artifactId>commons-codec</artifactId> - </dependency> - <dependency> - <groupId>org.apache.calcite</groupId> - <artifactId>calcite-core</artifactId> - </dependency> - <dependency> - <groupId>com.alibaba</groupId> - <artifactId>transmittable-thread-local</artifactId> - </dependency> - <dependency> - <groupId>org.springframework.cloud</groupId> - <artifactId>spring-cloud-commons</artifactId> - </dependency> - <dependency> - <groupId>org.springframework.integration</groupId> - <artifactId>spring-integration-jdbc</artifactId> + <groupId>com.h2database</groupId> + <artifactId>h2</artifactId> + <scope>test</scope> </dependency> - - <!-- Spark --> <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-core_2.12</artifactId> + <groupId>org.slf4j</groupId> + <artifactId>jcl-over-slf4j</artifactId> + <scope>test</scope> </dependency> - </dependencies> <build> <plugins> @@ -374,6 +226,18 @@ <groupId>net.alchim31.maven</groupId> <artifactId>scala-maven-plugin</artifactId> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>3.0.2</version> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> </plugins> </build> </project> diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 557901c837..7bd4da912a 100644 --- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -19,8 +19,8 @@ package org.apache.kylin.common; import static java.lang.Math.toIntExact; -import static org.apache.kylin.common.asyncprofiler.AsyncProfiler.ASYNC_PROFILER_LIB_LINUX_ARM64; -import static org.apache.kylin.common.asyncprofiler.AsyncProfiler.ASYNC_PROFILER_LIB_LINUX_X64; +import static org.apache.kylin.common.constant.AsyncProfilerConstants.ASYNC_PROFILER_LIB_LINUX_ARM64; +import static org.apache.kylin.common.constant.AsyncProfilerConstants.ASYNC_PROFILER_LIB_LINUX_X64; import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_CONNECTION_URL_KEY; import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_DRIVER_KEY; import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_PASS_KEY; diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java b/src/core-common/src/main/java/org/apache/kylin/common/constant/AsyncProfilerConstants.java similarity index 66% copy from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java copy to src/core-common/src/main/java/org/apache/kylin/common/constant/AsyncProfilerConstants.java index 1502b72de5..1949f0db98 100644 --- a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java +++ b/src/core-common/src/main/java/org/apache/kylin/common/constant/AsyncProfilerConstants.java @@ -15,25 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.kylin.common.constant; -package org.apache.kylin.common.metrics; +public final class AsyncProfilerConstants { -public enum MetricsObjectType { + private AsyncProfilerConstants() {} - MAX("max"), - MIN("min"), - COUNT("count"), - FIRST("first"), - LAST("last"); - - private String type; - - MetricsObjectType(String type) { - this.type = type; - } - - public String getVal() { - return this.type; - } + public static final String ASYNC_PROFILER_LIB_MAC = "libasyncProfiler-mac.so"; + public static final String ASYNC_PROFILER_LIB_LINUX_X64 = "libasyncProfiler-linux-x64.so"; + public static final String ASYNC_PROFILER_LIB_LINUX_ARM64 = "libasyncProfiler-linux-arm64.so"; } diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java index 94734b415e..caf500bc14 100644 --- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java +++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java @@ -36,7 +36,6 @@ import org.apache.kylin.common.exception.KylinException; import org.apache.kylin.common.msg.MsgPicker; import org.apache.kylin.common.persistence.metadata.PersistException; import org.apache.kylin.common.util.EncryptUtil; -import org.msgpack.core.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.dao.DataIntegrityViolationException; @@ -45,6 +44,7 @@ import org.springframework.jdbc.datasource.DataSourceTransactionManager; import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.support.DefaultTransactionDefinition; +import io.kyligence.kap.guava20.shaded.common.annotations.VisibleForTesting; import lombok.val; import lombok.extern.slf4j.Slf4j; diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java index 280e1987d7..4e576d4e4f 100644 --- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java +++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java @@ -18,7 +18,6 @@ package org.apache.kylin.common.persistence.transaction; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -31,11 +30,6 @@ import org.apache.kylin.common.persistence.RawResource; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.ThreadViewResourceStore; import org.apache.kylin.common.persistence.TombRawResource; -import org.apache.kylin.common.util.Pair; -import org.apache.kylin.common.util.RandomUtil; -import org.apache.kylin.common.metrics.MetricsCategory; -import org.apache.kylin.common.metrics.MetricsGroup; -import org.apache.kylin.common.metrics.MetricsName; import org.apache.kylin.common.persistence.UnitMessages; import org.apache.kylin.common.persistence.event.EndUnit; import org.apache.kylin.common.persistence.event.Event; @@ -44,6 +38,8 @@ import org.apache.kylin.common.persistence.event.ResourceDeleteEvent; import org.apache.kylin.common.persistence.event.ResourceRelatedEvent; import org.apache.kylin.common.persistence.event.StartUnit; import org.apache.kylin.common.scheduler.EventBusFactory; +import org.apache.kylin.common.util.Pair; +import org.apache.kylin.common.util.RandomUtil; import org.apache.kylin.common.util.Unsafe; import com.google.common.base.Preconditions; @@ -100,17 +96,6 @@ public class UnitOfWork { int retry = 0; val traceId = RandomUtil.randomUUIDStr(); while (retry++ < maxRetry) { - if (retry > 1) { - Map<String, String> tags = MetricsGroup.getHostTagMap(params.getUnitName()); - - if (!GLOBAL_UNIT.equals(params.getUnitName())) { - MetricsGroup.counterInc(MetricsName.TRANSACTION_RETRY_COUNTER, MetricsCategory.PROJECT, - params.getUnitName(), tags); - } else { - MetricsGroup.counterInc(MetricsName.TRANSACTION_RETRY_COUNTER, MetricsCategory.GLOBAL, "global", - tags); - } - } val ret = doTransaction(params, retry, traceId); if (ret.getSecond()) { @@ -147,9 +132,6 @@ public class UnitOfWork { long duration = System.currentTimeMillis() - startTransactionTime; logIfLongTransaction(duration, traceId); - MetricsGroup.hostTagHistogramUpdate(MetricsName.TRANSACTION_LATENCY, MetricsCategory.PROJECT, - !GLOBAL_UNIT.equals(params.getUnitName()) ? params.getUnitName() : "global", duration); - result = Pair.newPair(ret, true); } catch (Throwable throwable) { handleError(throwable, params, retry, traceId); diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java b/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java index 552b9edc44..cfd276f15d 100644 --- a/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java +++ b/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java @@ -24,12 +24,10 @@ import java.text.MessageFormat; import java.util.Locale; import java.util.Map; -import javax.servlet.http.HttpServletRequest; - import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.SystemPropertiesCache; import lombok.extern.slf4j.Slf4j; -import org.apache.kylin.common.SystemPropertiesCache; /** * Contains methods that call JDK methods that the @@ -81,10 +79,6 @@ public class Unsafe { return temp.format(arguments); } - public static String getUrlFromHttpServletRequest(HttpServletRequest request) { - return request.getRequestURL().toString(); - } - /** Reflection usage to work around access flags fails with SecurityManagers * and likely will not work anymore on runtime classes in Java 9 */ public static void changeAccessibleObject(AccessibleObject accessibleObject, boolean value) { diff --git a/src/core-job/pom.xml b/src/core-job/pom.xml index bbdbb0cf2f..a5ac6dff24 100644 --- a/src/core-job/pom.xml +++ b/src/core-job/pom.xml @@ -45,6 +45,18 @@ <groupId>io.kyligence.ke</groupId> <artifactId>kap-external-curator-test</artifactId> </dependency> + <dependency> + <groupId>io.kyligence.ke</groupId> + <artifactId>kap-external-curator</artifactId> + </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-core-metrics</artifactId> + </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-core-metrics</artifactId> + </dependency> <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-common</artifactId> @@ -60,6 +72,10 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> </dependency> + <dependency> + <groupId>com.github.oshi</groupId> + <artifactId>oshi-core</artifactId> + </dependency> <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java b/src/core-job/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java rename to src/core-job/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java b/src/core-job/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java similarity index 100% rename from src/core-metadata/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java rename to src/core-job/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java b/src/core-job/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java similarity index 100% rename from src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java rename to src/core-job/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java diff --git a/src/core-metadata/pom.xml b/src/core-metadata/pom.xml index 96760a701c..d61c5ab80e 100644 --- a/src/core-metadata/pom.xml +++ b/src/core-metadata/pom.xml @@ -36,22 +36,17 @@ </properties> <dependencies> - - <!-- provided --> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <scope>provided</scope> + <groupId>commons-collections</groupId> + <artifactId>commons-collections</artifactId> </dependency> - <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-common</artifactId> </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-jdbc</artifactId> + <groupId>io.kyligence.ke</groupId> + <artifactId>kap-external-influxdb</artifactId> </dependency> <dependency> <groupId>com.tdunning</groupId> @@ -65,17 +60,13 @@ <groupId>org.apache.commons</groupId> <artifactId>commons-dbcp2</artifactId> </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </dependency> <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> </dependency> <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> </dependency> <dependency> <groupId>com.esotericsoftware</groupId> @@ -109,6 +100,11 @@ <type>test-jar</type> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.junit.vintage</groupId> <artifactId>junit-vintage-engine</artifactId> diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java index 5cceedd5df..f807dea94d 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java @@ -32,7 +32,6 @@ import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.dialect.HiveSqlDialect; import org.apache.calcite.sql.util.SqlBasicVisitor; import org.apache.calcite.sql.util.SqlVisitor; import org.apache.commons.lang.StringUtils; @@ -40,7 +39,6 @@ import org.apache.kylin.common.exception.KylinException; import org.apache.kylin.common.msg.MsgPicker; import org.apache.kylin.measure.MeasureTypeFactory; import org.apache.kylin.metadata.model.tool.CalciteParser; -import org.apache.kylin.common.util.ModifyTableNameSqlVisitor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -211,11 +209,4 @@ public class ComputedColumnDesc implements Serializable { return innerExpression; } - public void changeTableAlias(String oldAlias, String newAlias) { - SqlVisitor<Object> modifyAlias = new ModifyTableNameSqlVisitor(oldAlias, newAlias); - SqlNode sqlNode = CalciteParser.getExpNode(getExpression()); - sqlNode.accept(modifyAlias); - setExpression(sqlNode.toSqlString(HiveSqlDialect.DEFAULT).toString()); - } - } diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java index b2a212f84f..28e3603d28 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java @@ -42,8 +42,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.directory.api.util.Strings; +import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.exception.KylinException; import org.apache.kylin.measure.MeasureType; import org.apache.kylin.measure.MeasureTypeFactory; @@ -104,7 +103,7 @@ public class FunctionDesc implements Serializable { break; } case FunctionDesc.FUNC_SUM_LC: { - Preconditions.checkArgument(Strings.isNotEmpty(colDataType), + Preconditions.checkArgument(StringUtils.isNotEmpty(colDataType), "SUM_LC Measure's input type shouldn't be null or empty"); checkSumLCDataType(colDataType); break; diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ModelEdgeCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ModelEdgeCollector.java index 8b94d8c7a9..ccda8a9527 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ModelEdgeCollector.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ModelEdgeCollector.java @@ -17,6 +17,7 @@ */ package org.apache.kylin.metadata.model.schema; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -26,14 +27,14 @@ import java.util.stream.Collectors; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.util.Pair; -import org.apache.kylin.metadata.model.JoinTableDesc; -import org.apache.kylin.metadata.model.ParameterDesc; -import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.cube.cuboid.NAggregationGroup; import org.apache.kylin.metadata.cube.model.IndexPlan; import org.apache.kylin.metadata.cube.model.LayoutEntity; import org.apache.kylin.metadata.model.ComputedColumnDesc; +import org.apache.kylin.metadata.model.JoinTableDesc; import org.apache.kylin.metadata.model.NDataModel; +import org.apache.kylin.metadata.model.ParameterDesc; +import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.model.util.ComputedColumnUtil; import com.google.common.collect.ImmutableBiMap; @@ -41,7 +42,6 @@ import com.google.common.collect.Maps; import io.kyligence.kap.guava20.shaded.common.graph.Graph; import io.kyligence.kap.guava20.shaded.common.graph.MutableGraph; -import io.kyligence.kap.shaded.curator.org.apache.curator.shaded.com.google.common.collect.Lists; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.val; @@ -79,9 +79,9 @@ class ModelEdgeCollector { collectModelSignificant(); collectDimensionAndMeasure(); - collectIndex(indexPlan.getWhitelistLayouts(), SchemaNodeType.WHITE_LIST_INDEX, Lists.newArrayList()); + collectIndex(indexPlan.getWhitelistLayouts(), SchemaNodeType.WHITE_LIST_INDEX, new ArrayList<>()); collectIndex(indexPlan.getToBeDeletedIndexes().stream().flatMap(index -> index.getLayouts().stream()) - .collect(Collectors.toList()), SchemaNodeType.TO_BE_DELETED_INDEX, Lists.newArrayList()); + .collect(Collectors.toList()), SchemaNodeType.TO_BE_DELETED_INDEX, new ArrayList<>()); collectIndex(indexPlan.getRuleBaseLayouts(), SchemaNodeType.RULE_BASED_INDEX, indexPlan.getAggShardByColumns()); collectAggGroup(); diff --git a/src/source-hive/src/main/java/org/apache/kylin/source/jdbc/H2Database.java b/src/core-metadata/src/main/java/org/apache/kylin/source/jdbc/H2Database.java similarity index 100% rename from src/source-hive/src/main/java/org/apache/kylin/source/jdbc/H2Database.java rename to src/core-metadata/src/main/java/org/apache/kylin/source/jdbc/H2Database.java index 2df4a82cda..15e933ee13 100644 --- a/src/source-hive/src/main/java/org/apache/kylin/source/jdbc/H2Database.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/source/jdbc/H2Database.java @@ -36,8 +36,8 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.metadata.model.ColumnDesc; -import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.NTableMetadataManager; +import org.apache.kylin.metadata.model.TableDesc; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/core-job/pom.xml b/src/core-metrics/pom.xml similarity index 51% copy from src/core-job/pom.xml copy to src/core-metrics/pom.xml index bbdbb0cf2f..a7e8742a4e 100644 --- a/src/core-job/pom.xml +++ b/src/core-metrics/pom.xml @@ -1,78 +1,83 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> +<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <artifactId>kylin-core-job</artifactId> - <packaging>jar</packaging> - <name>Kylin - Core Job</name> - <url>http://kylin.apache.org</url> - <description>Kylin Core Job</description> - <parent> - <groupId>org.apache.kylin</groupId> <artifactId>kylin</artifactId> + <groupId>org.apache.kylin</groupId> <version>5.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> + <name>Kylin - Core Metrics</name> + <url>http://kylin.apache.org</url> + <modelVersion>4.0.0</modelVersion> + + <artifactId>kylin-core-metrics</artifactId> <properties> + <maven.compiler.source>8</maven.compiler.source> + <maven.compiler.target>8</maven.compiler.target> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> </properties> <dependencies> - <!-- provided --> <dependency> <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-metadata</artifactId> + <artifactId>kylin-core-common</artifactId> </dependency> <dependency> <groupId>io.kyligence.ke</groupId> - <artifactId>kap-external-curator-test</artifactId> + <artifactId>kap-external-influxdb</artifactId> </dependency> <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-common</artifactId> - <type>test-jar</type> - <scope>test</scope> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-core</artifactId> </dependency> <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-metadata</artifactId> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-jvm</artifactId> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-json</artifactId> + </dependency> + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-jmx</artifactId> </dependency> + <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> + <groupId>commons-collections</groupId> + <artifactId>commons-collections</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-core</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>org.junit.vintage</groupId> - <artifactId>junit-vintage-engine</artifactId> - </exclusion> - </exclusions> </dependency> - <!--Env & Test--> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-core-common</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-test</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-api</artifactId> @@ -84,30 +89,41 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.assertj</groupId> - <artifactId>assertj-core</artifactId> + <groupId>org.junit-pioneer</groupId> + <artifactId>junit-pioneer</artifactId> <scope>test</scope> </dependency> <dependency> - <groupId>org.awaitility</groupId> - <artifactId>awaitility</artifactId> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> + <groupId>org.powermock</groupId> + <artifactId>powermock-module-junit4</artifactId> <scope>test</scope> </dependency> <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-inline</artifactId> + <groupId>org.powermock</groupId> + <artifactId>powermock-api-mockito2</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.assertj</groupId> + <artifactId>assertj-core</artifactId> <scope>test</scope> </dependency> <dependency> - <groupId>com.h2database</groupId> - <artifactId>h2</artifactId> + <groupId>org.javassist</groupId> + <artifactId>javassist</artifactId> <scope>test</scope> </dependency> </dependencies> -</project> +</project> \ No newline at end of file diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsCategory.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsCategory.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsCategory.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsCategory.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsConfig.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsConfig.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsConfig.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsConfig.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsController.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsController.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsController.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsController.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsName.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsName.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsName.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsName.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObject.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsObject.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObject.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsObject.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsReporter.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsReporter.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsReporter.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsReporter.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsTag.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsTag.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsTag.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsTag.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/context/ClusterContext.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/context/ClusterContext.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/context/ClusterContext.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/context/ClusterContext.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/context/ClusterInfo.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/context/ClusterInfo.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/context/ClusterInfo.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/context/ClusterInfo.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/gauges/QueryRatioGauge.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/gauges/QueryRatioGauge.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/gauges/QueryRatioGauge.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/gauges/QueryRatioGauge.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/prometheus/PrometheusMetrics.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/prometheus/PrometheusMetrics.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/prometheus/PrometheusMetrics.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/prometheus/PrometheusMetrics.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java index 965a183567..5686c85f97 100644 --- a/src/core-common/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java +++ b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java @@ -36,12 +36,12 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.function.Function; -import org.apache.kylin.common.util.Pair; import org.apache.kylin.common.metrics.MetricsCategory; import org.apache.kylin.common.metrics.MetricsGroup; import org.apache.kylin.common.metrics.MetricsName; import org.apache.kylin.common.metrics.service.InfluxDBInstance; import org.apache.kylin.common.util.AddressUtil; +import org.apache.kylin.common.util.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/reporter/ServerModeMetricFilter.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/reporter/ServerModeMetricFilter.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/reporter/ServerModeMetricFilter.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/reporter/ServerModeMetricFilter.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java index fc821e01e3..3f8e96f31b 100644 --- a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java +++ b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java @@ -26,10 +26,10 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KapConfig; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.ExecutorServiceUtil; -import org.apache.kylin.common.util.NamedThreadFactory; import org.apache.kylin.common.metrics.MetricsGroup; +import org.apache.kylin.common.util.ExecutorServiceUtil; import org.apache.kylin.common.util.InfluxDBUtils; +import org.apache.kylin.common.util.NamedThreadFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/JobStatusMonitorMetric.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/JobStatusMonitorMetric.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/JobStatusMonitorMetric.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/JobStatusMonitorMetric.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorDao.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorDao.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorDao.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorDao.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetric.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetric.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetric.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetric.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetricOperation.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetricOperation.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetricOperation.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetricOperation.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/QueryMonitorMetric.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/QueryMonitorMetric.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/QueryMonitorMetric.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/QueryMonitorMetric.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/InfluxDBUtils.java b/src/core-metrics/src/main/java/org/apache/kylin/common/util/InfluxDBUtils.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/util/InfluxDBUtils.java rename to src/core-metrics/src/main/java/org/apache/kylin/common/util/InfluxDBUtils.java diff --git a/src/core-common/src/test/java/org/apache/kylin/common/metric/InfluxDBInstanceTest.java b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/InfluxDBInstanceTest.java similarity index 100% rename from src/core-common/src/test/java/org/apache/kylin/common/metric/InfluxDBInstanceTest.java rename to src/core-metrics/src/test/java/org/apache/kylin/common/metric/InfluxDBInstanceTest.java diff --git a/src/core-common/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java similarity index 100% rename from src/core-common/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java rename to src/core-metrics/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java diff --git a/src/core-common/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java similarity index 89% rename from src/core-common/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java rename to src/core-metrics/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java index 64a1ba1a58..5786d4ed12 100644 --- a/src/core-common/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java +++ b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java @@ -42,18 +42,18 @@ import org.springframework.security.core.context.SecurityContextHolder; import io.kyligence.kap.shaded.influxdb.org.influxdb.InfluxDB; @RunWith(PowerMockRunner.class) -@PowerMockIgnore({"javax.net.ssl.*", "javax.management.*", "org.apache.hadoop.*", "javax.security.*", "javax.crypto.*", "javax.script.*"}) -@PrepareForTest({InfluxDBInstance.class, InfluxDBUtils.class, UserGroupInformation.class}) -public class MetricsControllerTest extends NLocalFileMetadataTestCase{ +@PowerMockIgnore({ "javax.net.ssl.*", "javax.management.*", "org.apache.hadoop.*", "javax.security.*", "javax.crypto.*", + "javax.script.*" }) +@PrepareForTest({ InfluxDBInstance.class, InfluxDBUtils.class, UserGroupInformation.class }) +public class MetricsControllerTest extends NLocalFileMetadataTestCase { public final static String ROLE_ADMIN = "ROLE_ADMIN"; private final Authentication authentication = new TestingAuthenticationToken("ADMIN", "ADMIN", ROLE_ADMIN); - @Before public void setUp() throws Exception { PowerMockito.mockStatic(UserGroupInformation.class); PowerMockito.mockStatic(InfluxDBUtils.class); - + UserGroupInformation userGroupInformation = Mockito.mock(UserGroupInformation.class); PowerMockito.when(UserGroupInformation.getCurrentUser()).thenAnswer(invocation -> userGroupInformation); overwriteSystemProp("HADOOP_USER_NAME", "root"); @@ -66,14 +66,14 @@ public class MetricsControllerTest extends NLocalFileMetadataTestCase{ cleanupTestMetadata(); } - @Test public void initTest() throws Exception { KapConfig config = KapConfig.wrap(KylinConfig.getInstanceFromEnv()); InfluxDB influxDB = Mockito.mock(InfluxDB.class); - PowerMockito.doAnswer(invocationOnMock -> influxDB).when(InfluxDBUtils.class, "getInfluxDBInstance", Mockito.anyString(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyBoolean()); + PowerMockito.doAnswer(invocationOnMock -> influxDB).when(InfluxDBUtils.class, "getInfluxDBInstance", + Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean(), + Mockito.anyBoolean()); Mockito.when(influxDB.databaseExists(Mockito.anyString())).thenReturn(true); MetricsController.init(config); diff --git a/src/core-common/src/test/java/org/apache/kylin/common/metric/MonitorDaoTest.java b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/MonitorDaoTest.java similarity index 100% rename from src/core-common/src/test/java/org/apache/kylin/common/metric/MonitorDaoTest.java rename to src/core-metrics/src/test/java/org/apache/kylin/common/metric/MonitorDaoTest.java diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java index a9eadeff1d..b5a2cec688 100644 --- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java +++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java @@ -68,7 +68,6 @@ import org.apache.kylin.common.msg.MsgPicker; import org.apache.kylin.common.persistence.transaction.TransactionException; import org.apache.kylin.common.util.DateFormat; import org.apache.kylin.common.util.JsonUtil; -import org.apache.kylin.common.util.Unsafe; import org.apache.kylin.job.dao.ExecutablePO; import org.apache.kylin.metadata.project.NProjectManager; import org.apache.kylin.metadata.project.ProjectInstance; @@ -151,7 +150,7 @@ public class BaseController { if (kylinException != null) { cause = kylinException; } - return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), cause); + return new ErrorResponse(req.getRequestURL().toString(), cause); } @ResponseStatus(HttpStatus.FORBIDDEN) @@ -159,7 +158,7 @@ public class BaseController { @ResponseBody ErrorResponse handleForbidden(HttpServletRequest req, Exception ex) { getLogger().error("", ex); - return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex); + return new ErrorResponse(req.getRequestURL().toString(), ex); } @ResponseStatus(HttpStatus.NOT_FOUND) @@ -167,7 +166,7 @@ public class BaseController { @ResponseBody ErrorResponse handleNotFound(HttpServletRequest req, Exception ex) { getLogger().error("", ex); - return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex); + return new ErrorResponse(req.getRequestURL().toString(), ex); } @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) @@ -191,7 +190,7 @@ public class BaseController { ErrorResponse handleAccessDenied(HttpServletRequest req, Throwable ex) { getLogger().error("", ex); KylinException e = new KylinException(ACCESS_DENIED, MsgPicker.getMsg().getAccessDeny()); - return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), e); + return new ErrorResponse(req.getRequestURL().toString(), e); } @ResponseStatus(HttpStatus.BAD_REQUEST) @@ -200,7 +199,7 @@ public class BaseController { ErrorResponse handleInvalidRequestParam(HttpServletRequest req, Throwable ex) { KylinException e = new KylinException(INVALID_PARAMETER, ex); getLogger().error("", e); - return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), e); + return new ErrorResponse(req.getRequestURL().toString(), e); } @ResponseStatus(HttpStatus.BAD_REQUEST) @@ -210,7 +209,7 @@ public class BaseController { String parameterName = ex.getParameterName(); KylinException e = new KylinException(REQUEST_PARAMETER_EMPTY_OR_VALUE_EMPTY, parameterName); getLogger().error("", e); - return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), e); + return new ErrorResponse(req.getRequestURL().toString(), e); } @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) @@ -219,14 +218,14 @@ public class BaseController { ErrorResponse handleErrorCode(HttpServletRequest req, Throwable ex) { getLogger().error("", ex); KylinException cause = (KylinException) ex; - return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), cause); + return new ErrorResponse(req.getRequestURL().toString(), cause); } @ResponseStatus(HttpStatus.BAD_REQUEST) @ExceptionHandler(MethodArgumentNotValidException.class) @ResponseBody ErrorResponse handleInvalidArgument(HttpServletRequest request, MethodArgumentNotValidException ex) { - val response = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(request), ex); + val response = new ErrorResponse(request.getRequestURL().toString(), ex); val target = ex.getBindingResult().getTarget(); if (target instanceof Validation) { response.setMsg(((Validation) target).getErrorMessage(ex.getBindingResult().getFieldErrors())); @@ -244,7 +243,7 @@ public class BaseController { ErrorResponse handleUnauthorized(HttpServletRequest req, Throwable ex) { KylinException e = new KylinException(USER_UNAUTHORIZED, ex); getLogger().error("", e); - return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex); + return new ErrorResponse(req.getRequestURL().toString(), ex); } protected void checkRequiredArg(String fieldName, Object fieldValue) { diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/filter/SegmentsRequestFilter.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/filter/SegmentsRequestFilter.java index 42477bede6..63256b746b 100644 --- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/filter/SegmentsRequestFilter.java +++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/filter/SegmentsRequestFilter.java @@ -42,7 +42,6 @@ import org.apache.kylin.common.exception.ErrorCode; import org.apache.kylin.common.exception.KylinException; import org.apache.kylin.common.msg.MsgPicker; import org.apache.kylin.common.util.JsonUtil; -import org.apache.kylin.common.util.Unsafe; import org.apache.kylin.rest.response.ErrorResponse; import org.springframework.core.annotation.Order; import org.springframework.http.HttpHeaders; @@ -84,7 +83,7 @@ public class SegmentsRequestFilter implements Filter { MsgPicker.setMsg(httpServletRequest.getHeader(HttpHeaders.ACCEPT_LANGUAGE)); ErrorCode.setMsg(httpServletRequest.getHeader(HttpHeaders.ACCEPT_LANGUAGE)); - ErrorResponse errorResponse = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(httpServletRequest), + ErrorResponse errorResponse = new ErrorResponse(httpServletRequest.getRequestURL().toString(), e); byte[] responseBody = JsonUtil.writeValueAsBytes(errorResponse); diff --git a/src/datasource-sdk/pom.xml b/src/datasource-sdk/pom.xml index 27bcc81db4..992fac1baa 100644 --- a/src/datasource-sdk/pom.xml +++ b/src/datasource-sdk/pom.xml @@ -14,11 +14,6 @@ <name>Kylin - Datasource SDK</name> <dependencies> - <!-- provided --> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - </dependency> <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-metadata</artifactId> @@ -28,10 +23,6 @@ <artifactId>jackson-dataformat-xml</artifactId> </dependency> - <dependency> - <groupId>org.apache.calcite</groupId> - <artifactId>calcite-linq4j</artifactId> - </dependency> <!--Test and Environment--> <dependency> <groupId>org.apache.kylin</groupId> @@ -40,14 +31,8 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-metadata</artifactId> - <type>test-jar</type> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-source-hive</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> <scope>test</scope> </dependency> <dependency> @@ -56,13 +41,13 @@ <scope>test</scope> </dependency> <dependency> - <groupId>com.h2database</groupId> - <artifactId>h2</artifactId> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> <scope>test</scope> </dependency> <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> + <groupId>com.h2database</groupId> + <artifactId>h2</artifactId> <scope>test</scope> </dependency> </dependencies> diff --git a/src/datasource-service/pom.xml b/src/datasource-service/pom.xml index 69e2d8427b..acd5eacfa9 100644 --- a/src/datasource-service/pom.xml +++ b/src/datasource-service/pom.xml @@ -80,6 +80,10 @@ <artifactId>spark-sql_2.12</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>distributed-lock-ext</artifactId> + </dependency> <!-- test --> <dependency> diff --git a/src/core-job/pom.xml b/src/distributed-lock-ext/pom.xml similarity index 51% copy from src/core-job/pom.xml copy to src/distributed-lock-ext/pom.xml index bbdbb0cf2f..3df312fd32 100644 --- a/src/core-job/pom.xml +++ b/src/distributed-lock-ext/pom.xml @@ -1,91 +1,79 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> +<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <artifactId>kylin-core-job</artifactId> - <packaging>jar</packaging> - <name>Kylin - Core Job</name> - <url>http://kylin.apache.org</url> - <description>Kylin Core Job</description> - <parent> - <groupId>org.apache.kylin</groupId> <artifactId>kylin</artifactId> + <groupId>org.apache.kylin</groupId> <version>5.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>distributed-lock-ext</artifactId> + <name>Kylin - Distributed Lock Extension</name> <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <maven.compiler.source>8</maven.compiler.source> + <maven.compiler.target>8</maven.compiler.target> </properties> <dependencies> - <!-- provided --> <dependency> <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-metadata</artifactId> + <artifactId>kylin-core-common</artifactId> </dependency> <dependency> <groupId>io.kyligence.ke</groupId> - <artifactId>kap-external-curator-test</artifactId> + <artifactId>kap-external-curator</artifactId> </dependency> <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-common</artifactId> - <type>test-jar</type> - <scope>test</scope> + <groupId>org.springframework.integration</groupId> + <artifactId>spring-integration-jdbc</artifactId> </dependency> <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-metadata</artifactId> + <groupId>org.mybatis</groupId> + <artifactId>mybatis</artifactId> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> + <groupId>org.mybatis.dynamic-sql</groupId> + <artifactId>mybatis-dynamic-sql</artifactId> </dependency> <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> + <groupId>org.projectlombok</groupId> + <artifactId>lombok</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-dbcp2</artifactId> <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>org.junit.vintage</groupId> - <artifactId>junit-vintage-engine</artifactId> - </exclusion> - </exclusions> </dependency> - <!--Env & Test--> <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter-api</artifactId> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-core-common</artifactId> + <type>test-jar</type> <scope>test</scope> </dependency> <dependency> - <groupId>org.junit.vintage</groupId> - <artifactId>junit-vintage-engine</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.h2database</groupId> + <artifactId>h2</artifactId> <scope>test</scope> </dependency> <dependency> - <groupId>org.assertj</groupId> - <artifactId>assertj-core</artifactId> + <groupId>org.springframework</groupId> + <artifactId>spring-test</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-test</artifactId> <scope>test</scope> </dependency> <dependency> @@ -94,20 +82,21 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> <scope>test</scope> </dependency> <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-inline</artifactId> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> <scope>test</scope> </dependency> <dependency> - <groupId>com.h2database</groupId> - <artifactId>h2</artifactId> + <groupId>org.junit-pioneer</groupId> + <artifactId>junit-pioneer</artifactId> <scope>test</scope> </dependency> </dependencies> -</project> + +</project> \ No newline at end of file diff --git a/src/core-common/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLock.java b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLock.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLock.java rename to src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLock.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactory.java b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactory.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactory.java rename to src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactory.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactory.java b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactory.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactory.java rename to src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactory.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java similarity index 99% rename from src/core-common/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java rename to src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java index be93e5da19..edd7eddb03 100644 --- a/src/core-common/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java +++ b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java @@ -18,17 +18,6 @@ package org.apache.kylin.common.lock.jdbc; -import lombok.extern.slf4j.Slf4j; -import lombok.val; -import org.apache.commons.dbcp2.BasicDataSource; -import org.apache.ibatis.jdbc.ScriptRunner; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.StorageURL; -import org.apache.kylin.common.logging.LogOutputStream; -import org.apache.kylin.common.persistence.metadata.JdbcDataSource; -import org.apache.kylin.common.persistence.metadata.jdbc.JdbcUtil; - -import javax.sql.DataSource; import java.io.ByteArrayInputStream; import java.io.InputStreamReader; import java.io.OutputStreamWriter; @@ -38,6 +27,19 @@ import java.sql.Connection; import java.util.Locale; import java.util.Properties; +import javax.sql.DataSource; + +import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.ibatis.jdbc.ScriptRunner; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.StorageURL; +import org.apache.kylin.common.logging.LogOutputStream; +import org.apache.kylin.common.persistence.metadata.JdbcDataSource; +import org.apache.kylin.common.persistence.metadata.jdbc.JdbcUtil; + +import lombok.val; +import lombok.extern.slf4j.Slf4j; + @Slf4j public class JdbcDistributedLockUtil { private static final Charset DEFAULT_CHARSET = Charset.defaultCharset(); diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/ZKUtil.java b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/util/ZKUtil.java similarity index 96% rename from src/core-common/src/main/java/org/apache/kylin/common/util/ZKUtil.java rename to src/distributed-lock-ext/src/main/java/org/apache/kylin/common/util/ZKUtil.java index dbbeb9277a..6ab6e860ab 100644 --- a/src/core-common/src/main/java/org/apache/kylin/common/util/ZKUtil.java +++ b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/util/ZKUtil.java @@ -18,9 +18,6 @@ package org.apache.kylin.common.util; -import static org.apache.kylin.common.ZookeeperConfig.geZKClientConnectionTimeoutMs; -import static org.apache.kylin.common.ZookeeperConfig.geZKClientSessionTimeoutMs; - import java.io.File; import java.io.IOException; import java.util.concurrent.Callable; @@ -28,6 +25,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.ZookeeperConfig; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -177,8 +175,8 @@ public class ZKUtil { if (!Strings.isNullOrEmpty(zkRoot)) { zkString += zkRoot; } - int sessionTimeout = geZKClientSessionTimeoutMs(); - int connectionTimeout = geZKClientConnectionTimeoutMs(); + int sessionTimeout = ZookeeperConfig.geZKClientSessionTimeoutMs(); + int connectionTimeout = ZookeeperConfig.geZKClientConnectionTimeoutMs(); return CuratorFrameworkFactory.newClient(zkString, sessionTimeout, connectionTimeout, retryPolicy); } diff --git a/src/core-common/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java similarity index 98% rename from src/core-common/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java rename to src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java index 2b774bcd41..ea90eed3ba 100644 --- a/src/core-common/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java +++ b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java @@ -18,8 +18,8 @@ package org.apache.kylin.common.lock; -import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; +import static org.apache.kylin.common.util.TestUtils.getTestConfig; +import static org.awaitility.Awaitility.await; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -27,8 +27,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; -import static org.apache.kylin.common.util.TestUtils.getTestConfig; -import static org.awaitility.Awaitility.await; +import org.junit.Assert; + +import lombok.extern.slf4j.Slf4j; @Slf4j public class DistributedLockFactoryTest { @@ -39,7 +40,7 @@ public class DistributedLockFactoryTest { final CountDownLatch tasks = new CountDownLatch(threadNum); - int[] count = new int[] {0}; + int[] count = new int[] { 0 }; for (int i = 0; i < threadNum; i++) { executorService.submit(new DirtyReadTest(key, times, tasks, count)); diff --git a/src/core-common/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java similarity index 73% rename from src/core-common/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java rename to src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java index ea3692a8d0..cec38b22ca 100644 --- a/src/core-common/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java +++ b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java @@ -18,9 +18,6 @@ package org.apache.kylin.common.lock.curator; -import static org.apache.kylin.common.util.TestUtils.getTestConfig; -import static org.awaitility.Awaitility.await; - import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -29,7 +26,9 @@ import java.util.concurrent.locks.Lock; import org.apache.curator.test.TestingServer; import org.apache.kylin.common.lock.DistributedLockFactoryTest; import org.apache.kylin.common.util.RandomUtil; +import org.apache.kylin.common.util.TestUtils; import org.apache.kylin.junit.annotation.MetadataInfo; +import org.awaitility.Awaitility; import org.junit.Assert; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -64,8 +63,9 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest { public void testBasic() throws Exception { String path = "/test/distributed_lock_factory_test/test_basic/" + RandomUtil.randomUUIDStr(); - getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString()); - CuratorDistributedLock lock = (CuratorDistributedLock) getTestConfig().getDistributedLockFactory().getLockForCurrentThread(path); + TestUtils.getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString()); + CuratorDistributedLock lock = (CuratorDistributedLock) TestUtils.getTestConfig().getDistributedLockFactory() + .getLockForCurrentThread(path); Assert.assertFalse(lock.isAcquiredInThisThread()); lock.lock(); @@ -81,14 +81,15 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest { ExecutorService executorService = Executors.newFixedThreadPool(1); - getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString()); - getTestConfig().setProperty("kap.env.zookeeper-max-retries", "1"); - getTestConfig().setProperty("kap.env.zookeeper-base-sleep-time", "1000"); + TestUtils.getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString()); + TestUtils.getTestConfig().setProperty("kap.env.zookeeper-max-retries", "1"); + TestUtils.getTestConfig().setProperty("kap.env.zookeeper-base-sleep-time", "1000"); executorService.submit(() -> { CuratorDistributedLock lock = null; try { - lock = (CuratorDistributedLock)getTestConfig().getDistributedLockFactory().getLockForCurrentThread(path); + lock = (CuratorDistributedLock) TestUtils.getTestConfig().getDistributedLockFactory() + .getLockForCurrentThread(path); } catch (Exception e) { e.printStackTrace(); } @@ -103,7 +104,7 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest { } }); - await().atMost(5, TimeUnit.SECONDS).until(() -> locked); + Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> locked); Assert.assertFalse(isInterrupted); locked = false; @@ -111,15 +112,15 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest { // zk for thread1 lost // thread1 will be interrupted - await().atMost(20, TimeUnit.SECONDS).until(() -> isInterrupted); + Awaitility.await().atMost(20, TimeUnit.SECONDS).until(() -> isInterrupted); Assert.assertFalse(locked); - getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer2.getConnectString()); + TestUtils.getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer2.getConnectString()); executorService.submit(() -> { Lock lock = null; try { - lock = getTestConfig().getDistributedLockFactory().getLockForCurrentThread(path); + lock = TestUtils.getTestConfig().getDistributedLockFactory().getLockForCurrentThread(path); } catch (Exception e) { e.printStackTrace(); } @@ -130,7 +131,7 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest { // thread1 released the lock // thread2 will get the lock - await().atMost(5, TimeUnit.SECONDS).until(() -> locked); + Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> locked); } @Test @@ -141,10 +142,10 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest { ExecutorService executorService = Executors.newFixedThreadPool(1); CuratorDistributedLockFactory lockFactory; CuratorDistributedLock lock1; - getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString()); - getTestConfig().setProperty("kap.env.zookeeper-max-retries", "1"); - getTestConfig().setProperty("kap.env.zookeeper-base-sleep-time", "1000"); - lockFactory = (CuratorDistributedLockFactory) getTestConfig().getDistributedLockFactory(); + TestUtils.getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString()); + TestUtils.getTestConfig().setProperty("kap.env.zookeeper-max-retries", "1"); + TestUtils.getTestConfig().setProperty("kap.env.zookeeper-base-sleep-time", "1000"); + lockFactory = (CuratorDistributedLockFactory) TestUtils.getTestConfig().getDistributedLockFactory(); lock1 = lockFactory.getLockForCurrentThread(path); executorService.submit(() -> { @@ -158,7 +159,7 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest { } }); - await().atMost(5, TimeUnit.SECONDS).until(() -> locked); + Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> locked); Assert.assertFalse(isInterrupted); @@ -169,12 +170,12 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest { // zk for thread1 suspended // thread1 will be interrupted - await().atMost(5, TimeUnit.SECONDS).until(() -> isInterrupted); + Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> isInterrupted); } @Test void testConcurrence() throws Exception { - getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString()); + TestUtils.getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString()); String key = "/test/distributed_lock_factory_test/test_concurrence/" + RandomUtil.randomUUIDStr(); super.testConcurrence(key, 10, 10); } diff --git a/src/core-common/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java similarity index 91% rename from src/core-common/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java rename to src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java index 45e8b533a4..4e5af52c0b 100644 --- a/src/core-common/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java +++ b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java @@ -18,26 +18,25 @@ package org.apache.kylin.common.lock.jdbc; +import static org.apache.kylin.common.util.TestUtils.getTestConfig; + +import java.util.UUID; + import org.apache.kylin.common.lock.DistributedLockFactoryTest; import org.apache.kylin.junit.annotation.MetadataInfo; import org.apache.kylin.junit.annotation.OverwriteProp; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.UUID; - -import static org.apache.kylin.common.util.TestUtils.getTestConfig; - @MetadataInfo(onlyProps = true) class JdbcDistributedLockFactoryTest extends DistributedLockFactoryTest { @BeforeEach public void setup() { - getTestConfig().getDistributedLockFactory().initialize(); + getTestConfig().getDistributedLockFactory().initialize(); } - @OverwriteProp(key = "kylin.metadata.distributed-lock-impl", - value = "org.apache.kylin.common.lock.jdbc.JdbcDistributedLockFactory") + @OverwriteProp(key = "kylin.metadata.distributed-lock-impl", value = "org.apache.kylin.common.lock.jdbc.JdbcDistributedLockFactory") @Test void testConcurrence() throws Exception { getTestConfig().getDistributedLockFactory().initialize(); diff --git a/src/kylin-it/pom.xml b/src/kylin-it/pom.xml index 19a5def3a6..abd2acc15f 100644 --- a/src/kylin-it/pom.xml +++ b/src/kylin-it/pom.xml @@ -11,6 +11,11 @@ <modelVersion>4.0.0</modelVersion> <artifactId>kylin-it</artifactId> + <name>Kylin - Integration Test</name> + + <properties> + <beanutils.version>1.9.4</beanutils.version> + </properties> <dependencies> <!--Springfox Swagger2 v2.9.2, swagger2markup v1.3.3. has specific requirement on guava and commons-beanutils --> diff --git a/src/kylin-it/src/test/java/org/apache/kylin/streaming/StreamingMergeEntryTest.java b/src/kylin-it/src/test/java/org/apache/kylin/streaming/StreamingMergeEntryTest.java index 59acd33a6f..59a6912cac 100644 --- a/src/kylin-it/src/test/java/org/apache/kylin/streaming/StreamingMergeEntryTest.java +++ b/src/kylin-it/src/test/java/org/apache/kylin/streaming/StreamingMergeEntryTest.java @@ -384,6 +384,8 @@ public class StreamingMergeEntryTest extends StreamingTestCase { val entry = Mockito.spy(new StreamingMergeEntry()); val segments = new Segments<NDataSegment>(); + val mgr = NDataflowManager.getInstance(getTestConfig(), PROJECT); + var dataflow = mgr.getDataflow(DATAFLOW_ID); for (int i = 0; i < 3; i++) { val start = LocalDate.parse("2000-01-01").plusMonths(i); val end = start.plusMonths(1); @@ -395,6 +397,7 @@ public class StreamingMergeEntryTest extends StreamingTestCase { seg.setSegmentRange(segRange); seg.setStatus(SegmentStatusEnum.READY); seg.setAdditionalInfo(null); + seg.setDataflow(dataflow); segments.add(seg); } @@ -407,6 +410,8 @@ public class StreamingMergeEntryTest extends StreamingTestCase { val segments = new Segments<NDataSegment>(); val addInfo = new HashMap<String, String>(); + val mgr = NDataflowManager.getInstance(getTestConfig(), PROJECT); + var dataflow = mgr.getDataflow(DATAFLOW_ID); addInfo.put("abc", "2"); for (int i = 0; i < 3; i++) { val start = LocalDate.parse("2000-01-01").plusMonths(i); @@ -419,6 +424,7 @@ public class StreamingMergeEntryTest extends StreamingTestCase { seg.setSegmentRange(segRange); seg.setStatus(SegmentStatusEnum.READY); seg.setAdditionalInfo(addInfo); + seg.setDataflow(dataflow); segments.add(seg); } @@ -431,6 +437,8 @@ public class StreamingMergeEntryTest extends StreamingTestCase { val segments = new Segments<NDataSegment>(); val addInfo = new HashMap<String, String>(); + val mgr = NDataflowManager.getInstance(getTestConfig(), PROJECT); + var dataflow = mgr.getDataflow(DATAFLOW_ID); addInfo.put(StreamingConstants.FILE_LAYER, "2"); for (int i = 0; i < 3; i++) { val start = LocalDate.parse("2000-01-01").plusMonths(i); @@ -443,6 +451,7 @@ public class StreamingMergeEntryTest extends StreamingTestCase { seg.setSegmentRange(segRange); seg.setStatus(SegmentStatusEnum.READY); seg.setAdditionalInfo(addInfo); + seg.setDataflow(dataflow); segments.add(seg); } diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/AddTableNameSqlVisitor.java b/src/modeling-service/src/main/java/org/apache/kylin/common/util/AddTableNameSqlVisitor.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/util/AddTableNameSqlVisitor.java rename to src/modeling-service/src/main/java/org/apache/kylin/common/util/AddTableNameSqlVisitor.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/ModifyTableNameSqlVisitor.java b/src/modeling-service/src/main/java/org/apache/kylin/common/util/ModifyTableNameSqlVisitor.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/util/ModifyTableNameSqlVisitor.java rename to src/modeling-service/src/main/java/org/apache/kylin/common/util/ModifyTableNameSqlVisitor.java diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java index 4decdbb834..f6866f2708 100644 --- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java +++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java @@ -423,7 +423,7 @@ public class ModelSemanticHelper extends BasicService { .forEach(x -> x.changeTableAlias(oldAliasName, newAliasName)); model.getAllMeasures().stream().filter(x -> !x.isTomb()) .forEach(x -> x.changeTableAlias(oldAliasName, newAliasName)); - model.getComputedColumnDescs().forEach(x -> x.changeTableAlias(oldAliasName, newAliasName)); + model.getComputedColumnDescs().forEach(x -> changeTableAlias(x, oldAliasName, newAliasName)); String filterCondition = model.getFilterCondition(); if (StringUtils.isNotEmpty(filterCondition)) { @@ -436,6 +436,13 @@ public class ModelSemanticHelper extends BasicService { } } + private void changeTableAlias(ComputedColumnDesc computedColumnDesc, String oldAlias, String newAlias) { + SqlVisitor<Object> modifyAlias = new ModifyTableNameSqlVisitor(oldAlias, newAlias); + SqlNode sqlNode = CalciteParser.getExpNode(computedColumnDesc.getExpression()); + sqlNode.accept(modifyAlias); + computedColumnDesc.setExpression(sqlNode.toSqlString(HiveSqlDialect.DEFAULT).toString()); + } + private Map<String, String> getAliasTransformMap(NDataModel originModel, NDataModel expectModel) { Map<String, String> matchAlias = Maps.newHashMap(); boolean match = originModel.getJoinsGraph().match(expectModel.getJoinsGraph(), matchAlias); diff --git a/src/query/pom.xml b/src/query/pom.xml index 1479d73d1f..a970b1e41d 100644 --- a/src/query/pom.xml +++ b/src/query/pom.xml @@ -64,6 +64,10 @@ <groupId>org.apache.calcite</groupId> <artifactId>calcite-linq4j</artifactId> </dependency> + <dependency> + <groupId>org.postgresql</groupId> + <artifactId>postgresql</artifactId> + </dependency> <dependency> <groupId>commons-collections</groupId> diff --git a/src/server/pom.xml b/src/server/pom.xml index 755e7c6ce4..d6a413ec2c 100644 --- a/src/server/pom.xml +++ b/src/server/pom.xml @@ -69,6 +69,10 @@ <groupId>org.apache.kylin</groupId> <artifactId>kap-second-storage-ui</artifactId> </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>distributed-lock-ext</artifactId> + </dependency> <!-- https://mvnrepository.com/artifact/org.scala-lang.modules/scala-xml --> <dependency> @@ -177,6 +181,26 @@ <groupId>com.taobao.arthas</groupId> <artifactId>arthas-spring-boot-starter</artifactId> </dependency> + <dependency> + <groupId>io.micrometer</groupId> + <artifactId>micrometer-registry-prometheus</artifactId> + </dependency> + <dependency> + <groupId>org.postgresql</groupId> + <artifactId>postgresql</artifactId> + </dependency> + <dependency> + <groupId>com.lmax</groupId> + <artifactId>disruptor</artifactId> + </dependency> + <dependency> + <groupId>org.latencyutils</groupId> + <artifactId>LatencyUtils</artifactId> + </dependency> + <dependency> + <groupId>org.javassist</groupId> + <artifactId>javassist</artifactId> + </dependency> <dependency> <groupId>org.apache.hadoop</groupId> diff --git a/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java b/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java index 54c07d3fbb..50e7276660 100644 --- a/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java +++ b/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java @@ -53,7 +53,7 @@ import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.transaction.UnitOfWork; import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.common.util.Pair; -import org.apache.kylin.common.util.Unsafe; +import org.apache.kylin.metadata.epoch.EpochManager; import org.apache.kylin.metadata.project.NProjectManager; import org.apache.kylin.metadata.resourcegroup.ResourceGroupManager; import org.apache.kylin.rest.cluster.ClusterManager; @@ -76,7 +76,6 @@ import org.springframework.web.context.request.ServletRequestAttributes; import com.google.common.collect.Sets; -import org.apache.kylin.metadata.epoch.EpochManager; import lombok.val; import lombok.extern.slf4j.Slf4j; @@ -236,7 +235,7 @@ public class QueryNodeFilter implements Filter { responseStatus = HttpServletResponse.SC_INTERNAL_SERVER_ERROR; Message msg = MsgPicker.getMsg(); KylinException exception = getKylinException(project, msg); - ErrorResponse errorResponse = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(servletRequest), + ErrorResponse errorResponse = new ErrorResponse(servletRequest.getRequestURL().toString(), exception); responseBody = JsonUtil.writeValueAsBytes(errorResponse); responseHeaders = new HttpHeaders(); @@ -358,7 +357,7 @@ public class QueryNodeFilter implements Filter { public void writeConnectionErrorResponse(HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws IOException { - ErrorResponse errorResponse = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(servletRequest), + ErrorResponse errorResponse = new ErrorResponse(servletRequest.getRequestURL().toString(), new KylinException(FAILED_CONNECT_CATALOG, MsgPicker.getMsg().getConnectDatabaseError(), false)); byte[] responseBody = JsonUtil.writeValueAsBytes(errorResponse); HttpHeaders responseHeaders = new HttpHeaders(); diff --git a/src/spark-project/engine-build-sdk/pom.xml b/src/spark-project/engine-build-sdk/pom.xml index 0e2b3d6866..da144f3e7d 100644 --- a/src/spark-project/engine-build-sdk/pom.xml +++ b/src/spark-project/engine-build-sdk/pom.xml @@ -29,7 +29,6 @@ <name>Kylin - Engine Build SDK</name> <artifactId>kylin-engine-build-sdk</artifactId> - <dependencies> <dependency> <groupId>org.projectlombok</groupId> diff --git a/src/spark-project/engine-spark/pom.xml b/src/spark-project/engine-spark/pom.xml index b03cffa85b..2b4edf32e3 100644 --- a/src/spark-project/engine-spark/pom.xml +++ b/src/spark-project/engine-spark/pom.xml @@ -62,6 +62,10 @@ <groupId>org.apache.kylin</groupId> <artifactId>kylin-engine-build-sdk</artifactId> </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>distributed-lock-ext</artifactId> + </dependency> <dependency> <groupId>io.kyligence.ke</groupId> diff --git a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java index 145e017504..805e8ae20f 100644 --- a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java +++ b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java @@ -35,6 +35,9 @@ import java.util.Set; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.exception.KylinException; +import org.apache.kylin.common.persistence.transaction.UnitOfWork; +import org.apache.kylin.common.util.NLocalFileMetadataTestCase; +import org.apache.kylin.engine.spark.ExecutableUtils; import org.apache.kylin.job.engine.JobEngineConfig; import org.apache.kylin.job.execution.AbstractExecutable; import org.apache.kylin.job.execution.ExecutableParams; @@ -42,11 +45,6 @@ import org.apache.kylin.job.execution.NExecutableManager; import org.apache.kylin.job.impl.threadpool.NDefaultScheduler; import org.apache.kylin.job.manager.JobManager; import org.apache.kylin.job.model.JobParam; -import org.apache.kylin.metadata.model.SegmentRange; -import org.apache.kylin.metadata.model.SegmentStatusEnum; -import org.apache.kylin.common.persistence.transaction.UnitOfWork; -import org.apache.kylin.common.util.NLocalFileMetadataTestCase; -import org.apache.kylin.engine.spark.ExecutableUtils; import org.apache.kylin.metadata.cube.model.IndexEntity; import org.apache.kylin.metadata.cube.model.IndexPlan; import org.apache.kylin.metadata.cube.model.LayoutEntity; @@ -60,6 +58,8 @@ import org.apache.kylin.metadata.cube.model.NIndexPlanManager; import org.apache.kylin.metadata.cube.model.PartitionStatusEnum; import org.apache.kylin.metadata.job.JobBucket; import org.apache.kylin.metadata.model.NDataModelManager; +import org.apache.kylin.metadata.model.SegmentRange; +import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -353,17 +353,18 @@ public class JobManagerTest extends NLocalFileMetadataTestCase { @Test public void testQuotaLimitReached() { - thrown.expect(KylinException.class); NDefaultScheduler defaultScheduler = NDefaultScheduler.getInstance(PROJECT); - defaultScheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv())); + defaultScheduler.init(new JobEngineConfig(getTestConfig())); defaultScheduler.getContext().setReachQuotaLimit(true); - JobParam param = new JobParam(Sets.newHashSet(), null, null, "ADMIn", Sets.newHashSet(), null); - try { - jobManager.addJob(param); - } finally { - defaultScheduler.forceShutdown(); - defaultScheduler.getContext().setReachQuotaLimit(false); - } + Assert.assertThrows(KylinException.class, () -> { + try { + JobManager.checkStorageQuota(PROJECT); + } finally { + defaultScheduler.forceShutdown(); + defaultScheduler.getContext().setReachQuotaLimit(false); + } + }); + } @Test diff --git a/src/spark-project/spark-common/pom.xml b/src/spark-project/spark-common/pom.xml index ff80077128..9556bd21d1 100644 --- a/src/spark-project/spark-common/pom.xml +++ b/src/spark-project/spark-common/pom.xml @@ -140,6 +140,11 @@ <artifactId>curator-test</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.awaitility</groupId> + <artifactId>awaitility</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-soft-affinity-cache</artifactId> diff --git a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java rename to src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java similarity index 94% rename from src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java rename to src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java index a0ba5f13a1..1c0b3a795e 100644 --- a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java +++ b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java @@ -47,18 +47,6 @@ public class AsyncProfiler { return profiler; } - // for ut test - public static AsyncProfiler utInstance() { - if (profiler == null) { - return new AsyncProfiler(""); - } - return profiler; - } - - private AsyncProfiler(String ignore) { - logger.info("Test arg for ut: {}", ignore); - } - private AsyncProfiler(boolean loadLocalLib) { try { boolean isTestingOnLocalMac = System.getProperty("os.name", "").contains("Mac") diff --git a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java similarity index 100% rename from src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java rename to src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java diff --git a/src/core-metadata/src/main/java/org/apache/kylin/source/dfs/DFSFileTable.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/source/dfs/DFSFileTable.java similarity index 100% rename from src/core-metadata/src/main/java/org/apache/kylin/source/dfs/DFSFileTable.java rename to src/spark-project/spark-common/src/main/java/org/apache/kylin/source/dfs/DFSFileTable.java diff --git a/src/core-metadata/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java similarity index 98% rename from src/core-metadata/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java rename to src/spark-project/spark-common/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java index 0a2d2a2ffb..17f63712ae 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java +++ b/src/spark-project/spark-common/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java @@ -204,7 +204,7 @@ public class DFSFileTableReader implements TableReader { String nextLine() throws IOException; // return null on EOF } - private class SeqRowReader implements RowReader { + private static class SeqRowReader implements RowReader { Reader reader; Writable key; Text value; @@ -230,7 +230,7 @@ public class DFSFileTableReader implements TableReader { } } - private class CsvRowReader implements RowReader { + private static class CsvRowReader implements RowReader { BufferedReader reader; CsvRowReader(FileSystem fs, String path) throws IOException { diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-arm64.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-arm64.so new file mode 100644 index 0000000000..b959823506 Binary files /dev/null and b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-arm64.so differ diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-x64.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-x64.so new file mode 100644 index 0000000000..6d961cec0d Binary files /dev/null and b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-x64.so differ diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-mac.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-mac.so new file mode 100644 index 0000000000..ad45237d5f Binary files /dev/null and b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-mac.so differ diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so new file mode 100755 index 0000000000..4153f52868 Binary files /dev/null and b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so differ diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so new file mode 100755 index 0000000000..c7298c6b82 Binary files /dev/null and b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so differ diff --git a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala similarity index 100% rename from src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala rename to src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala diff --git a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala similarity index 99% rename from src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala rename to src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala index 69bb7c72bb..d618896946 100644 --- a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala +++ b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala @@ -20,7 +20,7 @@ package org.apache.kylin.common.asyncprofiler import org.slf4j.{Logger, LoggerFactory} -/** +/* * this class is not thread safe */ object AsyncProfilerTool { diff --git a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala similarity index 98% rename from src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala rename to src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala index cd3019d528..881992904b 100644 --- a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala +++ b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala @@ -18,7 +18,8 @@ package org.apache.kylin.common.asyncprofiler -/** simple command in string +/* + * simple command in string * messages are in form of {command}{executor-id}:{param} * commands are limited to fixed 3 characters */ @@ -68,4 +69,3 @@ object Message { (getCommand(msg), getId(msg), getParam(msg)) } } - diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java similarity index 100% rename from src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java rename to src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java similarity index 79% rename from src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java rename to src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java index 176d9ad119..c819afc4f5 100644 --- a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java +++ b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java @@ -67,22 +67,4 @@ public class AsyncProfilerTest { AsyncProfiler.getInstance(true)::stop); } - @Test - public void testAsyncProfilerUtInstance() { - AsyncProfiler originInstance = AsyncProfiler.getInstance(true); - AsyncProfiler utInstance = AsyncProfiler.utInstance(); - Assert.assertSame(originInstance, utInstance); - } - - @Test - public void testStopUnload() { - AsyncProfiler asyncProfiler = AsyncProfiler.utInstance(); - Assert.assertFalse(asyncProfiler.isLoaded()); - asyncProfiler.stop(); - } - - @Test - public void testExecuteUnload() throws IOException { - Assert.assertEquals("", AsyncProfiler.utInstance().execute("")); - } } diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java similarity index 100% rename from src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java rename to src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java similarity index 96% rename from src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java rename to src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java index fba1ad54a6..90c4423b69 100644 --- a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java +++ b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java @@ -18,7 +18,6 @@ package org.apache.kylin.common.asyncprofiler; -import static org.awaitility.Awaitility.await; import static org.mockito.Mockito.mock; import java.io.File; @@ -30,6 +29,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import org.awaitility.Awaitility; import org.awaitility.Duration; import org.junit.Assert; import org.junit.Test; @@ -82,7 +82,7 @@ public class AsyncProfilerUtilsTest { ExecutorService cachedThreadPool = Executors.newCachedThreadPool(); Thread t1 = new Thread(() -> { - await().pollDelay(new Duration(1, TimeUnit.SECONDS)).until(() -> true); + Awaitility.await().pollDelay(new Duration(1, TimeUnit.SECONDS)).until(() -> true); for (int i = 0; i < 3; i++) { cachedThreadPool.execute(() -> asyncProfilerUtils.cachedResult.countDown()); } @@ -100,7 +100,7 @@ public class AsyncProfilerUtilsTest { ExecutorService cachedThreadPool = Executors.newCachedThreadPool(); Thread t1 = new Thread(() -> { - await().pollDelay(new Duration(1, TimeUnit.MILLISECONDS)).until(() -> true); + Awaitility.await().pollDelay(new Duration(1, TimeUnit.MILLISECONDS)).until(() -> true); for (int i = 0; i < 3; i++) { cachedThreadPool.execute(() -> asyncProfilerUtils.cachedResult.countDown()); } diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java similarity index 96% rename from src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java rename to src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java index eef0290854..dddaeaae68 100644 --- a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java +++ b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java @@ -37,7 +37,8 @@ public class MessageTest { @Test public void testExecutorMessage() { - String executorMessage = Message.createExecutorMessage(Message.START(), "-0", AsyncProfilerToolTest.DUMP_PARAMS); + String executorMessage = Message.createExecutorMessage(Message.START(), "-0", + AsyncProfilerToolTest.DUMP_PARAMS); Tuple3<String, String, String> tuple = Message.processMessage(executorMessage); Assert.assertEquals("STA-0:flamegraph", executorMessage); Assert.assertEquals(Message.START(), tuple._1()); diff --git a/src/systools/pom.xml b/src/systools/pom.xml index c9026a1dac..f6e3cd1530 100644 --- a/src/systools/pom.xml +++ b/src/systools/pom.xml @@ -189,26 +189,11 @@ <artifactId>mapstruct</artifactId> </dependency> - <!-- Swagger --> - <dependency> - <groupId>io.springfox</groupId> - <artifactId>springfox-boot-starter</artifactId> - </dependency> - <dependency> - <groupId>io.springfox</groupId> - <artifactId>springfox-boot-starter</artifactId> - <version>3.0.0</version> - </dependency> - - <dependency> - <groupId>org.mapstruct</groupId> - <artifactId>mapstruct</artifactId> - </dependency> - <!-- util --> <dependency> <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>redis.clients</groupId> @@ -230,6 +215,11 @@ <artifactId>mockito-core</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.javassist</groupId> + <artifactId>javassist</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.powermock</groupId> <artifactId>powermock-module-junit4</artifactId>