This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 68aa668a4c5d01e91cac6386c17ca03960f85f30 Author: qianhao.zhou <z.qian...@gmail.com> AuthorDate: Thu Dec 8 13:55:13 2022 +0800 KYLIN-5438 remove systools module * refactor engine spark pom refactor sys-tools remove BadRequestException move PasswordDecryptionException remove integration-service module refactor common-service refactor Broadcaster Co-authored-by: qhzhou <qianhao.z...@kyligence.io> --- pom.xml | 6 - src/common-server/pom.xml | 116 ++++++------------- .../rest/config/IntegrationAppInitializer.java | 13 ++- .../rest/config/initialize/BroadcastListener.java | 28 ++++- src/common-service/pom.xml | 18 ++- .../kylin/rest/aspect/InsensitiveNameAspect.java | 0 .../apache/kylin/rest/aspect/RateLimitAspect.java | 0 .../kylin/rest/aspect/SchedulerEnhancer.java | 0 .../kylin/rest/aspect/TransactionAspect.java | 3 +- .../rest/broadcaster/BroadcastEventHandler.java} | 13 +-- .../apache/kylin/rest/broadcaster/Broadcaster.java | 95 ++++++++-------- .../apache/kylin/rest/cluster/ClusterManager.java | 0 .../kylin/rest/cluster/DefaultClusterManager.java | 0 .../kylin/rest/cluster/MockClusterManager.java | 16 ++- .../config/KylinPropertySourceConfiguration.java | 5 +- .../rest/handler/KapNoOpResponseErrorHandler.java | 0 .../apache/kylin/rest/interceptor/KEFilter.java | 7 +- .../apache/kylin/rest/interceptor/V2ApiFilter.java | 5 +- .../kylin/rest/request/OwnerChangeRequest.java | 0 .../apache/kylin/rest/response/AclTCRResponse.java | 11 +- .../org/apache/kylin/rest/response/DataResult.java | 0 .../apache/kylin/rest/response/ErrorResponse.java | 13 ++- .../kylin/rest/response/ServerInfoResponse.java | 0 .../kylin/rest/response/UserGroupResponse.java | 2 + .../kylin/rest/response/UserGroupResponseKI.java | 0 .../kylin/rest/service/AclTCRServiceSupporter.java | 0 .../kylin/rest/service/IUserGroupService.java | 0 .../rest/service/UserAclServiceSupporter.java | 0 .../org/apache/kylin/rest/util/AclEvaluate.java | 0 .../org/apache/kylin/rest/util/SerializeUtil.java | 23 ++-- .../rest/util/TimestampedRollingFileOutputDir.java | 19 +--- .../apache/kylin/rest/util/AclEvaluateTest.java | 0 .../util/TimestampedRollingFileOutputDirTest.java | 0 .../transaction/TransactionException.java | 4 + .../org/apache/kylin/common/util/DateFormat.java | 1 - .../java/org/apache/kylin/common/util/Unsafe.java | 29 ----- .../kylin/job/exception/ExecuteException.java | 0 .../apache/kylin/rest/aspect/EnableRateLimit.java | 0 .../org/apache/kylin/rest/aspect/Transaction.java | 0 .../kylin/rest/aspect/TransactionProjectUnit.java | 0 .../exception/PasswordDecryptionException.java | 0 .../kylin/rest/response/EnvelopeResponse.java | 0 .../org/apache/kylin/rest/util/PagingUtil.java | 7 +- .../kylin/rest/util/SecurityLoggerUtils.java | 0 .../org/apache/kylin/rest/util/SpringContext.java | 2 +- .../apache/kylin/tool/restclient/RestClient.java | 13 +-- .../kylin/common/persistence/UnitOfWorkTest.java | 15 ++- .../apache/kylin/rest/service/PagingUtilTest.java | 0 .../kylin/rest/util/CreateTableFromJson.java | 12 +- .../kylin/rest/service/ModelServiceBuildTest.java | 3 +- src/integration-service/pom.xml | 125 --------------------- .../kylin/rest/broadcaster/BroadcasterTest.java | 20 +--- src/modeling-service/pom.xml | 4 - .../kylin/rest/service/ModelServiceTest.java | 6 +- src/query-server/pom.xml | 5 - src/query-service/pom.xml | 1 - .../apache/kylin/rest/service/MonitorService.java | 2 +- .../kylin/rest/service/MonitorServiceTest.java | 0 .../rest/service/QueryHistoryServiceTest.java | 9 +- src/second-storage/core-ui/pom.xml | 5 - src/server/pom.xml | 4 + .../org/apache/kylin/rest/config/CorsConfig.java | 0 .../org/apache/kylin/rest/config/WebMvcConfig.java | 4 +- src/spark-project/engine-build-sdk/pom.xml | 10 -- src/spark-project/engine-spark/pom.xml | 2 +- src/spark-project/spark-common/pom.xml | 6 +- .../common/logging/AbstractHdfsLogAppender.java | 5 +- src/streaming-sdk/pom.xml | 2 +- src/streaming/pom.xml | 2 +- src/systools/pom.xml | 12 -- .../kylin/rest/exception/BadRequestException.java | 78 ------------- .../rest/interceptor/HeaderMapRequestWrapper.java | 78 ------------- .../kylin/rest/security/ExternalAclProvider.java | 18 +-- 73 files changed, 237 insertions(+), 640 deletions(-) diff --git a/pom.xml b/pom.xml index b03a749eba..24555051fe 100644 --- a/pom.xml +++ b/pom.xml @@ -400,7 +400,6 @@ <module>src/common-service</module> <module>src/job-service</module> <module>src/streaming-service</module> - <module>src/integration-service</module> <module>src/kylin-it</module> <!-- controller --> @@ -580,11 +579,6 @@ <artifactId>kylin-systools</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-integration-service</artifactId> - <version>${project.version}</version> - </dependency> <dependency> <groupId>org.apache.kylin</groupId> <artifactId>distributed-lock-ext</artifactId> diff --git a/src/common-server/pom.xml b/src/common-server/pom.xml index 0e23257e0a..b563fe64ab 100644 --- a/src/common-server/pom.xml +++ b/src/common-server/pom.xml @@ -37,17 +37,16 @@ <dependencies> <dependency> - <groupId>org.hibernate.validator</groupId> - <artifactId>hibernate-validator</artifactId> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-core-common</artifactId> </dependency> - <!--Add dependencies of KAP extensions--> <dependency> <groupId>org.apache.kylin</groupId> - <artifactId>kylin-common-service</artifactId> + <artifactId>kylin-core-metadata</artifactId> </dependency> <dependency> <groupId>org.apache.kylin</groupId> - <artifactId>kylin-job-service</artifactId> + <artifactId>kylin-datasource-service</artifactId> </dependency> <dependency> <groupId>org.apache.kylin</groupId> @@ -55,13 +54,8 @@ </dependency> <dependency> <groupId>org.apache.kylin</groupId> - <artifactId>kylin-integration-service</artifactId> + <artifactId>kylin-query-service</artifactId> </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-datasource-service</artifactId> - </dependency> - <!-- Add spring boot --> <dependency> <groupId>org.springframework.boot</groupId> @@ -105,91 +99,20 @@ </exclusion> </exclusions> </dependency> - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-jdbc</artifactId> - </dependency> - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-log4j2</artifactId> - </dependency> <dependency> <groupId>io.springfox</groupId> <artifactId>springfox-boot-starter</artifactId> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdfs</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-core</artifactId> - </dependency> - <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> <scope>provided</scope> </dependency> - - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - </dependency> - - <dependency> - <groupId>com.netflix.servo</groupId> - <artifactId>servo-core</artifactId> - </dependency> - <dependency> - <groupId>io.reactivex</groupId> - <artifactId>rxjava</artifactId> - </dependency> - - <dependency> - <groupId>com.h2database</groupId> - <artifactId>h2</artifactId> - <scope>test</scope> - </dependency> - - <!-- mysql --> - <dependency> - <groupId>mysql</groupId> - <artifactId>mysql-connector-java</artifactId> - <scope>provided</scope> - </dependency> - <!-- Spring Core --> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-webmvc</artifactId> - </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-jdbc</artifactId> - </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-aop</artifactId> - </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-context-support</artifactId> - </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-test</artifactId> - <scope>test</scope> - </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-autoconfigure</artifactId> - <version>${spring.boot.version}</version> <exclusions> <exclusion> <artifactId>spring-context</artifactId> @@ -202,12 +125,32 @@ </exclusions> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.h2database</groupId> + <artifactId>h2</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-common-service</artifactId> <type>test-jar</type> <scope>test</scope> </dependency> + <dependency> + <groupId>org.awaitility</groupId> + <artifactId>awaitility</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-test</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.junit.vintage</groupId> <artifactId>junit-vintage-engine</artifactId> @@ -229,8 +172,13 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-modeling-service</artifactId> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> <scope>test</scope> </dependency> </dependencies> diff --git a/src/integration-service/src/main/java/org/apache/kylin/rest/config/IntegrationAppInitializer.java b/src/common-server/src/main/java/org/apache/kylin/rest/config/IntegrationAppInitializer.java similarity index 77% rename from src/integration-service/src/main/java/org/apache/kylin/rest/config/IntegrationAppInitializer.java rename to src/common-server/src/main/java/org/apache/kylin/rest/config/IntegrationAppInitializer.java index 87543a095f..22f0b94dca 100644 --- a/src/integration-service/src/main/java/org/apache/kylin/rest/config/IntegrationAppInitializer.java +++ b/src/common-server/src/main/java/org/apache/kylin/rest/config/IntegrationAppInitializer.java @@ -25,11 +25,18 @@ import org.springframework.stereotype.Component; @Component public class IntegrationAppInitializer implements InitializingBean { + @Autowired - private BroadcastListener broadcasterListener; + public IntegrationAppInitializer(BroadcastListener broadcasterListener) { + this.broadcasterListener = broadcasterListener; + } + + private final BroadcastListener broadcasterListener; @Override - public void afterPropertiesSet() throws Exception { - EventBusFactory.getInstance().registerBroadcast(broadcasterListener); + public void afterPropertiesSet() { + if (broadcasterListener != null) { + EventBusFactory.getInstance().registerBroadcast(broadcasterListener); + } } } diff --git a/src/integration-service/src/main/java/org/apache/kylin/rest/config/initialize/BroadcastListener.java b/src/common-server/src/main/java/org/apache/kylin/rest/config/initialize/BroadcastListener.java similarity index 91% rename from src/integration-service/src/main/java/org/apache/kylin/rest/config/initialize/BroadcastListener.java rename to src/common-server/src/main/java/org/apache/kylin/rest/config/initialize/BroadcastListener.java index c21b38d02b..4188ac6d17 100644 --- a/src/integration-service/src/main/java/org/apache/kylin/rest/config/initialize/BroadcastListener.java +++ b/src/common-server/src/main/java/org/apache/kylin/rest/config/initialize/BroadcastListener.java @@ -19,7 +19,9 @@ package org.apache.kylin.rest.config.initialize; import java.io.IOException; -import org.apache.kylin.common.KylinConfig; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + import org.apache.kylin.common.persistence.transaction.AccessBatchGrantEventNotifier; import org.apache.kylin.common.persistence.transaction.AccessGrantEventNotifier; import org.apache.kylin.common.persistence.transaction.AccessRevokeEventNotifier; @@ -34,6 +36,7 @@ import org.apache.kylin.common.persistence.transaction.StopQueryBroadcastEventNo import org.apache.kylin.common.persistence.transaction.UpdateJobStatusEventNotifier; import org.apache.kylin.metadata.epoch.EpochManager; import org.apache.kylin.metadata.model.TableExtDesc; +import org.apache.kylin.rest.broadcaster.BroadcastEventHandler; import org.apache.kylin.rest.broadcaster.Broadcaster; import org.apache.kylin.rest.security.AdminUserSyncEventNotifier; import org.apache.kylin.rest.service.AccessService; @@ -41,7 +44,6 @@ import org.apache.kylin.rest.service.AclTCRService; import org.apache.kylin.rest.service.AuditLogService; import org.apache.kylin.rest.service.JobService; import org.apache.kylin.rest.service.QueryService; -import org.apache.kylin.rest.service.TableExtService; import org.apache.kylin.rest.service.UserAclService; import org.apache.spark.sql.SparderEnv; import org.springframework.beans.factory.annotation.Autowired; @@ -53,7 +55,7 @@ import lombok.extern.slf4j.Slf4j; @Component @Slf4j -public class BroadcastListener { +public class BroadcastListener implements BroadcastEventHandler { @Autowired private AuditLogService auditLogService; @@ -72,13 +74,22 @@ public class BroadcastListener { private UserAclService userAclService; @Autowired - private TableExtService tableExtService; + private JobService jobService; @Autowired - private JobService jobService; + private Broadcaster broadcaster; - private Broadcaster broadcaster = Broadcaster.getInstance(KylinConfig.getInstanceFromEnv(), this); + @PostConstruct + public void init() { + broadcaster.register(this); + broadcaster.start(); + } + @PreDestroy + public void cleanup() { + broadcaster.close(); + broadcaster.unregister(); + } @Subscribe public void onEventReady(BroadcastEventReadyNotifier notifier) { broadcaster.announce(notifier); @@ -121,4 +132,9 @@ public class BroadcastListener { adminUserSyncEventNotifier.isUseEmptyPermission()); } } + + @Override + public void handleLocally(BroadcastEventReadyNotifier notifier) throws IOException { + handle(notifier); + } } diff --git a/src/common-service/pom.xml b/src/common-service/pom.xml index 66c8a53bfd..8a9c91c534 100644 --- a/src/common-service/pom.xml +++ b/src/common-service/pom.xml @@ -50,6 +50,18 @@ <groupId>org.springframework.security</groupId> <artifactId>spring-security-config</artifactId> </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-context-support</artifactId> + </dependency> + <dependency> + <groupId>org.aspectj</groupId> + <artifactId>aspectjrt</artifactId> + </dependency> + <dependency> + <groupId>org.aspectj</groupId> + <artifactId>aspectjweaver</artifactId> + </dependency> <dependency> <groupId>redis.clients</groupId> <artifactId>jedis</artifactId> @@ -127,12 +139,6 @@ </dependency> <!-- spark --> - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-core_2.12</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.junit.vintage</groupId> <artifactId>junit-vintage-engine</artifactId> diff --git a/src/systools/src/main/java/org/apache/kylin/rest/aspect/InsensitiveNameAspect.java b/src/common-service/src/main/java/org/apache/kylin/rest/aspect/InsensitiveNameAspect.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/aspect/InsensitiveNameAspect.java rename to src/common-service/src/main/java/org/apache/kylin/rest/aspect/InsensitiveNameAspect.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/aspect/RateLimitAspect.java b/src/common-service/src/main/java/org/apache/kylin/rest/aspect/RateLimitAspect.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/aspect/RateLimitAspect.java rename to src/common-service/src/main/java/org/apache/kylin/rest/aspect/RateLimitAspect.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/aspect/SchedulerEnhancer.java b/src/common-service/src/main/java/org/apache/kylin/rest/aspect/SchedulerEnhancer.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/aspect/SchedulerEnhancer.java rename to src/common-service/src/main/java/org/apache/kylin/rest/aspect/SchedulerEnhancer.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/aspect/TransactionAspect.java b/src/common-service/src/main/java/org/apache/kylin/rest/aspect/TransactionAspect.java similarity index 97% rename from src/systools/src/main/java/org/apache/kylin/rest/aspect/TransactionAspect.java rename to src/common-service/src/main/java/org/apache/kylin/rest/aspect/TransactionAspect.java index a6f9db75a8..974c9a3a3e 100644 --- a/src/systools/src/main/java/org/apache/kylin/rest/aspect/TransactionAspect.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/aspect/TransactionAspect.java @@ -50,7 +50,6 @@ public class TransactionAspect { @Around("callAt(transaction)") public Object around(ProceedingJoinPoint pjp, Transaction transaction) throws TransactionException { - Object result = null; String unitName = UnitOfWork.GLOBAL_UNIT; if (transaction.project() != -1) { Object unitObject = pjp.getArgs()[transaction.project()]; @@ -74,7 +73,7 @@ public class TransactionAspect { try { return pjp.proceed(); } catch (Throwable throwable) { - throw new RuntimeException(throwable); + throw new TransactionException(throwable); } }).build()); } diff --git a/src/systools/src/main/java/org/apache/kylin/rest/request/OwnerChangeRequest.java b/src/common-service/src/main/java/org/apache/kylin/rest/broadcaster/BroadcastEventHandler.java similarity index 74% copy from src/systools/src/main/java/org/apache/kylin/rest/request/OwnerChangeRequest.java copy to src/common-service/src/main/java/org/apache/kylin/rest/broadcaster/BroadcastEventHandler.java index 704c108619..1aa778c2a9 100644 --- a/src/systools/src/main/java/org/apache/kylin/rest/request/OwnerChangeRequest.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/broadcaster/BroadcastEventHandler.java @@ -15,17 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.kylin.rest.broadcaster; -package org.apache.kylin.rest.request; +import org.apache.kylin.common.persistence.transaction.BroadcastEventReadyNotifier; -import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.IOException; -import lombok.Data; +public interface BroadcastEventHandler { -@Data -public class OwnerChangeRequest { - private String project; + void handleLocally(BroadcastEventReadyNotifier notifier) throws IOException; - @JsonProperty(value = "owner") - private String owner; } diff --git a/src/integration-service/src/main/java/org/apache/kylin/rest/broadcaster/Broadcaster.java b/src/common-service/src/main/java/org/apache/kylin/rest/broadcaster/Broadcaster.java similarity index 78% rename from src/integration-service/src/main/java/org/apache/kylin/rest/broadcaster/Broadcaster.java rename to src/common-service/src/main/java/org/apache/kylin/rest/broadcaster/Broadcaster.java index f10e3eb367..c7b36369c1 100644 --- a/src/integration-service/src/main/java/org/apache/kylin/rest/broadcaster/Broadcaster.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/broadcaster/Broadcaster.java @@ -39,53 +39,54 @@ import java.util.stream.Stream; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.ArrayUtils; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.DaemonThreadFactory; -import org.apache.kylin.common.util.NamedThreadFactory; -import org.apache.kylin.rest.util.SpringContext; import org.apache.kylin.common.persistence.transaction.BroadcastEventReadyNotifier; import org.apache.kylin.common.util.AddressUtil; +import org.apache.kylin.common.util.DaemonThreadFactory; +import org.apache.kylin.common.util.NamedThreadFactory; import org.apache.kylin.rest.cluster.ClusterManager; -import org.apache.kylin.rest.config.initialize.BroadcastListener; import org.apache.kylin.rest.response.ServerInfoResponse; import org.apache.kylin.tool.restclient.RestClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; import com.google.common.collect.Sets; +@Component public class Broadcaster implements Closeable { private static final Logger logger = LoggerFactory.getLogger(Broadcaster.class); - private KylinConfig config; - private ClusterManager clusterManager; - - private ExecutorService eventPollExecutor; - private ExecutorService eventHandlerExecutor; - private BlockingQueue<Runnable> runnableQueue = new LinkedBlockingQueue<>(); - private BlockingQueue<BroadcastEventReadyNotifier> eventQueue = new LinkedBlockingQueue<>(); - - private BroadcastListener localHandler; - private ConcurrentHashMap<String, RestClient> restClientMap = new ConcurrentHashMap<>(); - - public static Broadcaster getInstance(KylinConfig config, BroadcastListener localHandler) { - Broadcaster broadcaster = config.getManager(Broadcaster.class); - broadcaster.localHandler = localHandler; - return broadcaster; + private final ClusterManager clusterManager; + private final ExecutorService eventPollExecutor; + private final ExecutorService eventHandlerExecutor; + private final BlockingQueue<Runnable> runnableQueue; + private final BlockingQueue<BroadcastEventReadyNotifier> eventQueue; + private final ConcurrentHashMap<String, RestClient> restClientMap; + private volatile boolean isRunning; + private volatile BroadcastEventHandler handler; + + @Autowired + public Broadcaster(ClusterManager clusterManager) { + this.clusterManager = clusterManager; + this.runnableQueue = new LinkedBlockingQueue<>(); + this.eventQueue = new LinkedBlockingQueue<>(); + this.restClientMap = new ConcurrentHashMap<>(); + this.eventHandlerExecutor = new ThreadPoolExecutor(10, 10, 60L, TimeUnit.SECONDS, runnableQueue, + new DaemonThreadFactory("BroadcastEvent-handler"), new ThreadPoolExecutor.DiscardPolicy()); + this.eventPollExecutor = Executors.newSingleThreadExecutor(new NamedThreadFactory("BroadcastEvent-poll")); } - // called by reflection - static Broadcaster newInstance(KylinConfig config) { - return new Broadcaster(config); + public void start() { + this.isRunning = true; + this.eventPollExecutor.submit(this::consumeEvent); + } + public void register(BroadcastEventHandler handler) { + this.handler = handler; } - private Broadcaster(final KylinConfig config) { - this.config = config; - this.eventHandlerExecutor = new ThreadPoolExecutor(10, 10, 60L, TimeUnit.SECONDS, runnableQueue, - new DaemonThreadFactory("BroadcastEvent-handler"), new ThreadPoolExecutor.DiscardPolicy()); - - this.eventPollExecutor = Executors.newSingleThreadExecutor(new NamedThreadFactory("BroadcastEvent-poll")); - eventPollExecutor.submit(() -> consumeEvent()); + public void unregister() { + this.handler = null; } public void announce(BroadcastEventReadyNotifier event) { @@ -93,17 +94,20 @@ public class Broadcaster implements Closeable { logger.debug("broadcast event queue has contain this event: {}", event); return; } - eventQueue.offer(event); + if (!eventQueue.offer(event)) { + logger.warn("unable to send broadcast "); + } } public void consumeEvent() { try { - while (true) { + while (isRunning) { BroadcastEventReadyNotifier notifier = eventQueue.take(); handleEvent(notifier); } } catch (InterruptedException e) { logger.error("consume broadcast event fail: ", e); + Thread.currentThread().interrupt(); } } @@ -122,7 +126,9 @@ public class Broadcaster implements Closeable { eventHandlerExecutor.submit(() -> { try { if (identity.equals(node)) { - localHandle(notifier); + if (handler != null) { + handler.handleLocally(notifier); + } } else { remoteHandle(node, notifier); } @@ -140,23 +146,17 @@ public class Broadcaster implements Closeable { runnableQueue.size()); } + } catch (InterruptedException e) { + logger.warn("Thread interrupted"); + Thread.currentThread().interrupt(); } catch (Exception e) { logger.warn("failed to broadcast", e); } } - private void localHandle(BroadcastEventReadyNotifier notifier) throws IOException { - localHandler.handle(notifier); - } - private void remoteHandle(String node, BroadcastEventReadyNotifier notifier) throws IOException { - RestClient client = restClientMap.get(node); - if (client == null) { - client = new RestClient(node); - restClientMap.put(node, client); - } - RestClient finalClient = client; - finalClient.notify(notifier); + restClientMap.computeIfAbsent(node, RestClient::new); + restClientMap.get(node).notify(notifier); } private Set<String> getBroadcastNodes(BroadcastEventReadyNotifier notifier) { @@ -194,9 +194,6 @@ public class Broadcaster implements Closeable { Set<String> serverModeNameSets = Stream.of(serverModeEnums).filter(Objects::nonNull) .map(ServerModeEnum::getName).collect(Collectors.toSet()); - if (clusterManager == null) { - clusterManager = SpringContext.getApplicationContext().getBean(ClusterManager.class); - } final List<ServerInfoResponse> nodes = clusterManager.getServersFromCache(); Set<String> result = Sets.newHashSet(); if (CollectionUtils.isEmpty(nodes)) { @@ -209,7 +206,7 @@ public class Broadcaster implements Closeable { } @Override - public void close() throws IOException { - //do nothing + public void close() { + isRunning = false; } } diff --git a/src/systools/src/main/java/org/apache/kylin/rest/cluster/ClusterManager.java b/src/common-service/src/main/java/org/apache/kylin/rest/cluster/ClusterManager.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/cluster/ClusterManager.java rename to src/common-service/src/main/java/org/apache/kylin/rest/cluster/ClusterManager.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/cluster/DefaultClusterManager.java b/src/common-service/src/main/java/org/apache/kylin/rest/cluster/DefaultClusterManager.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/cluster/DefaultClusterManager.java rename to src/common-service/src/main/java/org/apache/kylin/rest/cluster/DefaultClusterManager.java diff --git a/src/systools/src/test/java/org/apache/kylin/rest/cluster/MockClusterManager.java b/src/common-service/src/main/java/org/apache/kylin/rest/cluster/MockClusterManager.java similarity index 69% rename from src/systools/src/test/java/org/apache/kylin/rest/cluster/MockClusterManager.java rename to src/common-service/src/main/java/org/apache/kylin/rest/cluster/MockClusterManager.java index 7083df7cd8..1df4429562 100644 --- a/src/systools/src/test/java/org/apache/kylin/rest/cluster/MockClusterManager.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/cluster/MockClusterManager.java @@ -25,29 +25,33 @@ import org.apache.kylin.rest.response.ServerInfoResponse; import com.google.common.collect.Lists; public class MockClusterManager implements ClusterManager { + + public static final String DEFAULT_ADDRESS = "127.0.0.1:7070"; + public static final String DEFAULT_ADDRESS_HA = "127.0.0.1:7071"; + @Override public String getLocalServer() { - return "127.0.0.1:7070"; + return DEFAULT_ADDRESS; } @Override public List<ServerInfoResponse> getQueryServers() { - return Lists.newArrayList(new ServerInfoResponse("127.0.0.1:7070", ClusterConstant.QUERY), - new ServerInfoResponse("127.0.0.1:7071", ClusterConstant.QUERY)); + return Lists.newArrayList(new ServerInfoResponse(DEFAULT_ADDRESS, ClusterConstant.QUERY), + new ServerInfoResponse(DEFAULT_ADDRESS_HA, ClusterConstant.QUERY)); } @Override public List<ServerInfoResponse> getServersFromCache() { - return Lists.newArrayList(new ServerInfoResponse("127.0.0.1:7070", ClusterConstant.ALL)); + return Lists.newArrayList(new ServerInfoResponse(DEFAULT_ADDRESS, ClusterConstant.ALL)); } @Override public List<ServerInfoResponse> getJobServers() { - return Lists.newArrayList(new ServerInfoResponse("127.0.0.1:7070", ClusterConstant.ALL)); + return Lists.newArrayList(new ServerInfoResponse(DEFAULT_ADDRESS, ClusterConstant.ALL)); } @Override public List<ServerInfoResponse> getServers() { - return Lists.newArrayList(new ServerInfoResponse("127.0.0.1:7070", ClusterConstant.ALL)); + return Lists.newArrayList(new ServerInfoResponse(DEFAULT_ADDRESS, ClusterConstant.ALL)); } } diff --git a/src/systools/src/main/java/org/apache/kylin/rest/config/KylinPropertySourceConfiguration.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/KylinPropertySourceConfiguration.java similarity index 93% rename from src/systools/src/main/java/org/apache/kylin/rest/config/KylinPropertySourceConfiguration.java rename to src/common-service/src/main/java/org/apache/kylin/rest/config/KylinPropertySourceConfiguration.java index fcfd767e0b..8f2da01468 100644 --- a/src/systools/src/main/java/org/apache/kylin/rest/config/KylinPropertySourceConfiguration.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/KylinPropertySourceConfiguration.java @@ -46,9 +46,8 @@ public class KylinPropertySourceConfiguration implements EnvironmentPostProcesso val kylinConfig = KylinConfig.getInstanceFromEnv(); val storageURL = kylinConfig.getMetadataUrl(); if (storageURL.getScheme().equals("jdbc")) { - JdbcUtil.datasourceParameters(storageURL).forEach((key, value) -> { - kylinConfig.setProperty("spring.datasource." + key, value.toString()); - }); + JdbcUtil.datasourceParameters(storageURL) + .forEach((key, value) -> kylinConfig.setProperty("spring.datasource." + key, value.toString())); } PropertySource<String> source = new PropertySource<String>("kylin") { Properties properties = KylinConfig.getInstanceFromEnv().exportToProperties(); diff --git a/src/systools/src/main/java/org/apache/kylin/rest/handler/KapNoOpResponseErrorHandler.java b/src/common-service/src/main/java/org/apache/kylin/rest/handler/KapNoOpResponseErrorHandler.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/handler/KapNoOpResponseErrorHandler.java rename to src/common-service/src/main/java/org/apache/kylin/rest/handler/KapNoOpResponseErrorHandler.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/interceptor/KEFilter.java b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/KEFilter.java similarity index 92% rename from src/systools/src/main/java/org/apache/kylin/rest/interceptor/KEFilter.java rename to src/common-service/src/main/java/org/apache/kylin/rest/interceptor/KEFilter.java index d09e0b0571..0e238cec4f 100644 --- a/src/systools/src/main/java/org/apache/kylin/rest/interceptor/KEFilter.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/KEFilter.java @@ -44,10 +44,7 @@ public class KEFilter extends OncePerRequestFilter { @Override protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) throws IOException, ServletException { - String lang = "en"; - if (request != null) { - lang = request.getHeader("Accept-Language"); - } + String lang = request.getHeader("Accept-Language"); MsgPicker.setMsg(lang); ErrorCode.setMsg(lang); ExceptionResolve.setLang(lang); @@ -55,7 +52,7 @@ public class KEFilter extends OncePerRequestFilter { ErrorMsg.setMsg(lang); ErrorSuggestion.setMsg(lang); - if (request != null && ("/kylin/api/query".equals(request.getRequestURI()) + if (("/kylin/api/query".equals(request.getRequestURI()) || "/kylin/api/async_query".equals(request.getRequestURI()))) { QueryContext.reset(); // reset it anyway QueryContext.current(); // init query context to set the timer diff --git a/src/systools/src/main/java/org/apache/kylin/rest/interceptor/V2ApiFilter.java b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/V2ApiFilter.java similarity index 96% rename from src/systools/src/main/java/org/apache/kylin/rest/interceptor/V2ApiFilter.java rename to src/common-service/src/main/java/org/apache/kylin/rest/interceptor/V2ApiFilter.java index 65d5191651..03820d0d45 100644 --- a/src/systools/src/main/java/org/apache/kylin/rest/interceptor/V2ApiFilter.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/V2ApiFilter.java @@ -42,6 +42,8 @@ import org.springframework.web.servlet.HandlerExecutionChain; import org.springframework.web.servlet.HandlerMapping; import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping; +import com.google.common.base.Preconditions; + import lombok.extern.slf4j.Slf4j; @Slf4j @@ -71,7 +73,7 @@ public class V2ApiFilter implements Filter { } } } catch (Exception e) { - log.warn("get hander from request uri {} failed", uri, e); + log.warn("get handler from request uri {} failed", uri, e); throw new NotFoundException( String.format(Locale.ROOT, "%s API of version v2 is no longer supported", uri)); } @@ -92,6 +94,7 @@ public class V2ApiFilter implements Filter { return null; } WebApplicationContext appContext = WebApplicationContextUtils.getWebApplicationContext(servletContext); + Preconditions.checkNotNull(appContext); Map<String, HandlerMapping> allRequestMappings = BeanFactoryUtils.beansOfTypeIncludingAncestors(appContext, HandlerMapping.class, true, false); diff --git a/src/systools/src/main/java/org/apache/kylin/rest/request/OwnerChangeRequest.java b/src/common-service/src/main/java/org/apache/kylin/rest/request/OwnerChangeRequest.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/request/OwnerChangeRequest.java rename to src/common-service/src/main/java/org/apache/kylin/rest/request/OwnerChangeRequest.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/response/AclTCRResponse.java b/src/common-service/src/main/java/org/apache/kylin/rest/response/AclTCRResponse.java similarity index 95% rename from src/systools/src/main/java/org/apache/kylin/rest/response/AclTCRResponse.java rename to src/common-service/src/main/java/org/apache/kylin/rest/response/AclTCRResponse.java index bed0bc7354..5a69701462 100644 --- a/src/systools/src/main/java/org/apache/kylin/rest/response/AclTCRResponse.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/response/AclTCRResponse.java @@ -23,12 +23,14 @@ import java.util.Collection; import java.util.List; import java.util.stream.Collectors; +import lombok.NoArgsConstructor; import org.apache.kylin.metadata.acl.DependentColumn; import org.apache.kylin.metadata.acl.SensitiveDataMask; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; import lombok.Data; @Data @@ -143,6 +145,8 @@ public class AclTCRResponse { } @Data + @AllArgsConstructor + @NoArgsConstructor public static class DependentColumnData { @JsonProperty("column_identity") private String columnIdentity; @@ -150,12 +154,5 @@ public class AclTCRResponse { @JsonProperty("values") private String[] values; - public DependentColumnData(String columnIdentity, String[] values) { - this.columnIdentity = columnIdentity; - this.values = values; - } - - public DependentColumnData() { - } } } diff --git a/src/systools/src/main/java/org/apache/kylin/rest/response/DataResult.java b/src/common-service/src/main/java/org/apache/kylin/rest/response/DataResult.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/response/DataResult.java rename to src/common-service/src/main/java/org/apache/kylin/rest/response/DataResult.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/response/ErrorResponse.java b/src/common-service/src/main/java/org/apache/kylin/rest/response/ErrorResponse.java similarity index 93% rename from src/systools/src/main/java/org/apache/kylin/rest/response/ErrorResponse.java rename to src/common-service/src/main/java/org/apache/kylin/rest/response/ErrorResponse.java index 7b96a6bb16..a4ca040cb8 100644 --- a/src/systools/src/main/java/org/apache/kylin/rest/response/ErrorResponse.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/response/ErrorResponse.java @@ -27,21 +27,24 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonParseException; import com.google.common.base.Throwables; +import lombok.Data; + /** * response to client when the return HTTP code is not 200 */ -public class ErrorResponse extends EnvelopeResponse { +@Data +public class ErrorResponse extends EnvelopeResponse<Object> { //stacktrace of the exception - public String stacktrace; + private String stacktrace; //same as EnvelopeResponse.msg, kept for legacy reasons - public String exception; + private String exception; //request URL, kept from legacy codes - public String url; + private String url; - public String suggestion; + private String suggestion; @JsonProperty("error_code") public String errorCode; diff --git a/src/systools/src/main/java/org/apache/kylin/rest/response/ServerInfoResponse.java b/src/common-service/src/main/java/org/apache/kylin/rest/response/ServerInfoResponse.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/response/ServerInfoResponse.java rename to src/common-service/src/main/java/org/apache/kylin/rest/response/ServerInfoResponse.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/response/UserGroupResponse.java b/src/common-service/src/main/java/org/apache/kylin/rest/response/UserGroupResponse.java similarity index 96% rename from src/systools/src/main/java/org/apache/kylin/rest/response/UserGroupResponse.java rename to src/common-service/src/main/java/org/apache/kylin/rest/response/UserGroupResponse.java index 768e098069..064bd30600 100644 --- a/src/systools/src/main/java/org/apache/kylin/rest/response/UserGroupResponse.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/response/UserGroupResponse.java @@ -22,8 +22,10 @@ import java.util.Set; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@NoArgsConstructor public class UserGroupResponse { @JsonProperty("uuid") protected String uuid; diff --git a/src/systools/src/main/java/org/apache/kylin/rest/response/UserGroupResponseKI.java b/src/common-service/src/main/java/org/apache/kylin/rest/response/UserGroupResponseKI.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/response/UserGroupResponseKI.java rename to src/common-service/src/main/java/org/apache/kylin/rest/response/UserGroupResponseKI.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/service/AclTCRServiceSupporter.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/AclTCRServiceSupporter.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/service/AclTCRServiceSupporter.java rename to src/common-service/src/main/java/org/apache/kylin/rest/service/AclTCRServiceSupporter.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/service/IUserGroupService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/IUserGroupService.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/service/IUserGroupService.java rename to src/common-service/src/main/java/org/apache/kylin/rest/service/IUserGroupService.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/service/UserAclServiceSupporter.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/UserAclServiceSupporter.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/service/UserAclServiceSupporter.java rename to src/common-service/src/main/java/org/apache/kylin/rest/service/UserAclServiceSupporter.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/util/AclEvaluate.java b/src/common-service/src/main/java/org/apache/kylin/rest/util/AclEvaluate.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/util/AclEvaluate.java rename to src/common-service/src/main/java/org/apache/kylin/rest/util/AclEvaluate.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/util/SerializeUtil.java b/src/common-service/src/main/java/org/apache/kylin/rest/util/SerializeUtil.java similarity index 69% rename from src/systools/src/main/java/org/apache/kylin/rest/util/SerializeUtil.java rename to src/common-service/src/main/java/org/apache/kylin/rest/util/SerializeUtil.java index 12d4429b03..06f4cb735b 100644 --- a/src/systools/src/main/java/org/apache/kylin/rest/util/SerializeUtil.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/util/SerializeUtil.java @@ -23,29 +23,26 @@ import java.io.ByteArrayOutputStream; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import lombok.experimental.UtilityClass; + +@UtilityClass public class SerializeUtil { public static byte[] serialize(Object object) { - ObjectOutputStream oos = null; - ByteArrayOutputStream baos = null; - try { - baos = new ByteArrayOutputStream(); - oos = new ObjectOutputStream(baos); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ObjectOutputStream oos = new ObjectOutputStream(baos)) { oos.writeObject(object); - byte[] bytes = baos.toByteArray(); - return bytes; + return baos.toByteArray(); } catch (Exception e) { - throw new RuntimeException("serialize failed", e); + throw new IllegalStateException("serialize failed", e); } } public static Object deserialize(byte[] bytes) { - ByteArrayInputStream bais = null; - try { - bais = new ByteArrayInputStream(bytes); - ObjectInputStream ois = new ObjectInputStream(bais); + + try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes))) { return ois.readObject(); } catch (Exception e) { - throw new RuntimeException("deserialize failed", e); + throw new IllegalStateException("deserialize failed", e); } } } \ No newline at end of file diff --git a/src/systools/src/main/java/org/apache/kylin/rest/util/TimestampedRollingFileOutputDir.java b/src/common-service/src/main/java/org/apache/kylin/rest/util/TimestampedRollingFileOutputDir.java similarity index 85% rename from src/systools/src/main/java/org/apache/kylin/rest/util/TimestampedRollingFileOutputDir.java rename to src/common-service/src/main/java/org/apache/kylin/rest/util/TimestampedRollingFileOutputDir.java index 683921e51f..e3f0eaed11 100644 --- a/src/systools/src/main/java/org/apache/kylin/rest/util/TimestampedRollingFileOutputDir.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/util/TimestampedRollingFileOutputDir.java @@ -45,10 +45,7 @@ public class TimestampedRollingFileOutputDir { public File newOutputFile() throws IOException { File[] files = outputDir.listFiles(fileFilter()); - if (files == null) { - throw new RuntimeException("Invalid output directory " + outputDir); - } - + Preconditions.checkNotNull(files, "Invalid output directory " + outputDir); logger.debug("found {} output files under output dir", files.length); if (files.length > 0) { Arrays.sort(files, fileComparatorByAge()); @@ -68,21 +65,11 @@ public class TimestampedRollingFileOutputDir { } protected FileFilter fileFilter() { - return new FileFilter() { - @Override - public boolean accept(File pathname) { - return pathname.getName().startsWith(fileNamePrefix); - } - }; + return pathname -> pathname.getName().startsWith(fileNamePrefix); } protected Comparator<File> fileComparatorByAge() { - return new Comparator<File>() { - @Override - public int compare(File file1, File file2) { - return file2.getName().compareTo(file1.getName()); - } - }; + return (file1, file2) -> file2.getName().compareTo(file1.getName()); } protected void removeOldFiles(File[] files) { diff --git a/src/systools/src/test/java/org/apache/kylin/rest/util/AclEvaluateTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/util/AclEvaluateTest.java similarity index 100% rename from src/systools/src/test/java/org/apache/kylin/rest/util/AclEvaluateTest.java rename to src/common-service/src/test/java/org/apache/kylin/rest/util/AclEvaluateTest.java diff --git a/src/systools/src/test/java/org/apache/kylin/rest/util/TimestampedRollingFileOutputDirTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/util/TimestampedRollingFileOutputDirTest.java similarity index 100% rename from src/systools/src/test/java/org/apache/kylin/rest/util/TimestampedRollingFileOutputDirTest.java rename to src/common-service/src/test/java/org/apache/kylin/rest/util/TimestampedRollingFileOutputDirTest.java diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionException.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionException.java index 173c84a09d..484db6f1f4 100644 --- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionException.java +++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionException.java @@ -26,4 +26,8 @@ public class TransactionException extends RuntimeException { public TransactionException(String message) { super(message); } + + public TransactionException(Throwable throwable) { + super(throwable); + } } diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/src/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java index fd8dd243fa..f94cf91213 100644 --- a/src/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java +++ b/src/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java @@ -163,7 +163,6 @@ public class DateFormat { return getDateFormat(pattern, timeZone).format(new Date(millis)); } - @VisibleForTesting public static String formatToTimeWithoutMilliStr(long millis) { return formatToTimeStr(millis, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS); } diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java b/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java index cfd276f15d..2fbf08fce1 100644 --- a/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java +++ b/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java @@ -18,7 +18,6 @@ package org.apache.kylin.common.util; -import java.io.StringWriter; import java.lang.reflect.AccessibleObject; import java.text.MessageFormat; import java.util.Locale; @@ -27,8 +26,6 @@ import java.util.Map; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.SystemPropertiesCache; -import lombok.extern.slf4j.Slf4j; - /** * Contains methods that call JDK methods that the * <a href="https://github.com/policeman-tools/forbidden-apis">forbidden @@ -37,7 +34,6 @@ import lombok.extern.slf4j.Slf4j; * <p>This class is excluded from the check, so methods called via this class * will not fail the build. */ -@Slf4j public class Unsafe { private Unsafe() { @@ -48,31 +44,6 @@ public class Unsafe { System.exit(status); } - /** Calls {@link Object#notifyAll()}. */ - public static void notifyAll(Object o) { - o.notifyAll(); - } - - /** Calls {@link Object#notify()}. */ - public static void notify(Object o) { - o.notify(); - } - - /** Calls {@link Object#wait()}. */ - public static void wait(Object o) throws InterruptedException { - o.wait(); - } - - public static void wait(Object o, long ms) throws InterruptedException { - o.wait(ms); - } - - /** Clears the contents of a {@link StringWriter}. */ - public static void clear(StringWriter sw) { - // Included in this class because StringBuffer is banned. - sw.getBuffer().setLength(0); - } - /** For {@link MessageFormat#format(String, Object...)} cannot set locale*/ public static String format(Locale locale, String pattern, Object... arguments) { MessageFormat temp = new MessageFormat(pattern, locale); diff --git a/src/core-job/src/main/java/org/apache/kylin/job/exception/ExecuteException.java b/src/core-common/src/main/java/org/apache/kylin/job/exception/ExecuteException.java similarity index 100% rename from src/core-job/src/main/java/org/apache/kylin/job/exception/ExecuteException.java rename to src/core-common/src/main/java/org/apache/kylin/job/exception/ExecuteException.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/aspect/EnableRateLimit.java b/src/core-common/src/main/java/org/apache/kylin/rest/aspect/EnableRateLimit.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/aspect/EnableRateLimit.java rename to src/core-common/src/main/java/org/apache/kylin/rest/aspect/EnableRateLimit.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/aspect/Transaction.java b/src/core-common/src/main/java/org/apache/kylin/rest/aspect/Transaction.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/aspect/Transaction.java rename to src/core-common/src/main/java/org/apache/kylin/rest/aspect/Transaction.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/aspect/TransactionProjectUnit.java b/src/core-common/src/main/java/org/apache/kylin/rest/aspect/TransactionProjectUnit.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/aspect/TransactionProjectUnit.java rename to src/core-common/src/main/java/org/apache/kylin/rest/aspect/TransactionProjectUnit.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/exception/PasswordDecryptionException.java b/src/core-common/src/main/java/org/apache/kylin/rest/exception/PasswordDecryptionException.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/exception/PasswordDecryptionException.java rename to src/core-common/src/main/java/org/apache/kylin/rest/exception/PasswordDecryptionException.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java b/src/core-common/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java rename to src/core-common/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/util/PagingUtil.java b/src/core-common/src/main/java/org/apache/kylin/rest/util/PagingUtil.java similarity index 95% rename from src/systools/src/main/java/org/apache/kylin/rest/util/PagingUtil.java rename to src/core-common/src/main/java/org/apache/kylin/rest/util/PagingUtil.java index 43c9d99267..6ae7c40c25 100644 --- a/src/systools/src/main/java/org/apache/kylin/rest/util/PagingUtil.java +++ b/src/core-common/src/main/java/org/apache/kylin/rest/util/PagingUtil.java @@ -27,9 +27,12 @@ import org.apache.commons.lang.StringUtils; public class PagingUtil { + private PagingUtil() {} + public static <T> List<T> cutPage(List<T> full, int pageOffset, int pageSize) { - if (full == null) - return null; + if (full == null) { + return Collections.emptyList(); + } int begin = pageOffset * pageSize; int end = begin + pageSize; diff --git a/src/systools/src/main/java/org/apache/kylin/rest/util/SecurityLoggerUtils.java b/src/core-common/src/main/java/org/apache/kylin/rest/util/SecurityLoggerUtils.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/util/SecurityLoggerUtils.java rename to src/core-common/src/main/java/org/apache/kylin/rest/util/SecurityLoggerUtils.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/util/SpringContext.java b/src/core-common/src/main/java/org/apache/kylin/rest/util/SpringContext.java similarity index 96% rename from src/systools/src/main/java/org/apache/kylin/rest/util/SpringContext.java rename to src/core-common/src/main/java/org/apache/kylin/rest/util/SpringContext.java index c8909cec07..507280086e 100644 --- a/src/systools/src/main/java/org/apache/kylin/rest/util/SpringContext.java +++ b/src/core-common/src/main/java/org/apache/kylin/rest/util/SpringContext.java @@ -35,7 +35,7 @@ public class SpringContext implements ApplicationContextAware { @Override public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - this.applicationContext = applicationContext; + SpringContext.applicationContext = applicationContext; } public static Object getBean(String name) { diff --git a/src/systools/src/main/java/org/apache/kylin/tool/restclient/RestClient.java b/src/core-common/src/main/java/org/apache/kylin/tool/restclient/RestClient.java similarity index 97% rename from src/systools/src/main/java/org/apache/kylin/tool/restclient/RestClient.java rename to src/core-common/src/main/java/org/apache/kylin/tool/restclient/RestClient.java index de7b647d9b..0ea62b4e92 100644 --- a/src/systools/src/main/java/org/apache/kylin/tool/restclient/RestClient.java +++ b/src/core-common/src/main/java/org/apache/kylin/tool/restclient/RestClient.java @@ -65,18 +65,17 @@ public class RestClient { private static final Logger logger = LoggerFactory.getLogger(RestClient.class); - protected static Pattern fullRestPattern = Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?"); + private static final Pattern FULL_REST_PATTERN = Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?"); private static final int HTTP_CONNECTION_TIMEOUT_MS = 30000; private static final int HTTP_SOCKET_TIMEOUT_MS = 120000; - public static final String SCHEME_HTTP = "http://"; + private static final String SCHEME_HTTP = "http://"; private static final String ROUTED = "routed"; - public static final String KYLIN_API_PATH = "/kylin/api"; + private static final String KYLIN_API_PATH = "/kylin/api"; public static boolean matchFullRestPattern(String uri) { - Matcher m = fullRestPattern.matcher(uri); - return m.matches(); + return FULL_REST_PATTERN.matcher(uri).matches(); } // ============================================================================ @@ -92,9 +91,9 @@ public class RestClient { * @param uri "user:pwd@host:port" */ public RestClient(String uri) { - Matcher m = fullRestPattern.matcher(uri); + Matcher m = FULL_REST_PATTERN.matcher(uri); if (!m.matches()) - throw new IllegalArgumentException("URI: " + uri + " -- does not match pattern " + fullRestPattern); + throw new IllegalArgumentException("URI: " + uri + " -- does not match pattern " + FULL_REST_PATTERN); String mUser = m.group(1); String mPwd = m.group(2); diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/UnitOfWorkTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/UnitOfWorkTest.java index b9a0cdcca5..db77cce00f 100644 --- a/src/core-common/src/test/java/org/apache/kylin/common/persistence/UnitOfWorkTest.java +++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/UnitOfWorkTest.java @@ -25,7 +25,6 @@ import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.transaction.TransactionException; import org.apache.kylin.common.persistence.transaction.UnitOfWork; import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams; -import org.apache.kylin.common.util.Unsafe; import org.apache.kylin.junit.annotation.MetadataInfo; import org.apache.kylin.junit.annotation.OverwriteProp; import org.junit.Assert; @@ -173,12 +172,12 @@ public class UnitOfWorkTest { UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.builder().unitName(UnitOfWork.GLOBAL_UNIT) .readonly(true).maxRetry(1).processor(() -> { synchronized (condition) { - Unsafe.notify(condition); + condition.notify(); } boolean interrupted = false; while (!interrupted && !Thread.interrupted() && !stop.get()) { synchronized (condition) { - Unsafe.notify(condition); + condition.notify(); } try { Thread.sleep(1000); @@ -192,7 +191,7 @@ public class UnitOfWorkTest { readLockHelder.start(); synchronized (condition) { try { - Unsafe.wait(condition); + condition.wait(); } catch (InterruptedException e) { e.printStackTrace(); } @@ -246,12 +245,12 @@ public class UnitOfWorkTest { resourceStoreInTransaction.checkAndPutResource("/_global/path/to/res1", ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L); synchronized (condition) { - Unsafe.notify(condition); + condition.notify(); } boolean interrupted = false; while (!interrupted && !Thread.interrupted() && !stop.get()) { synchronized (condition) { - Unsafe.notify(condition); + condition.notify(); } try { Thread.sleep(1000); @@ -260,7 +259,7 @@ public class UnitOfWorkTest { } } synchronized (condition) { - Unsafe.notify(condition); + condition.notify(); } return 0; }).build()); @@ -268,7 +267,7 @@ public class UnitOfWorkTest { writeLockHelder.start(); synchronized (condition) { try { - Unsafe.wait(condition); + condition.wait(); } catch (InterruptedException e) { e.printStackTrace(); } diff --git a/src/systools/src/test/java/org/apache/kylin/rest/service/PagingUtilTest.java b/src/core-common/src/test/java/org/apache/kylin/rest/service/PagingUtilTest.java similarity index 100% rename from src/systools/src/test/java/org/apache/kylin/rest/service/PagingUtilTest.java rename to src/core-common/src/test/java/org/apache/kylin/rest/service/PagingUtilTest.java diff --git a/src/systools/src/test/java/org/apache/kylin/rest/util/CreateTableFromJson.java b/src/core-metadata/src/main/java/org/apache/kylin/rest/util/CreateTableFromJson.java similarity index 92% rename from src/systools/src/test/java/org/apache/kylin/rest/util/CreateTableFromJson.java rename to src/core-metadata/src/main/java/org/apache/kylin/rest/util/CreateTableFromJson.java index fa54dc3348..635c8c816a 100644 --- a/src/systools/src/test/java/org/apache/kylin/rest/util/CreateTableFromJson.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/rest/util/CreateTableFromJson.java @@ -31,6 +31,8 @@ import org.apache.kylin.metadata.model.TableDesc; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A tool to generate database and tables from metadata backups of customer @@ -40,6 +42,8 @@ public class CreateTableFromJson { private static final String QUOTE = "`"; private static final Map<String, String> TYPE_MAP = Maps.newHashMap(); + private static final Logger logger = LoggerFactory.getLogger(CreateTableFromJson.class); + static { TYPE_MAP.put("integer", "int"); TYPE_MAP.put("long", "bigint"); @@ -51,11 +55,11 @@ public class CreateTableFromJson { Map<String, List<String>> map = createDbAndTables(pathDir); map.forEach((k, v) -> { - System.out.println(k); - v.forEach(System.out::println); + logger.info(k); + v.forEach(item -> logger.info(item)); }); - System.out.println("\n\n\n\n\n"); + logger.info("\n\n\n\n\n"); } // the path is /{metadata_backup_path}/{project_name}/table/ @@ -73,7 +77,7 @@ public class CreateTableFromJson { columnNameTypeList.add(String.format(Locale.ROOT, "%s %s", quote(name), type)); } - String databaseSql = String.format(Locale.ROOT, "create database %s;\nuse %s;", + String databaseSql = String.format(Locale.ROOT, "create database %s;%nuse %s;", quote(tableDesc.getDatabase()), quote(tableDesc.getDatabase())); map.putIfAbsent(databaseSql, Lists.newArrayList()); String tableSql = createTableSql(tableDesc.getName(), columnNameTypeList); diff --git a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java index de7cef0bf1..ed87e6c628 100644 --- a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java +++ b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java @@ -97,7 +97,6 @@ import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore; import org.apache.kylin.query.util.PushDownUtil; import org.apache.kylin.query.util.QueryUtil; import org.apache.kylin.rest.config.initialize.ModelBrokenListener; -import org.apache.kylin.rest.exception.BadRequestException; import org.apache.kylin.rest.request.ModelRequest; import org.apache.kylin.rest.request.PartitionsRefreshRequest; import org.apache.kylin.rest.request.SegmentTimeRequest; @@ -550,7 +549,7 @@ public class ModelServiceBuildTest extends SourceTestCase { modelRequest.setManagementType(ManagementType.MODEL_BASED); modelRequest.setLastModified(0L); modelRequest.setProject("match"); - thrown.expect(BadRequestException.class); + thrown.expect(RuntimeException.class); thrown.expectMessage("Can not build segments, please define table index or aggregate index first!"); modelService.createModel(modelRequest.getProject(), modelRequest); modelBuildService.buildSegmentsManually("match", "new_model", "0", "100"); diff --git a/src/integration-service/pom.xml b/src/integration-service/pom.xml index e935737fa1..e69de29bb2 100644 --- a/src/integration-service/pom.xml +++ b/src/integration-service/pom.xml @@ -1,125 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <parent> - <artifactId>kylin</artifactId> - <groupId>org.apache.kylin</groupId> - <version>5.0.0-alpha-SNAPSHOT</version> - <relativePath>../../pom.xml</relativePath> - </parent> - <modelVersion>4.0.0</modelVersion> - - <name>Kylin - Integration Service</name> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-integration-service</artifactId> - <dependencies> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-query-service</artifactId> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-data-loading-service</artifactId> - </dependency> - <dependency> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> - </dependency> - - <!-- test --> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-systools</artifactId> - <type>test-jar</type> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-hive_2.12</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-metastore</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-core</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-test</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-datasource-service</artifactId> - <type>test-jar</type> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.powermock</groupId> - <artifactId>powermock-module-junit4</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.powermock</groupId> - <artifactId>powermock-api-mockito2</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.junit.vintage</groupId> - <artifactId>junit-vintage-engine</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.awaitility</groupId> - <artifactId>awaitility</artifactId> - </dependency> - <dependency> - <groupId>com.h2database</groupId> - <artifactId>h2</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - - <properties> - <maven.compiler.source>8</maven.compiler.source> - <maven.compiler.target>8</maven.compiler.target> - </properties> - -</project> diff --git a/src/integration-service/src/test/java/org/apache/kylin/rest/broadcaster/BroadcasterTest.java b/src/kylin-it/src/test/java/org/apache/kylin/rest/broadcaster/BroadcasterTest.java similarity index 90% rename from src/integration-service/src/test/java/org/apache/kylin/rest/broadcaster/BroadcasterTest.java rename to src/kylin-it/src/test/java/org/apache/kylin/rest/broadcaster/BroadcasterTest.java index 717e570023..1f267173ae 100644 --- a/src/integration-service/src/test/java/org/apache/kylin/rest/broadcaster/BroadcasterTest.java +++ b/src/kylin-it/src/test/java/org/apache/kylin/rest/broadcaster/BroadcasterTest.java @@ -18,10 +18,7 @@ package org.apache.kylin.rest.broadcaster; -import static org.awaitility.Awaitility.await; - import java.util.Arrays; -import java.util.concurrent.TimeUnit; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.transaction.AddS3CredentialToSparkBroadcastEventNotifier; @@ -71,18 +68,8 @@ class BroadcasterTest { try (ConfigurableApplicationContext context = this.application.run("--kylin.server.mode=all")) { SpringContext springContext = context.getBean(SpringContext.class); ReflectionTestUtils.setField(springContext, "applicationContext", context); - Broadcaster broadcaster = Broadcaster.getInstance(KylinConfig.getInstanceFromEnv(), - new BroadcastListener() { - @Override - public void handle(BroadcastEventReadyNotifier notifier) { - log.info("received notifier {}.", notifier); - } - }); - + Broadcaster broadcaster = context.getBean(Broadcaster.class); broadcaster.announce(new BroadcastEventReadyNotifier()); - - await().atLeast(2, TimeUnit.SECONDS); - ClusterManager clusterManager = (ClusterManager) ReflectionTestUtils.getField(broadcaster, "clusterManager"); Assertions.assertNotNull(clusterManager); @@ -122,6 +109,11 @@ class BroadcasterTest { public ClusterManager clusterManager() { return new DefaultClusterManager(7070); } + + @Bean + public Broadcaster broadcaster() { + return new Broadcaster(clusterManager()); + } } } diff --git a/src/modeling-service/pom.xml b/src/modeling-service/pom.xml index 61b9b9f040..39c35f8856 100644 --- a/src/modeling-service/pom.xml +++ b/src/modeling-service/pom.xml @@ -100,10 +100,6 @@ <artifactId>powermock-api-mockito2</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>org.junit.vintage</groupId> - <artifactId>junit-vintage-engine</artifactId> - </dependency> <dependency> <groupId>org.springframework</groupId> diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java index eb859e0ed9..df0b60b2b6 100644 --- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java +++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java @@ -162,7 +162,6 @@ import org.apache.kylin.query.util.QueryUtil; import org.apache.kylin.rest.config.initialize.ModelBrokenListener; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum; -import org.apache.kylin.rest.exception.BadRequestException; import org.apache.kylin.rest.request.ModelConfigRequest; import org.apache.kylin.rest.request.ModelRequest; import org.apache.kylin.rest.request.MultiPartitionMappingRequest; @@ -3006,16 +3005,15 @@ public class ModelServiceTest extends SourceTestCase { } @Test - @Ignore("Metadata changed! Model nmodel_basic is not existed") public void testSetIncrementing_LimitedFactTable_exception() { val modelManager = NDataModelManager.getInstance(getTestConfig(), "default"); val model = modelManager.getDataModelDesc("89af4ee2-2cdb-4b07-b39e-4c29856309aa"); val joinTableDesc = new JoinTableDesc(); joinTableDesc.setTable("DEFAULT.TEST_KYLIN_FACT"); model.setJoinTables(Lists.newArrayList(joinTableDesc)); - thrown.expect(BadRequestException.class); + thrown.expect(KylinException.class); thrown.expectMessage( - "Can not set table 'DEFAULT.TEST_KYLIN_FACT' incremental loading, due to another incremental loading table existed in model 'nmodel_basic'!"); + "Can‘t set table \"DEFAULT.TEST_KYLIN_FACT\" as incremental loading. It’s been used as a dimension table in model \"nmodel_basic\"."); modelService.checkSingleIncrementingLoadingTable("default", "DEFAULT.TEST_KYLIN_FACT"); } diff --git a/src/query-server/pom.xml b/src/query-server/pom.xml index 3ed9aecaab..01fd906d4f 100644 --- a/src/query-server/pom.xml +++ b/src/query-server/pom.xml @@ -69,11 +69,6 @@ <artifactId>kylin-systools</artifactId> </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-integration-service</artifactId> - </dependency> - <dependency> <groupId>io.springfox</groupId> <artifactId>springfox-boot-starter</artifactId> diff --git a/src/query-service/pom.xml b/src/query-service/pom.xml index 26fbb531f6..a64b54af78 100644 --- a/src/query-service/pom.xml +++ b/src/query-service/pom.xml @@ -171,7 +171,6 @@ <dependency> <groupId>org.awaitility</groupId> <artifactId>awaitility</artifactId> - <version>${awaitility.version}</version> <scope>test</scope> </dependency> <dependency> diff --git a/src/integration-service/src/main/java/org/apache/kylin/rest/service/MonitorService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/MonitorService.java similarity index 99% rename from src/integration-service/src/main/java/org/apache/kylin/rest/service/MonitorService.java rename to src/query-service/src/main/java/org/apache/kylin/rest/service/MonitorService.java index b774049c75..1a096b5f01 100644 --- a/src/integration-service/src/main/java/org/apache/kylin/rest/service/MonitorService.java +++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/MonitorService.java @@ -528,7 +528,7 @@ public class MonitorService extends BasicService implements ApplicationListener< // to do config, 90d monitor data, OOM warning long range = 90L * 24 * 60 * 60 * 1000 / interval; if (size > range) { - throw new RuntimeException("Out of data range, only can calculate 90 days monitor data!"); + throw new IllegalArgumentException("Out of data range, only can calculate 90 days monitor data!"); } T[] fullMetrics = createMetricArray(); diff --git a/src/integration-service/src/test/java/org/apache/kylin/rest/service/MonitorServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/MonitorServiceTest.java similarity index 100% rename from src/integration-service/src/test/java/org/apache/kylin/rest/service/MonitorServiceTest.java rename to src/query-service/src/test/java/org/apache/kylin/rest/service/MonitorServiceTest.java diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java index c1f23bb5d7..2c46e619ea 100644 --- a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java +++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java @@ -56,7 +56,6 @@ import org.apache.kylin.metadata.query.QueryStatistics; import org.apache.kylin.metadata.query.RDBMSQueryHistoryDAO; import org.apache.kylin.metadata.query.RDBMSQueryHistoryDaoTest; import org.apache.kylin.rest.constant.Constant; -import org.apache.kylin.rest.exception.BadRequestException; import org.apache.kylin.rest.response.QueryStatisticsResponse; import org.apache.kylin.rest.util.AclEvaluate; import org.apache.kylin.rest.util.AclUtil; @@ -368,12 +367,8 @@ public class QueryHistoryServiceTest extends NLocalFileMetadataTestCase { Assert.assertEquals(28, tableMap.size()); // not existing project - try { - tableMap = queryHistoryService.getQueryHistoryTableMap(Lists.newArrayList("not_existing_project")); - } catch (Exception ex) { - Assert.assertEquals(BadRequestException.class, ex.getClass()); - Assert.assertEquals("Cannot find project 'not_existing_project'.", ex.getMessage()); - } + tableMap = queryHistoryService.getQueryHistoryTableMap(Lists.newArrayList("not_existing_project")); + Assert.assertTrue(tableMap.isEmpty()); } @Test diff --git a/src/second-storage/core-ui/pom.xml b/src/second-storage/core-ui/pom.xml index dea9225ee0..679020e968 100644 --- a/src/second-storage/core-ui/pom.xml +++ b/src/second-storage/core-ui/pom.xml @@ -121,11 +121,6 @@ <artifactId>hadoop-mapreduce-client-core</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <scope>test</scope> - </dependency> <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-metastore</artifactId> diff --git a/src/server/pom.xml b/src/server/pom.xml index be92d6ad50..dec842832d 100644 --- a/src/server/pom.xml +++ b/src/server/pom.xml @@ -221,6 +221,10 @@ <groupId>org.javassist</groupId> <artifactId>javassist</artifactId> </dependency> + <dependency> + <groupId>com.netflix.servo</groupId> + <artifactId>servo-core</artifactId> + </dependency> <dependency> <groupId>org.apache.hadoop</groupId> diff --git a/src/systools/src/main/java/org/apache/kylin/rest/config/CorsConfig.java b/src/server/src/main/java/org/apache/kylin/rest/config/CorsConfig.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/config/CorsConfig.java rename to src/server/src/main/java/org/apache/kylin/rest/config/CorsConfig.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/config/WebMvcConfig.java b/src/server/src/main/java/org/apache/kylin/rest/config/WebMvcConfig.java similarity index 94% rename from src/systools/src/main/java/org/apache/kylin/rest/config/WebMvcConfig.java rename to src/server/src/main/java/org/apache/kylin/rest/config/WebMvcConfig.java index 8483801ab7..63b2b6f7ba 100644 --- a/src/systools/src/main/java/org/apache/kylin/rest/config/WebMvcConfig.java +++ b/src/server/src/main/java/org/apache/kylin/rest/config/WebMvcConfig.java @@ -19,10 +19,10 @@ package org.apache.kylin.rest.config; import org.springframework.context.annotation.Configuration; import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; @Configuration -public class WebMvcConfig extends WebMvcConfigurerAdapter { +public class WebMvcConfig implements WebMvcConfigurer { @Override public void addResourceHandlers(ResourceHandlerRegistry registry) { diff --git a/src/spark-project/engine-build-sdk/pom.xml b/src/spark-project/engine-build-sdk/pom.xml index e5497f43b0..8a7068675c 100644 --- a/src/spark-project/engine-build-sdk/pom.xml +++ b/src/spark-project/engine-build-sdk/pom.xml @@ -48,16 +48,6 @@ </exclusion> </exclusions> </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-job</artifactId> - <exclusions> - <exclusion> - <groupId>*</groupId> - <artifactId>*</artifactId> - </exclusion> - </exclusions> - </dependency> <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-common</artifactId> diff --git a/src/spark-project/engine-spark/pom.xml b/src/spark-project/engine-spark/pom.xml index e8aa56ba40..99f02a7cf8 100644 --- a/src/spark-project/engine-spark/pom.xml +++ b/src/spark-project/engine-spark/pom.xml @@ -40,7 +40,7 @@ <dependencies> <dependency> <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-job</artifactId> + <artifactId>kylin-core-metadata</artifactId> </dependency> <dependency> <groupId>org.apache.kylin</groupId> diff --git a/src/spark-project/spark-common/pom.xml b/src/spark-project/spark-common/pom.xml index 0e9a3ccbe8..b4d5b433ef 100644 --- a/src/spark-project/spark-common/pom.xml +++ b/src/spark-project/spark-common/pom.xml @@ -40,11 +40,7 @@ <dependencies> <dependency> <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-common</artifactId> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-job</artifactId> + <artifactId>kylin-core-metadata</artifactId> </dependency> <dependency> <groupId>org.apache.kylin</groupId> diff --git a/src/spark-project/spark-common/src/main/java/org/apache/kylin/spark/common/logging/AbstractHdfsLogAppender.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/spark/common/logging/AbstractHdfsLogAppender.java index 3403a1d36d..b4c118e7e6 100644 --- a/src/spark-project/spark-common/src/main/java/org/apache/kylin/spark/common/logging/AbstractHdfsLogAppender.java +++ b/src/spark-project/spark-common/src/main/java/org/apache/kylin/spark/common/logging/AbstractHdfsLogAppender.java @@ -40,7 +40,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.kylin.common.util.ExecutorServiceUtil; -import org.apache.kylin.common.util.Unsafe; import org.apache.logging.log4j.core.Filter; import org.apache.logging.log4j.core.Layout; import org.apache.logging.log4j.core.LogEvent; @@ -49,6 +48,7 @@ import org.apache.logging.log4j.core.appender.AppenderLoggingException; import org.apache.logging.log4j.core.appender.OutputStreamManager; import org.apache.logging.log4j.core.config.Property; import org.apache.logging.log4j.status.StatusLogger; +import org.apache.spark.utils.SparkHadoopUtils; import com.google.common.collect.Lists; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -56,7 +56,6 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import lombok.Getter; import lombok.Setter; import lombok.val; -import org.apache.spark.utils.SparkHadoopUtils; public abstract class AbstractHdfsLogAppender extends AbstractOutputStreamAppender<AbstractHdfsLogAppender.HdfsManager> { @@ -294,7 +293,7 @@ public abstract class AbstractHdfsLogAppender } try { - Unsafe.wait(initWriterLock, 1000); //waiting for acl to turn to current user + initWriterLock.wait(1000); } catch (InterruptedException e) { StatusLogger.getLogger().warn("Init writer interrupted!", e); // Restore interrupted state... diff --git a/src/streaming-sdk/pom.xml b/src/streaming-sdk/pom.xml index de78450f29..c82e8a1315 100644 --- a/src/streaming-sdk/pom.xml +++ b/src/streaming-sdk/pom.xml @@ -41,5 +41,5 @@ <artifactId>commons-lang3</artifactId> </dependency> </dependencies> - + </project> diff --git a/src/streaming/pom.xml b/src/streaming/pom.xml index 2545dc3da9..8b0dbd5b6e 100644 --- a/src/streaming/pom.xml +++ b/src/streaming/pom.xml @@ -39,7 +39,7 @@ <!-- KE dependencies --> <dependency> <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-job</artifactId> + <artifactId>kylin-core-metadata</artifactId> </dependency> <dependency> <groupId>org.apache.kylin</groupId> diff --git a/src/systools/pom.xml b/src/systools/pom.xml index e64ff36e53..2b14821cea 100644 --- a/src/systools/pom.xml +++ b/src/systools/pom.xml @@ -34,18 +34,6 @@ <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-metadata</artifactId> </dependency> - <dependency> - <groupId>org.aspectj</groupId> - <artifactId>aspectjrt</artifactId> - </dependency> - <dependency> - <groupId>org.aspectj</groupId> - <artifactId>aspectjweaver</artifactId> - </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-context-support</artifactId> - </dependency> <dependency> <groupId>org.springframework</groupId> <artifactId>spring-webmvc</artifactId> diff --git a/src/systools/src/main/java/org/apache/kylin/rest/exception/BadRequestException.java b/src/systools/src/main/java/org/apache/kylin/rest/exception/BadRequestException.java deleted file mode 100644 index 1c7e94606f..0000000000 --- a/src/systools/src/main/java/org/apache/kylin/rest/exception/BadRequestException.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.rest.exception; - -import org.apache.kylin.common.exception.KylinException; -import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.ResponseStatus; - -/** - * @author xduo - * - */ -//use KylinException instead -@Deprecated -@ResponseStatus(value = HttpStatus.BAD_REQUEST) -public class BadRequestException extends RuntimeException { - - private static final long serialVersionUID = -6798154278095441848L; - - private String code; - - /** - * legacy support, new APIs should not call this. Instead, new APIs should provide return code - */ - public BadRequestException(String msg) { - super(msg); - this.code = KylinException.CODE_UNDEFINED; - } - - public BadRequestException(String msg, String code) { - super(msg); - this.code = code; - } - - public BadRequestException(String msg, String code, Throwable cause) { - super(msg, cause); - this.code = code; - } - - public String getCode() { - return code; - } - -} diff --git a/src/systools/src/main/java/org/apache/kylin/rest/interceptor/HeaderMapRequestWrapper.java b/src/systools/src/main/java/org/apache/kylin/rest/interceptor/HeaderMapRequestWrapper.java deleted file mode 100644 index 75de97a00c..0000000000 --- a/src/systools/src/main/java/org/apache/kylin/rest/interceptor/HeaderMapRequestWrapper.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kylin.rest.interceptor; - -import java.util.Arrays; -import java.util.Collections; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletRequestWrapper; - -import com.google.common.collect.Sets; - -public class HeaderMapRequestWrapper extends HttpServletRequestWrapper { - private Map<String, String> headerMap = new HashMap<>(); - - /** - * construct a wrapper for this request - * - * @param request - */ - public HeaderMapRequestWrapper(HttpServletRequest request) { - super(request); - } - - /** - * add a header with given name and value - * - * @param name - * @param value - */ - public void addHeader(String name, String value) { - headerMap.put(name, value); - } - - @Override - public String getHeader(String name) { - String headerValue = super.getHeader(name); - if (headerMap.containsKey(name)) { - headerValue = headerMap.get(name); - } - return headerValue; - } - - @Override - public Enumeration<String> getHeaderNames() { - List<String> names = Collections.list(super.getHeaderNames()); - names.addAll(headerMap.keySet()); - return Collections.enumeration(Sets.newHashSet(names)); - } - - @Override - public Enumeration<String> getHeaders(String name) { - List<String> values = Collections.list(super.getHeaders(name)); - if (headerMap.containsKey(name)) { - values = Arrays.asList(headerMap.get(name)); - } - return Collections.enumeration(values); - } -} diff --git a/src/systools/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java b/src/systools/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java index 3230401fa2..345f271ba7 100644 --- a/src/systools/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java +++ b/src/systools/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java @@ -35,7 +35,7 @@ import org.springframework.security.acls.model.Permission; /** */ -abstract public class ExternalAclProvider { +public abstract class ExternalAclProvider { public static ExternalAclProvider getInstance() { return Singletons.getInstance(ExternalAclProvider.class, clz -> { @@ -51,11 +51,11 @@ abstract public class ExternalAclProvider { // ============================================================================ - public final static String ADMINISTRATION = "ADMIN"; - public final static String MANAGEMENT = "MANAGEMENT"; - public final static String OPERATION = "OPERATION"; - public final static String READ = "QUERY"; - public final static String EMPTY = "EMPTY"; + public static final String ADMINISTRATION = "ADMIN"; + public static final String MANAGEMENT = "MANAGEMENT"; + public static final String OPERATION = "OPERATION"; + public static final String READ = "QUERY"; + public static final String EMPTY = "EMPTY"; public static final String DATA_QUERY = "DATA_QUERY"; // used by ranger ExternalAclProvider @@ -116,7 +116,7 @@ abstract public class ExternalAclProvider { // ============================================================================ - abstract public void init(); + public abstract void init(); /** * Checks if a user has permission on an entity. @@ -129,7 +129,7 @@ abstract public class ExternalAclProvider { * * @return true if has permission */ - abstract public boolean checkPermission(String user, List<String> userRoles, // + public abstract boolean checkPermission(String user, List<String> userRoles, // String entityType, String entityUuid, Permission permission); /** @@ -139,6 +139,6 @@ abstract public class ExternalAclProvider { * @param entityUuid * @return a list of (user/role, permission) */ - abstract public List<Pair<String, AclPermission>> getAcl(String entityType, String entityUuid); + public abstract List<Pair<String, AclPermission>> getAcl(String entityType, String entityUuid); }