This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/kylin.git
from fd4a472e34 minor, fix sonar reported bugs (#1875)
add 9706a9dd76 KYLIN-5187 Support soft affinity and local cache feature
add f2cdedb474 KYLIN-5187, Problem fix, 1. Fix exception to get more
detailed message for other FileSystem 2. Fix
de.thetaphi:forbiddenapis:2.3:check for forbidden api, add default charset UTF-8
No new revisions were added by this update.
Summary of changes:
.../org/apache/kylin/common/KylinConfigBase.java | 7 +-
.../test/resources/query/test_query}/query01.sql | 19 +-
.../pom.xml | 42 +--
.../apache/kylin/cache/KylinCacheConstants.java | 12 +-
.../kylin/cache/fs/AbstractCacheFileSystem.java | 330 +++++++++++++++++++++
.../kylin/cache/fs/AlluxioHdfsFileInputStream.java | 223 ++++++++++++++
.../kylin/cache/fs/CacheFileInputStream.java | 312 +++++++++++++++++++
.../kylin/cache/fs/CacheFileSystemConstants.java | 54 ++++
.../kylin/cache/fs/MemCacheFileInputStream.java | 216 ++++++++++++++
.../kylin/cache/fs/OnlyForTestCacheFileSystem.java | 19 +-
.../kylin/cache/fs/kylin/CacheAllFileSystem.java | 21 +-
.../kylin/cache/fs/kylin/KylinCacheFileSystem.java | 47 +++
.../apache/kylin/cache/utils/ConsistentHash.java | 88 ++++++
.../apache/kylin/cache/utils/ReflectionUtil.java | 168 +++++++++++
.../kylin/softaffinity/SoftAffinityConstants.java | 48 +--
.../kylin/softaffinity/SoftAffinityManager.scala | 146 +++++++++
.../scheduler/SoftAffinityListener.scala} | 33 +--
.../strategy/SoftAffinityAllocationTrait.scala | 33 +--
.../strategy/SoftAffinityStrategy.scala | 55 ++++
.../execution/datasources/CacheFilePartition.scala | 72 +++++
.../execution/datasources/CacheFileScanRDD.scala | 77 +++++
kylin-spark-project/kylin-spark-common/pom.xml | 5 +
.../common/logging/AbstractHdfsLogAppender.java | 3 +-
.../common/logging/SparkDriverHdfsLogAppender.java | 3 +-
.../common/logging/SparkExecutorHdfsAppender.java | 4 +-
.../sql/execution/datasource/FilePruner.scala | 9 +
.../org/apache/spark/utils/SparkHadoopUtils.scala | 15 +-
.../sql/execution/KylinFileSourceScanExec.scala | 19 +-
.../org/apache/spark/sql/SparderContext.scala | 31 +-
kylin-spark-project/pom.xml | 1 +
30 files changed, 1959 insertions(+), 153 deletions(-)
copy {build/CI/kylin-system-testing/query/sql/sql_timestamp =>
kylin-it/src/test/resources/query/test_query}/query01.sql (55%)
copy kylin-spark-project/{kylin-spark-common =>
kylin-soft-affinity-cache}/pom.xml (80%)
copy
core-job/src/main/java/org/apache/kylin/job/exception/JobStoppedException.java
=>
kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/KylinCacheConstants.java
(76%)
create mode 100644
kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/AbstractCacheFileSystem.java
create mode 100644
kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/AlluxioHdfsFileInputStream.java
create mode 100644
kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/CacheFileInputStream.java
create mode 100644
kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/CacheFileSystemConstants.java
create mode 100644
kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/MemCacheFileInputStream.java
copy core-job/src/main/java/org/apache/kylin/job/lock/MockJobLock.java =>
kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/OnlyForTestCacheFileSystem.java
(77%)
copy core-job/src/main/java/org/apache/kylin/job/lock/MockJobLock.java =>
kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/kylin/CacheAllFileSystem.java
(73%)
create mode 100644
kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/kylin/KylinCacheFileSystem.java
create mode 100644
kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/utils/ConsistentHash.java
create mode 100644
kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/utils/ReflectionUtil.java
copy core-common/src/main/java/org/apache/kylin/common/SourceDialect.java =>
kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/softaffinity/SoftAffinityConstants.java
(51%)
create mode 100644
kylin-spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/SoftAffinityManager.scala
copy
kylin-spark-project/{kylin-spark-engine/src/main/scala/org/apache/kylin/cluster/parser/SchedulerParser.scala
=>
kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/scheduler/SoftAffinityListener.scala}
(54%)
copy
assembly/src/test/java/org/apache/kylin/job/streaming/StreamDataLoader.java =>
kylin-spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/strategy/SoftAffinityAllocationTrait.scala
(56%)
create mode 100644
kylin-spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/strategy/SoftAffinityStrategy.scala
create mode 100644
kylin-spark-project/kylin-soft-affinity-cache/src/main/spark31/org/apache/spark/sql/execution/datasources/CacheFilePartition.scala
create mode 100644
kylin-spark-project/kylin-soft-affinity-cache/src/main/spark31/org/apache/spark/sql/execution/datasources/CacheFileScanRDD.scala
copy
core-job/src/main/java/org/apache/kylin/job/exception/JobStoppedException.java
=>
kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/utils/SparkHadoopUtils.scala
(73%)
