This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a change to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 95b1af0 Minor, refactor
new a8de6d4 KYLIN-4660 Create two module
new db1eba8 KYLIN-4660 remove some useless modules
new db866b7 KYLIN-4660 remove some unsupported features
new 82045c2 KYLIN-4660 Fix UT and compile error
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
assembly/pom.xml | 10 +-
{engine-mr => build-engine}/pom.xml | 17 +-
.../mr/BatchOptimizeJobCheckpointBuilder.java | 0
.../apache/kylin/engine/mr/ByteArrayWritable.java | 0
.../java/org/apache/kylin/engine/mr/CubingJob.java | 0
.../org/apache/kylin/engine/mr/DFSFileTable.java | 0
.../apache/kylin/engine/mr/DFSFileTableReader.java | 0
.../java/org/apache/kylin/engine/mr/IInput.java | 0
.../kylin/engine/mr/ILookupMaterializer.java | 0
.../java/org/apache/kylin/engine/mr/IMRInput.java | 0
.../java/org/apache/kylin/engine/mr/IMROutput.java | 0
.../org/apache/kylin/engine/mr/IMROutput2.java | 0
.../apache/kylin/engine/mr/JobBuilderSupport.java | 221 ++++
.../org/apache/kylin/engine/mr/KylinMapper.java | 0
.../org/apache/kylin/engine/mr/KylinReducer.java | 0
.../kylin/engine/mr/LookupMaterializeContext.java | 0
.../kylin/engine/mr/LookupSnapshotBuildJob.java | 0
.../java/org/apache/kylin/engine/mr/MRUtil.java | 5 -
.../kylin/engine/mr/SortedColumnDFSFile.java | 0
.../kylin/engine/mr/SortedColumnDFSFileReader.java | 0
.../kylin/engine/mr/common/AbstractHadoopJob.java | 0
.../kylin/engine/mr/common/BatchConstants.java | 0
.../kylin/engine/mr/common/CubeJobLockUtil.java | 0
.../kylin/engine/mr/common/CubeStatsReader.java | 0
.../kylin/engine/mr/common/CubeStatsWriter.java | 0
.../engine/mr/common/CuboidRecommenderUtil.java | 0
.../engine/mr/common/CuboidSchedulerUtil.java | 0
.../kylin/engine/mr/common/CuboidShardUtil.java | 0
.../engine/mr/common/CuboidStatsReaderUtil.java | 0
.../mr/common/DefaultSslProtocolSocketFactory.java | 0
.../engine/mr/common/DefaultX509TrustManager.java | 0
.../kylin/engine/mr/common/HadoopCmdOutput.java | 3 +-
.../engine/mr/common/HadoopJobStatusChecker.java | 0
.../engine/mr/common/HadoopShellExecutable.java | 0
.../kylin/engine/mr/common/JobInfoConverter.java | 0
.../kylin/engine/mr/common/JobRelatedMetaUtil.java | 0
.../engine/mr/common/MapReduceExecutable.java | 0
.../kylin/engine/mr/common/MapReduceUtil.java | 0
.../kylin/engine/mr/common/NDCuboidBuilder.java | 0
.../mr/common/SerializableConfiguration.java | 0
.../engine/mr/common/StatisticsDecisionUtil.java | 0
.../engine/mr/exception/HadoopShellException.java | 0
.../engine/mr/exception/MapReduceException.java | 0
.../mr/exception/SegmentNotFoundException.java | 0
.../engine/mr/steps/CubingExecutableUtil.java | 0
.../steps/UpdateCubeInfoAfterCheckpointStep.java | 0
.../src/test/resources/data/8d_cuboid/part-r-00000 | Bin
.../test/resources/data/base_cuboid/part-r-00000 | Bin
.../src/test/resources/dict/DW_SITES | Bin
.../src/test/resources/multi_file_double/data_0 | 0
.../src/test/resources/multi_file_double/data_1 | 0
.../src/test/resources/multi_file_double/data_2 | 0
.../src/test/resources/multi_file_double/data_3 | 0
.../src/test/resources/multi_file_double/data_4 | 0
.../test/resources/multi_file_empty_file/empty.txt | 0
.../resources/multi_file_empty_file/has_value.txt | 0
.../src/test/resources/multi_file_int/data_0 | 0
.../src/test/resources/multi_file_int/data_1 | 0
.../src/test/resources/multi_file_int/data_2 | 0
.../src/test/resources/multi_file_int/data_3 | 0
.../src/test/resources/multi_file_int/data_4 | 0
.../src/test/resources/multi_file_str/data_0 | 0
.../src/test/resources/multi_file_str/data_1 | 0
.../src/test/resources/multi_file_str/data_2 | 0
.../src/test/resources/multi_file_str/data_3 | 0
.../src/test/resources/multi_file_str/data_4 | 0
build/bin/download-flink.sh | 81 --
build/bin/find-flink-dependency.sh | 52 -
build/script/download-flink.sh | 71 --
.../org/apache/kylin/common/KylinConfigBase.java | 246 +++--
.../java/org/apache/kylin/common/QueryContext.java | 5 +
.../kylin/common/annotation/Clarification.java | 31 +-
.../apache/kylin/common/annotation/ConfigTag.java | 74 ++
.../apache/kylin/common/livy/LivyRestBuilder.java | 5 +-
.../java/org/apache/kylin/cube/CubeManager.java | 18 +-
.../kylin/cube/cli/DictionaryGeneratorCLI.java | 6 +-
.../java/org/apache/kylin/cube/model/CubeDesc.java | 18 -
.../apache/kylin/cube/model/SnapshotTableDesc.java | 12 -
.../org/apache/kylin/dict/ShrunkenDictionary.java | 160 ---
.../kylin/dict/ShrunkenDictionaryBuilder.java | 49 -
.../dict/lookup/AbstractLookupRowEncoder.java | 122 ---
.../kylin/dict/lookup/ExtTableSnapshotInfo.java | 149 ---
.../dict/lookup/ExtTableSnapshotInfoManager.java | 213 ----
.../kylin/dict/lookup/IExtLookupProvider.java | 36 -
.../kylin/dict/lookup/IExtLookupTableCache.java | 39 -
.../kylin/dict/lookup/LookupProviderFactory.java | 82 --
.../dict/lookup/cache/RocksDBLookupBuilder.java | 83 --
.../dict/lookup/cache/RocksDBLookupRowEncoder.java | 70 --
.../dict/lookup/cache/RocksDBLookupTable.java | 116 ---
.../dict/lookup/cache/RocksDBLookupTableCache.java | 430 --------
.../apache/kylin/dict/ShrunkenDictionaryTest.java | 144 ---
.../lookup/cache/RocksDBLookupRowEncoderTest.java | 80 --
.../lookup/cache/RocksDBLookupTableCacheTest.java | 231 -----
.../dict/lookup/cache/RocksDBLookupTableTest.java | 161 ---
.../org/apache/kylin/job/JobBuilderSupport.java | 184 ++++
engine-flink/pom.xml | 10 +-
engine-mr/pom.xml | 21 +-
engine-spark/pom.xml | 10 +-
.../test_kylin_cube_with_slr_left_join_desc.json | 4 +-
.../kylin/source/hive/ITHiveTableReaderTest.java | 28 +-
.../kylin-spark-classloader/pom.xml | 2 +-
kylin-spark-project/kylin-spark-common/pom.xml | 2 +-
kylin-spark-project/kylin-spark-engine/pom.xml | 22 +-
.../org/apache/spark/conf/rule/SparkConfRule.scala | 8 +-
kylin-spark-project/kylin-spark-metadata/pom.xml | 2 +-
kylin-spark-project/kylin-spark-query/pom.xml | 50 +-
kylin-spark-project/kylin-spark-test/pom.xml | 35 +-
kylin-spark-project/pom.xml | 2 +-
{storage-hbase => metastore-hbase}/pom.xml | 145 ++-
.../kylin/storage/hbase/HBaseConnection.java | 0
.../kylin/storage/hbase/HBaseResourceStore.java | 0
.../storage/hbase/steps/SandboxMetastoreCLI.java | 0
.../src/test/resources/data/8d_cuboid/part-r-00000 | Bin
.../test/resources/data/base_cuboid/part-r-00000 | Bin
.../test/resources/partition_list/_partition.lst | Bin
.../src/test/resources/partition_list/part-r-00000 | Bin
metrics-reporter-hive/pom.xml | 5 +-
parquet-assembly/pom.xml | 28 +-
pom.xml | 185 ++--
server-base/pom.xml | 79 +-
.../kylin/rest/controller/CubeController.java | 140 ++-
.../kylin/rest/controller/JobController.java | 40 +-
.../kylin/rest/controller/StreamingController.java | 564 +++++------
.../controller/StreamingCoordinatorController.java | 302 +++---
.../rest/controller/StreamingV2Controller.java | 964 +++++++++---------
.../kylin/rest/controller/TableController.java | 62 +-
.../kylin/rest/job/StorageCleanJobHbaseUtil.java | 35 -
.../apache/kylin/rest/service/BasicService.java | 9 +-
.../org/apache/kylin/rest/service/CubeService.java | 59 +-
.../apache/kylin/rest/service/HBaseInfoUtil.java | 59 +-
.../org/apache/kylin/rest/service/JobService.java | 95 +-
.../kylin/rest/service/KafkaConfigService.java | 129 ++-
.../apache/kylin/rest/service/QueryService.java | 27 +-
.../rest/service/StreamingCoordinatorService.java | 173 ++--
.../kylin/rest/service/StreamingV2Service.java | 1057 +++++++++-----------
.../apache/kylin/rest/service/TableService.java | 202 ++--
.../kylin/rest/job/StorageCleanupJobTest.java | 1 -
server/ServiceConfig.md | 37 -
server/pom.xml | 3 +-
.../java/org/apache/kylin/rest/DebugDeployCLI.java | 31 -
.../kylin/rest/service/AdminServiceTest.java | 2 +-
source-hive/pom.xml | 10 +-
.../apache/kylin/source/hive/HiveFlinkInput.java | 92 --
.../apache/kylin/source/hive/HiveInputBase.java | 57 --
.../org/apache/kylin/source/hive/HiveSource.java | 13 +-
.../apache/kylin/source/hive/HiveSparkInput.java | 54 -
source-jdbc/pom.xml | 10 +-
source-kafka/pom.xml | 10 +-
storage-hbase/pom.xml | 25 +-
storage-stream/pom.xml | 10 +-
stream-coordinator/pom.xml | 10 +-
stream-core/pom.xml | 10 +-
stream-receiver/pom.xml | 10 +-
stream-source-kafka/pom.xml | 10 +-
tool/pom.xml | 19 +-
.../org/apache/kylin/tool/CubeMigrationCLI.java | 9 +-
.../org/apache/kylin/tool/DiagnosisInfoCLI.java | 31 +-
.../kylin/tool/extractor/ClientEnvExtractor.java | 1 -
.../kylin/tool/extractor/CubeMetaExtractor.java | 64 +-
.../kylin/tool/extractor/HBaseUsageExtractor.java | 249 -----
.../kylin/tool/extractor}/NamedThreadFactory.java | 2 +-
.../kylin/tool/metrics/systemcube/SCCreator.java | 27 +-
.../streamingv2/StreamingMetadataCreator.java | 100 --
163 files changed, 2844 insertions(+), 5832 deletions(-)
copy {engine-mr => build-engine}/pom.xml (87%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/BatchOptimizeJobCheckpointBuilder.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/ByteArrayWritable.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/CubingJob.java (100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java (100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/IInput.java (100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/ILookupMaterializer.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/IMRInput.java (100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/IMROutput.java (100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java (100%)
create mode 100644
build-engine/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java (100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java (100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/LookupMaterializeContext.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/LookupSnapshotBuildJob.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/MRUtil.java (94%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFileReader.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/CubeJobLockUtil.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/CuboidSchedulerUtil.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/DefaultSslProtocolSocketFactory.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/DefaultX509TrustManager.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
(97%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/HadoopJobStatusChecker.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/HadoopShellExecutable.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/JobRelatedMetaUtil.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/MapReduceUtil.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/SerializableConfiguration.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/exception/HadoopShellException.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/exception/MapReduceException.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/exception/SegmentNotFoundException.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
(100%)
copy {engine-mr =>
build-engine}/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterCheckpointStep.java
(100%)
copy {engine-mr =>
build-engine}/src/test/resources/data/8d_cuboid/part-r-00000 (100%)
copy {engine-mr =>
build-engine}/src/test/resources/data/base_cuboid/part-r-00000 (100%)
copy {engine-mr => build-engine}/src/test/resources/dict/DW_SITES (100%)
copy {engine-mr => build-engine}/src/test/resources/multi_file_double/data_0
(100%)
copy {engine-mr => build-engine}/src/test/resources/multi_file_double/data_1
(100%)
copy {engine-mr => build-engine}/src/test/resources/multi_file_double/data_2
(100%)
copy {engine-mr => build-engine}/src/test/resources/multi_file_double/data_3
(100%)
copy {engine-mr => build-engine}/src/test/resources/multi_file_double/data_4
(100%)
copy {engine-mr =>
build-engine}/src/test/resources/multi_file_empty_file/empty.txt (100%)
copy {engine-mr =>
build-engine}/src/test/resources/multi_file_empty_file/has_value.txt (100%)
copy {engine-mr => build-engine}/src/test/resources/multi_file_int/data_0
(100%)
copy {engine-mr => build-engine}/src/test/resources/multi_file_int/data_1
(100%)
copy {engine-mr => build-engine}/src/test/resources/multi_file_int/data_2
(100%)
copy {engine-mr => build-engine}/src/test/resources/multi_file_int/data_3
(100%)
copy {engine-mr => build-engine}/src/test/resources/multi_file_int/data_4
(100%)
copy {engine-mr => build-engine}/src/test/resources/multi_file_str/data_0
(100%)
copy {engine-mr => build-engine}/src/test/resources/multi_file_str/data_1
(100%)
copy {engine-mr => build-engine}/src/test/resources/multi_file_str/data_2
(100%)
copy {engine-mr => build-engine}/src/test/resources/multi_file_str/data_3
(100%)
copy {engine-mr => build-engine}/src/test/resources/multi_file_str/data_4
(100%)
delete mode 100755 build/bin/download-flink.sh
delete mode 100755 build/bin/find-flink-dependency.sh
delete mode 100644 build/script/download-flink.sh
copy
stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/annotations/NonSideEffect.java
=>
core-common/src/main/java/org/apache/kylin/common/annotation/Clarification.java
(65%)
create mode 100644
core-common/src/main/java/org/apache/kylin/common/annotation/ConfigTag.java
delete mode 100644
core-dictionary/src/main/java/org/apache/kylin/dict/ShrunkenDictionary.java
delete mode 100644
core-dictionary/src/main/java/org/apache/kylin/dict/ShrunkenDictionaryBuilder.java
delete mode 100644
core-dictionary/src/main/java/org/apache/kylin/dict/lookup/AbstractLookupRowEncoder.java
delete mode 100644
core-dictionary/src/main/java/org/apache/kylin/dict/lookup/ExtTableSnapshotInfo.java
delete mode 100644
core-dictionary/src/main/java/org/apache/kylin/dict/lookup/ExtTableSnapshotInfoManager.java
delete mode 100644
core-dictionary/src/main/java/org/apache/kylin/dict/lookup/IExtLookupProvider.java
delete mode 100644
core-dictionary/src/main/java/org/apache/kylin/dict/lookup/IExtLookupTableCache.java
delete mode 100644
core-dictionary/src/main/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupBuilder.java
delete mode 100644
core-dictionary/src/main/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupRowEncoder.java
delete mode 100644
core-dictionary/src/main/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTable.java
delete mode 100644
core-dictionary/src/main/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCache.java
delete mode 100644
core-dictionary/src/test/java/org/apache/kylin/dict/ShrunkenDictionaryTest.java
delete mode 100644
core-dictionary/src/test/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupRowEncoderTest.java
delete mode 100644
core-dictionary/src/test/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCacheTest.java
delete mode 100644
core-dictionary/src/test/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableTest.java
create mode 100644
core-job/src/main/java/org/apache/kylin/job/JobBuilderSupport.java
copy {storage-hbase => metastore-hbase}/pom.xml (50%)
copy {storage-hbase =>
metastore-hbase}/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
(100%)
copy {storage-hbase =>
metastore-hbase}/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
(100%)
copy {storage-hbase =>
metastore-hbase}/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
(100%)
copy {storage-hbase =>
metastore-hbase}/src/test/resources/data/8d_cuboid/part-r-00000 (100%)
copy {storage-hbase =>
metastore-hbase}/src/test/resources/data/base_cuboid/part-r-00000 (100%)
copy {storage-hbase =>
metastore-hbase}/src/test/resources/partition_list/_partition.lst (100%)
copy {storage-hbase =>
metastore-hbase}/src/test/resources/partition_list/part-r-00000 (100%)
delete mode 100644 server/ServiceConfig.md
delete mode 100644
server/src/main/java/org/apache/kylin/rest/DebugDeployCLI.java
delete mode 100644
source-hive/src/main/java/org/apache/kylin/source/hive/HiveFlinkInput.java
delete mode 100644
source-hive/src/main/java/org/apache/kylin/source/hive/HiveSparkInput.java
delete mode 100644
tool/src/main/java/org/apache/kylin/tool/extractor/HBaseUsageExtractor.java
copy {stream-core/src/main/java/org/apache/kylin/stream/core/util =>
tool/src/main/java/org/apache/kylin/tool/extractor}/NamedThreadFactory.java
(97%)
delete mode 100644
tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/streamingv2/StreamingMetadataCreator.java
