This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a change to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 39086e3 KYLIN-4846 Set the related query id to sparder job description
add 32f8d66 fix KYLIN-4850, merge Cube's override kylin_properties into
building job's configuration
add a6f78cd KYLIN-4818 Calculate cuboid rowcount via HLL
add 3d6db49 KYLIN-4818 Support Cube Planner Phase One in Kylin 4
add 9e33d93 KYLIN-4818 Persist metadata in SparkExecutable
add d8e5db8 KYLIN-4818 Performance profile for CuboidStatisticsJob
add 2e9597a KYLIN-4818 Reduce toString method call
add af8c8fe KYLIN-4818 Refine CuboidStatisticsJob to improve performance
add 6f4e356 KYLIN-4858 Support Kylin4 deployment on CDH 6.X (#1535)
add 44402d3 KYLIN-4853 QueryPreparedStatementCache invalid in Spark Query
Engine. (#1532)
add b08c1be KYLIN-4859 Log4J reinitialized/reconfigured by Spark Logging
(#1537)
add 124dd05 KYLIN-4818 Not store HLL binary file into RDBMS directly
add 2ad58c7 KYLIN-4872 Fix NPE when there are more than one segment if
cube planner is open
add af3a1b5 KYLIN-4857 Refactor system cube for kylin4
add ad7b472 add test case
add 7431240 KYLIN-4496: Metric data missing
add 3400338 KYLIN-4573 Add option to indicate whether to close file for
every append for Hive Producer
add 6f48ce8 KYLIN-4653 Make the capacity for the LinkedBlockingQueue of
BlockingReservoir configurable
add 5b3859c KYLIN-4653 Make the capacity for the LinkedBlockingQueue of
BlockingReservoir configurable
add dc5f295 Fix test case and fix system-cube.sh script
add e719f7b KYLIn-4874 Fix CubeMigrationCLI bug for kylin4
add 588e08b KYLIN-4877 Use all dimension columns as sort columns when
saving cuboid data
add 3aa72f3 KYLIN-4875 Remove executor configurations when execute
resource detect step (local mode)
add 76492b5 Minor, fix replace-jars-under-spark.sh
add d7d2064 Fix storage clean up tool
add 5650a4e HOTFIX Fix the issue of closing FileSystem
add dc2fc39 Update version of shade-guava
add 1bd6714 [maven-release-plugin] prepare release kylin-4.0.0-beta
add aa67482 [maven-release-plugin] prepare for next development iteration
No new revisions were added by this update.
Summary of changes:
.github/workflows/maven.yml | 8 +-
.../kylin/engine/mr/common/CubeStatsReader.java | 59 +-
.../engine/mr/common/CuboidRecommenderUtil.java | 6 +-
.../engine/mr/common/StatisticsDecisionUtil.java | 8 +-
build/bin/find-hadoop-conf-dir.sh | 5 +
build/bin/find-hive-dependency.sh | 5 +-
build/bin/kylin.sh | 3 +
build/bin/replace-jars-under-spark.sh | 149 ++++
build/bin/system-cube.sh | 38 +-
build/conf/kylin-server-log4j.properties | 11 +-
build/conf/spark-driver-log4j.properties | 2 -
.../org/apache/kylin/common/KylinConfigBase.java | 69 +-
.../org/apache/kylin/common/KylinConfigExt.java | 5 +-
.../kylin/common/annotation/Clarification.java | 6 +-
.../kylin/common/persistence/ResourceStore.java | 17 +-
.../org/apache/kylin/common/KylinConfigTest.java | 20 +
.../java/org/apache/kylin/cube/CubeSegment.java | 23 +-
.../cube/cuboid/algorithm/CuboidRecommender.java | 3 +-
.../cuboid/algorithm/greedy/GreedyAlgorithm.java | 16 +-
.../org/apache/kylin/cube/kv/CubeDimEncMap.java | 2 +
.../org/apache/kylin/measure/hllc/HLLCounter.java | 4 +-
.../apache/kylin/metrics/QuerySparkMetrics.java | 763 +++++++++++++++++++++
.../kylin/metrics/lib/impl/BlockingReservoir.java | 31 +-
.../kylin/metrics/lib/impl/MetricsSystem.java | 2 +
...pertyEnum.java => QuerySparkExecutionEnum.java} | 49 +-
...RPCPropertyEnum.java => QuerySparkJobEnum.java} | 41 +-
...CPropertyEnum.java => QuerySparkStageEnum.java} | 40 +-
examples/test_case_data/localmeta/kylin.properties | 5 +-
external/pom.xml | 3 +-
.../common/logging/AbstractHdfsLogAppender.java | 33 +-
.../engine/spark/application/SparkApplication.java | 2 +-
.../engine/spark/builder/NBuildSourceInfo.java | 16 +-
.../engine/spark/job/NResourceDetectStep.java | 9 +
.../kylin/engine/spark/job/NSparkExecutable.java | 23 +-
.../engine/spark/utils/UpdateMetadataUtil.java | 38 +-
.../kylin/engine/spark/job/CubeBuildJob.java | 136 +++-
.../kylin/engine/spark/job/CubeMergeJob.java | 4 +-
.../kylin/engine/spark/job/CuboidAggregator.scala | 1 +
.../engine/spark/job/CuboidStatisticsJob.scala | 167 +++++
.../engine/spark/job/ParentSourceChooser.scala | 36 +-
.../kylin/engine/spark/utils/Repartitioner.java | 4 +-
.../kylin/engine/spark/metadata/MetaData.scala | 21 +-
.../engine/spark/metadata/MetadataConverter.scala | 14 +-
.../org/apache/spark/sql/SparderContext.scala | 6 +
.../spark/sql/metrics/SparderMetricsListener.scala | 144 ++++
.../kylin/storage/hbase/HBaseResourceStore.java | 3 +-
.../kylin/metrics/lib/impl/hive/HiveProducer.java | 97 ++-
pom.xml | 2 +-
.../kylin/rest/controller/CubeController.java | 56 +-
.../apache/kylin/rest/init/InitialTaskManager.java | 2 +
.../apache/kylin/rest/job/StorageCleanupJob.java | 27 +-
.../kylin/rest/metrics/QueryMetricsFacade.java | 166 +----
.../apache/kylin/rest/response/SQLResponse.java | 10 +
.../org/apache/kylin/rest/service/CubeService.java | 6 +-
.../kylin/rest/service/DashboardService.java | 20 +-
.../apache/kylin/rest/service/QueryService.java | 38 +-
.../kylin/rest/job/StorageCleanupJobTest.java | 27 +-
.../kylin/rest/response/SQLResponseTest.java | 9 +-
server/src/main/resources/kylinMetrics.xml | 9 +-
.../kylin/rest/metrics/QueryMetricsTest.java | 254 ++++++-
.../org/apache/kylin/tool/CubeMigrationCLI.java | 13 +-
.../tool/metrics/systemcube/CubeDescCreator.java | 302 +++++---
.../metrics/systemcube/CubeInstanceCreator.java | 14 +-
.../tool/metrics/systemcube/HiveTableCreator.java | 136 ++--
.../tool/metrics/systemcube/KylinTableCreator.java | 20 +-
.../tool/metrics/systemcube/ModelCreator.java | 138 ++--
.../kylin/tool/metrics/systemcube/SCCreator.java | 18 +-
.../systemcube/streamingv2/KafkaTopicCreator.java | 6 +-
webapp/app/js/controllers/cube.js | 4 +-
webapp/app/js/services/cubes.js | 4 +-
webapp/app/partials/cubes/cube_detail.html | 2 +-
71 files changed, 2719 insertions(+), 711 deletions(-)
create mode 100644 build/bin/replace-jars-under-spark.sh
create mode 100644
core-metrics/src/main/java/org/apache/kylin/metrics/QuerySparkMetrics.java
copy
core-metrics/src/main/java/org/apache/kylin/metrics/property/{QueryPropertyEnum.java
=> QuerySparkExecutionEnum.java} (55%)
copy
core-metrics/src/main/java/org/apache/kylin/metrics/property/{QueryRPCPropertyEnum.java
=> QuerySparkJobEnum.java} (58%)
copy
core-metrics/src/main/java/org/apache/kylin/metrics/property/{QueryRPCPropertyEnum.java
=> QuerySparkStageEnum.java} (56%)
create mode 100644
kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/job/CuboidStatisticsJob.scala
create mode 100644
kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/metrics/SparderMetricsListener.scala
