This is an automated email from the ASF dual-hosted git repository.
mridulm80 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
from 99739ae068d [SPARK-41840][CONNECT][TESTS] Remove the invalid JIRA in
the comment
add fb67ce26800 [SPARK-36620][SHUFFLE] Add Push Based Shuffle client side
read metrics
No new revisions were added by this update.
Summary of changes:
.../apache/spark/status/protobuf/store_types.proto | 61 ++
.../org/apache/spark/InternalAccumulator.scala | 11 +
.../scala/org/apache/spark/executor/Executor.scala | 70 ++-
.../org/apache/spark/executor/ExecutorSource.scala | 20 +
.../apache/spark/executor/ShuffleReadMetrics.scala | 143 +++++
.../org/apache/spark/executor/TaskMetrics.scala | 10 +
.../org/apache/spark/scheduler/DAGScheduler.scala | 2 +-
.../org/apache/spark/scheduler/StageInfo.scala | 4 +-
.../scala/org/apache/spark/shuffle/metrics.scala | 10 +
.../org/apache/spark/status/AppStatusStore.scala | 97 ++-
.../scala/org/apache/spark/status/LiveEntity.scala | 99 ++-
.../scala/org/apache/spark/status/api/v1/api.scala | 44 +-
.../status/protobuf/CachedQuantileSerializer.scala | 20 +
.../protobuf/StageDataWrapperSerializer.scala | 111 +++-
.../protobuf/TaskDataWrapperSerializer.scala | 20 +
.../scala/org/apache/spark/status/storeTypes.scala | 65 +-
.../spark/storage/PushBasedFetchHelper.scala | 24 +-
.../storage/ShuffleBlockFetcherIterator.scala | 85 ++-
.../scala/org/apache/spark/ui/jobs/JobPage.scala | 14 +-
.../scala/org/apache/spark/util/JsonProtocol.scala | 59 +-
.../complete_stage_list_json_expectation.json | 36 ++
.../excludeOnFailure_for_stage_expectation.json | 180 +++++-
...xcludeOnFailure_node_for_stage_expectation.json | 208 +++++-
.../failed_stage_list_json_expectation.json | 14 +-
..._json_details_with_failed_task_expectation.json | 12 +
.../one_stage_attempt_json_expectation.json | 124 +++-
.../one_stage_json_expectation.json | 124 +++-
.../one_stage_json_with_details_expectation.json | 108 ++++
...ne_stage_json_with_partitionId_expectation.json | 156 ++++-
.../stage_list_json_expectation.json | 56 +-
...age_list_with_accumulable_json_expectation.json | 14 +-
.../stage_list_with_peak_metrics_expectation.json | 56 +-
.../stage_task_list_expectation.json | 280 ++++++++-
...from_multi_attempt_app_json_1__expectation.json | 112 +++-
...from_multi_attempt_app_json_2__expectation.json | 112 +++-
...e_task_list_w__offset___length_expectation.json | 700 +++++++++++++++++++--
.../stage_task_list_w__sortBy_expectation.json | 280 ++++++++-
...__sortBy_short_names___runtime_expectation.json | 280 ++++++++-
...w__sortBy_short_names__runtime_expectation.json | 280 ++++++++-
...st_w__status___offset___length_expectation.json | 28 +-
...___sortBy_short_names__runtime_expectation.json | 280 ++++++++-
.../stage_task_list_w__status_expectation.json | 140 ++++-
...age_task_list_with_partitionId_expectation.json | 112 +++-
...sk_summary_w__custom_quantiles_expectation.json | 14 +-
...ge_task_summary_w_shuffle_read_expectation.json | 16 +-
...e_task_summary_w_shuffle_write_expectation.json | 14 +-
.../stage_with_accumulable_json_expectation.json | 126 +++-
.../stage_with_peak_metrics_expectation.json | 310 +++++++--
...stage_with_speculation_summary_expectation.json | 82 ++-
.../stage_with_summaries_expectation.json | 326 ++++++++--
.../resources/spark-events/app-20200706201101-0003 | 96 +--
.../apache/spark/status/AppStatusStoreSuite.scala | 1 +
.../protobuf/KVStoreProtobufSerializerSuite.scala | 158 ++++-
.../storage/ShuffleBlockFetcherIteratorSuite.scala | 78 ++-
.../scala/org/apache/spark/ui/StagePageSuite.scala | 14 +-
.../org/apache/spark/util/JsonProtocolSuite.scala | 189 +++++-
project/MimaExcludes.scala | 5 +
.../metric/SQLShuffleMetricsReporter.scala | 84 ++-
58 files changed, 5622 insertions(+), 552 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
