KKcorps opened a new pull request, #10337:
URL: https://github.com/apache/pinot/pull/10337
Currently, we only return aggregated statistics in the broker response. This
already helps out in debugging performance issues. However, stats aggregated
at stage level will be much more useful because the operations done by leaf
stage vs intermediate stage differ a lot.
The PR contains the following changes:
* Add a new BrokerResponseNativeV2 class that contains Stage level stats
* Add support for BrokerMetrics by passing down the table name in
OperatorStats
Example
```json
{
"resultTable":{
"dataSchema":{
"columnNames":[
"playerName",
"teamID",
"teamName"
],
"columnDataTypes":[
"STRING",
"STRING",
"STRING"
]
},
"rows":[
[
"Fausto Andres",
"CIN",
"Cincinnati Reds/Red Stockings"
],
[
"Fernando Antonio",
"HOU",
"Houston Astros/Colt .45s"
],
[
"Fernando Antonio",
"HOU",
"Houston Astros/Colt .45s"
],
[
"Fernando Antonio",
"HOU",
"Houston Astros/Colt .45s"
],
[
"Harry Frederick",
"CLE",
"Cleveland Indians/Naps/Broncos/Bluebirds/Lake Shores"
],
[
"Harry Frederick",
"CLE",
"Cleveland Indians/Naps/Broncos/Bluebirds/Lake Shores"
],
[
"William Glenn",
"DET",
"Detroit Tigers"
],
[
"William Glenn",
"DET",
"Detroit Tigers"
],
[
"Paul David",
"CLE",
"Cleveland Indians/Naps/Broncos/Bluebirds/Lake Shores"
],
[
"Albert Julius",
"CLE",
"Cleveland Indians/Naps/Broncos/Bluebirds/Lake Shores"
]
]
},
"stageStats":{
"1":{
"numBlocks":19,
"numRows":30,
"threadExecutionTime":642,
"operatorIds":[
"MAILBOX_RECEIVE_3_1_0@127.0.0.1:8421",
"SORT_3_1_0@127.0.0.1:8444",
"SORT_3_1_0@127.0.0.1:8442",
"SORT_3_1_0@127.0.0.1:8443",
"TRANSFORM_3_1_0@127.0.0.1:8444",
"TRANSFORM_3_1_0@127.0.0.1:8443",
"TRANSFORM_3_1_0@127.0.0.1:8442"
]
},
"2":{
"numBlocks":40,
"numRows":47795,
"threadExecutionTime":630,
"operatorIds":[
"MAILBOX_RECEIVE_3_2_0@127.0.0.1:8444",
"MAILBOX_RECEIVE_3_2_0@127.0.0.1:8443",
"MAILBOX_RECEIVE_3_2_0@127.0.0.1:8442",
"SORT_3_2_0@127.0.0.1:8442",
"HASH_JOIN_3_2_0@127.0.0.1:8444",
"HASH_JOIN_3_2_0@127.0.0.1:8443",
"SORT_3_2_0@127.0.0.1:8444",
"HASH_JOIN_3_2_0@127.0.0.1:8442",
"SORT_3_2_0@127.0.0.1:8443"
]
},
"3":{
"numBlocks":12,
"numRows":195778,
"threadExecutionTime":75,
"numSegmentsQueried":1,
"numSegmentsProcessed":1,
"numSegmentsMatched":1,
"numDocsScanned":97889,
"numEntriesScannedPostFilter":195778,
"totalDocs":97889,
"operatorIds":[
"MAILBOX_RECEIVE_3_3_0@127.0.0.1:8444",
"MAILBOX_RECEIVE_3_3_0@127.0.0.1:8443",
"MAILBOX_RECEIVE_3_3_0@127.0.0.1:8442",
"LEAF_STAGE_TRANSFER_OPERATOR_3_3_0@127.0.0.1:8443"
]
},
"4":{
"numBlocks":13,
"numRows":102,
"threadExecutionTime":21,
"numSegmentsQueried":1,
"numSegmentsProcessed":1,
"numSegmentsMatched":1,
"numDocsScanned":51,
"numEntriesScannedPostFilter":102,
"totalDocs":51,
"operatorIds":[
"MAILBOX_RECEIVE_3_4_0@127.0.0.1:8442",
"MAILBOX_RECEIVE_3_4_0@127.0.0.1:8444",
"MAILBOX_RECEIVE_3_4_0@127.0.0.1:8443",
"LEAF_STAGE_TRANSFER_OPERATOR_3_4_0@127.0.0.1:8442"
]
}
},
"exceptions":[
],
"numServersQueried":0,
"numServersResponded":0,
"numSegmentsQueried":2,
"numSegmentsProcessed":2,
"numSegmentsMatched":2,
"numConsumingSegmentsQueried":0,
"numConsumingSegmentsProcessed":0,
"numConsumingSegmentsMatched":0,
"numDocsScanned":97940,
"numEntriesScannedInFilter":0,
"numEntriesScannedPostFilter":195880,
"numGroupsLimitReached":false,
"totalDocs":97940,
"timeUsedMs":108,
"offlineThreadCpuTimeNs":0,
"realtimeThreadCpuTimeNs":0,
"offlineSystemActivitiesCpuTimeNs":0,
"realtimeSystemActivitiesCpuTimeNs":0,
"offlineResponseSerializationCpuTimeNs":0,
"realtimeResponseSerializationCpuTimeNs":0,
"offlineTotalCpuTimeNs":0,
"realtimeTotalCpuTimeNs":0,
"segmentStatistics":[
],
"traceInfo":{
},
"minConsumingFreshnessTimeMs":0,
"numSegmentsPrunedByBroker":0,
"numSegmentsPrunedByServer":0,
"numSegmentsPrunedInvalid":0,
"numSegmentsPrunedByLimit":0,
"numSegmentsPrunedByValue":0,
"explainPlanNumEmptyFilterSegments":0,
"explainPlanNumMatchAllFilterSegments":0,
"numRowsResultSet":10
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
