haowanv opened a new issue, #11764:
URL: https://github.com/apache/pinot/issues/11764
```
Resetting CallbackHandler: 28. Is resetting for shutdown: true.
197 START: CallbackHandler 28, INVOKE /pinot/IDEALSTATES listener:
org.apache.helix.controller.GenericHelixController@500051c5 type: FINALIZE
CallbackHandler 28 subscribing changes listener to path: /pinot/IDEALSTATES,
callback type: FINALIZE, event types: [NodeChildrenChanged], listener:
org.apache.helix.controller.GenericHelixController@500051c5, watchChild: true
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe child-change. path: /pinot/IDEALSTATES, listener:
org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path:
/pinot/IDEALSTATES/anga_track_alignments_OFFLINE, listener:
org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path: /pinot/IDEALSTATES/anga_track_vehicles_OFFLINE,
listener: org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path:
/pinot/IDEALSTATES/anga_workflow_review_tasks_OFFLINE, listener:
org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path: /pinot/IDEALSTATES/brokerResource, listener:
org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path: /pinot/IDEALSTATES/jobrunner_batches_OFFLINE,
listener: org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path: /pinot/IDEALSTATES/jobrunner_batches_REALTIME,
listener: org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path: /pinot/IDEALSTATES/jobrunner_jobs_OFFLINE,
listener: org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path: /pinot/IDEALSTATES/jobrunner_jobs_REALTIME,
listener: org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path:
/pinot/IDEALSTATES/kratos_drivesync_predrive_test_OFFLINE, listener:
org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path: /pinot/IDEALSTATES/leadControllerResource,
listener: org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path:
/pinot/IDEALSTATES/maglev_attendance_tickets_OFFLINE, listener:
org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path:
/pinot/IDEALSTATES/maglev_session_features_mining_OFFLINE, listener:
org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path:
/pinot/IDEALSTATES/maglev_surveymap_all_task_summary_OFFLINE, listener:
org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path: /pinot/IDEALSTATES/map_labeling_logs_OFFLINE,
listener: org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path: /pinot/IDEALSTATES/map_labeling_logs_REALTIME,
listener: org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path: /pinot/IDEALSTATES/map_making_logs_OFFLINE,
listener: org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path: /pinot/IDEALSTATES/map_making_logs_REALTIME,
listener: org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path: /pinot/IDEALSTATES/map_making_metrics_OFFLINE,
listener: org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path: /pinot/IDEALSTATES/map_making_metrics_REALTIME,
listener: org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path:
/pinot/IDEALSTATES/map_making_metrics_info_OFFLINE, listener:
org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path: /pinot/IDEALSTATES/ops_experiences_OFFLINE,
listener: org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path: /pinot/IDEALSTATES/patches_info_OFFLINE,
listener: org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path:
/pinot/IDEALSTATES/pipeline_task_run_stats_stats_OFFLINE, listener:
org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path:
/pinot/IDEALSTATES/pipeline_task_run_stats_stats_REALTIME, listener:
org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path:
/pinot/IDEALSTATES/region_customer_region_customer_regions_OFFLINE, listener:
org.apache.helix.controller.GenericHelixController@500051c5
CallbackHandler28, Subscribing to path: /pinot/IDEALSTATES took: 2
START: Generic GenericClusterController.onIdealStateChange() for cluster
pinot
END: GenericClusterController.onIdealStateChange() for cluster pinot
Controller stopping periodical rebalance timer at period 9223372036854775807
Controller stopping periodical rebalance timer at period 9223372036854775807
Get FINALIZE notification, skip the pipeline. Event :IdealStateChange
197 END:INVOKE CallbackHandler 28, /pinot/IDEALSTATES listener:
org.apache.helix.controller.GenericHelixController@500051c5 type: FINALIZE
Took: 3ms
Get FINALIZE notification, skip the pipeline. Event :IdealStateChange
reset handler: /pinot/IDEALSTATES,
org.apache.helix.controller.GenericHelixController@500051c5
Resetting CallbackHandler: 0. Is resetting for shutdown: true.
197 START: CallbackHandler 0, INVOKE /pinot/CONTROLLER listener:
org.apache.helix.manager.zk.DistributedLeaderElection@d88f893 type: FINALIZE
CallbackHandler 0 subscribing changes listener to path: /pinot/CONTROLLER,
callback type: FINALIZE, event types: [NodeCreated, NodeDeleted,
NodeChildrenChanged], listener:
org.apache.helix.manager.zk.DistributedLeaderElection@d88f893, watchChild: false
CallbackHandler 0 subscribing data change listener to path: /pinot/CONTROLLER
CallbackHandler0,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe data-change. path: /pinot/CONTROLLER, listener:
org.apache.helix.manager.zk.DistributedLeaderElection@d88f893
CallbackHandler0,
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
unsubscribe child-change. path: /pinot/CONTROLLER, listener:
org.apache.helix.manager.zk.DistributedLeaderElection@d88f893
CallbackHandler0, Subscribing to path: /pinot/CONTROLLER took: 1
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
tries to relinquish leadership for cluster: pinot
Stop StatusDumpTask
Removing listener:
org.apache.helix.controller.GenericHelixController@500051c5 on path:
/pinot/IDEALSTATES from cluster: pinot by instance:
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
Resetting CallbackHandler: 28. Is resetting for shutdown: true.
197 START: CallbackHandler 28, INVOKE /pinot/IDEALSTATES listener:
org.apache.helix.controller.GenericHelixController@500051c5 type: FINALIZE
Callback handler 28 received event in wrong order. Listener:
org.apache.helix.controller.GenericHelixController@500051c5, path:
/pinot/IDEALSTATES, expected types: [INIT], but was FINALIZE
Removing listener:
org.apache.helix.controller.GenericHelixController@500051c5 on path:
/pinot/LIVEINSTANCES from cluster: pinot by instance:
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
Resetting CallbackHandler: 7. Is resetting for shutdown: true.
197 START: CallbackHandler 7, INVOKE /pinot/LIVEINSTANCES listener:
org.apache.helix.controller.GenericHelixController@500051c5 type: FINALIZE
Callback handler 7 received event in wrong order. Listener:
org.apache.helix.controller.GenericHelixController@500051c5, path:
/pinot/LIVEINSTANCES, expected types: [INIT], but was FINALIZE
Removing listener:
org.apache.helix.controller.GenericHelixController@500051c5 on path:
/pinot/CONFIGS/CUSTOMIZED_STATE/pinot from cluster: pinot by instance:
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
Resetting CallbackHandler: 6. Is resetting for shutdown: true.
197 START: CallbackHandler 6, INVOKE /pinot/CONFIGS/CUSTOMIZED_STATE/pinot
listener: org.apache.helix.controller.GenericHelixController@500051c5 type:
FINALIZE
Callback handler 6 received event in wrong order. Listener:
org.apache.helix.controller.GenericHelixController@500051c5, path:
/pinot/CONFIGS/CUSTOMIZED_STATE/pinot, expected types: [INIT], but was FINALIZE
Removing listener:
org.apache.helix.controller.GenericHelixController@500051c5 on path:
/pinot/CONFIGS/CLUSTER/pinot from cluster: pinot by instance:
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
Resetting CallbackHandler: 5. Is resetting for shutdown: true.
197 START: CallbackHandler 5, INVOKE /pinot/CONFIGS/CLUSTER/pinot listener:
org.apache.helix.controller.GenericHelixController@500051c5 type: FINALIZE
Callback handler 5 received event in wrong order. Listener:
org.apache.helix.controller.GenericHelixController@500051c5, path:
/pinot/CONFIGS/CLUSTER/pinot, expected types: [INIT], but was FINALIZE
Removing listener:
org.apache.helix.controller.GenericHelixController@500051c5 on path:
/pinot/CONFIGS/RESOURCE from cluster: pinot by instance:
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
Resetting CallbackHandler: 4. Is resetting for shutdown: true.
197 START: CallbackHandler 4, INVOKE /pinot/CONFIGS/RESOURCE listener:
org.apache.helix.controller.GenericHelixController@500051c5 type: FINALIZE
Callback handler 4 received event in wrong order. Listener:
org.apache.helix.controller.GenericHelixController@500051c5, path:
/pinot/CONFIGS/RESOURCE, expected types: [INIT], but was FINALIZE
Removing listener:
org.apache.helix.controller.GenericHelixController@500051c5 on path:
/pinot/CONFIGS/PARTICIPANT from cluster: pinot by instance:
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
Resetting CallbackHandler: 3. Is resetting for shutdown: true.
197 START: CallbackHandler 3, INVOKE /pinot/CONFIGS/PARTICIPANT listener:
org.apache.helix.controller.GenericHelixController@500051c5 type: FINALIZE
Callback handler 3 received event in wrong order. Listener:
org.apache.helix.controller.GenericHelixController@500051c5, path:
/pinot/CONFIGS/PARTICIPANT, expected types: [INIT], but was FINALIZE
Removing listener:
org.apache.helix.controller.GenericHelixController@500051c5 on path:
/pinot/CONTROLLER from cluster: pinot by instance:
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
Resetting CallbackHandler: 2. Is resetting for shutdown: true.
197 START: CallbackHandler 2, INVOKE /pinot/CONTROLLER listener:
org.apache.helix.controller.GenericHelixController@500051c5 type: FINALIZE
Callback handler 2 received event in wrong order. Listener:
org.apache.helix.controller.GenericHelixController@500051c5, path:
/pinot/CONTROLLER, expected types: [INIT], but was FINALIZE
Removing listener:
org.apache.helix.messaging.handling.HelixTaskExecutor@6418e39e on path:
/pinot/CONTROLLER/MESSAGES from cluster: pinot by instance:
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
Resetting CallbackHandler: 1. Is resetting for shutdown: true.
197 START: CallbackHandler 1, INVOKE /pinot/CONTROLLER/MESSAGES listener:
org.apache.helix.messaging.handling.HelixTaskExecutor@6418e39e type: FINALIZE
Callback handler 1 received event in wrong order. Listener:
org.apache.helix.messaging.handling.HelixTaskExecutor@6418e39e, path:
/pinot/CONTROLLER/MESSAGES, expected types: [INIT], but was FINALIZE
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
relinquishes leadership for cluster: pinot, took: 1ms
197 END:INVOKE CallbackHandler 0, /pinot/CONTROLLER listener:
org.apache.helix.manager.zk.DistributedLeaderElection@d88f893 type: FINALIZE
Took: 2ms
Controller stopping periodical rebalance timer at period 9223372036854775807
GenericHelixController stopping onDemand timer
Shutting down DEFAULT pipeline
ClusterEventProcessor interrupted default-pinot
java.lang.InterruptedException: null
at java.lang.Object.wait(Native Method) ~[?:?]
at java.lang.Object.wait(Object.java:328) ~[?:?]
at
org.apache.helix.common.DedupEventBlockingQueue.take(DedupEventBlockingQueue.java:102)
~[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
at
org.apache.helix.common.ClusterEventBlockingQueue.take(ClusterEventBlockingQueue.java:75)
~[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
at
org.apache.helix.controller.GenericHelixController$ClusterEventProcessor.run(GenericHelixController.java:1550)
[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
END ClusterEventProcessor thread default-pinot
Shutting down TASK pipeline
ClusterEventProcessor interrupted task-pinot
java.lang.InterruptedException: null
at java.lang.Object.wait(Native Method) ~[?:?]
at java.lang.Object.wait(Object.java:328) ~[?:?]
at
org.apache.helix.common.DedupEventBlockingQueue.take(DedupEventBlockingQueue.java:102)
~[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
at
org.apache.helix.common.ClusterEventBlockingQueue.take(ClusterEventBlockingQueue.java:75)
~[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
at
org.apache.helix.controller.GenericHelixController$ClusterEventProcessor.run(GenericHelixController.java:1550)
[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
END ClusterEventProcessor thread task-pinot
Shutdown async worker PersistAssignmentWorker-pinot
Shutdown async worker ExternalViewComputeWorker-pinot
Shutdown async worker MaintenanceRecoveryWorker-pinot
Shutdown async worker TargetExternalViewCalcWorker-pinot
Shutdown async worker TaskJobPurgeWorker-pinot
Shutdown async worker CustomizedStateViewComputeWorker-pinot
Reset ClusterStatusMonitor
PersistAssignmentWorker thread interrupted
java.lang.InterruptedException: null
at java.lang.Object.wait(Native Method) ~[?:?]
at java.lang.Object.wait(Object.java:328) ~[?:?]
at
org.apache.helix.common.DedupEventBlockingQueue.take(DedupEventBlockingQueue.java:102)
~[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
at
org.apache.helix.common.DedupEventProcessor.run(DedupEventProcessor.java:60)
[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
MaintenanceRecoveryWorker thread interrupted
java.lang.InterruptedException: null
at java.lang.Object.wait(Native Method) ~[?:?]
at java.lang.Object.wait(Object.java:328) ~[?:?]
at
org.apache.helix.common.DedupEventBlockingQueue.take(DedupEventBlockingQueue.java:102)
~[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
at
org.apache.helix.common.DedupEventProcessor.run(DedupEventProcessor.java:60)
[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
END PersistAssignmentWorker thread for cluster pinot
TargetExternalViewCalcWorker thread interrupted
java.lang.InterruptedException: null
at java.lang.Object.wait(Native Method) ~[?:?]
at java.lang.Object.wait(Object.java:328) ~[?:?]
at
org.apache.helix.common.DedupEventBlockingQueue.take(DedupEventBlockingQueue.java:102)
~[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
at
org.apache.helix.common.DedupEventProcessor.run(DedupEventProcessor.java:60)
[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
TaskJobPurgeWorker thread interrupted
java.lang.InterruptedException: null
at java.lang.Object.wait(Native Method) ~[?:?]
at java.lang.Object.wait(Object.java:328) ~[?:?]
at
org.apache.helix.common.DedupEventBlockingQueue.take(DedupEventBlockingQueue.java:102)
~[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
at
org.apache.helix.common.DedupEventProcessor.run(DedupEventProcessor.java:60)
[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
ExternalViewComputeWorker thread interrupted
java.lang.InterruptedException: null
at java.lang.Object.wait(Native Method) ~[?:?]
at java.lang.Object.wait(Object.java:328) ~[?:?]
at
org.apache.helix.common.DedupEventBlockingQueue.take(DedupEventBlockingQueue.java:102)
~[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
at
org.apache.helix.common.DedupEventProcessor.run(DedupEventProcessor.java:60)
[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
CustomizedStateViewComputeWorker thread interrupted
java.lang.InterruptedException: null
at java.lang.Object.wait(Native Method) ~[?:?]
at java.lang.Object.wait(Object.java:328) ~[?:?]
at
org.apache.helix.common.DedupEventBlockingQueue.take(DedupEventBlockingQueue.java:102)
~[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
at
org.apache.helix.common.DedupEventProcessor.run(DedupEventProcessor.java:60)
[pinot-all-0.11.0-jar-with-dependencies.jar:0.11.0-1b4d6b6b0a27422c1552ea1a936ad145056f7033]
END MaintenanceRecoveryWorker thread for cluster pinot
END CustomizedStateViewComputeWorker thread for cluster pinot
END TargetExternalViewCalcWorker thread for cluster pinot
END TaskJobPurgeWorker thread for cluster pinot
END ExternalViewComputeWorker thread for cluster pinot
ZkClient AsyncCallback retry thread is interrupted.
Terminate ZkClient AsyncCallback retry thread.
Terminate ZkClient event thread.
Terminate ZkClient event thread.
Closing zkclient uid:6, zk:State:CONNECTED Timeout:30000
sessionid:0x3043dc07efa0188 local:/10.10.170.113:56550
remoteserver:pinot-zookeeper/10.96.109.137:2181 lastZxid:4294990497 xid:4
sent:13 recv:13 queuedpkts:0 pendingresp:0 queuedevents:0
Session: 0x3043dc07efa0188 closed
Closed zkclient with uid:6
EventThread shut down for session: 0x3043dc07efa0188
MBean Rebalancer: EntityName=WagedRebalancer,ClusterName=pinot has been
un-registered.
MBean
HelixCallback:Change=EXTERNAL_VIEW,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=CUSTOMIZED_VIEW,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=HEALTH,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=CONTROLLER,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=IDEAL_STATE,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=MESSAGE,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=CUSTOMIZED_VIEW_ROOT,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=LIVE_INSTANCE,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=TASK_CURRENT_STATE,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=TARGET_EXTERNAL_VIEW,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=CUSTOMIZED_STATE_CONFIG,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=MESSAGES_CONTROLLER,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=CURRENT_STATE,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=CONFIG,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=CUSTOMIZED_STATE_ROOT,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=CLUSTER_CONFIG,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=INSTANCE_CONFIG,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=CUSTOMIZED_STATE,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixCallback:Change=RESOURCE_CONFIG,Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
ZkClient AsyncCallback retry thread is interrupted.
Terminate ZkClient AsyncCallback retry thread.
Terminate ZkClient event thread.
Terminate ZkClient event thread.
Closing zkclient uid:5, zk:State:CONNECTED Timeout:30000
sessionid:0x2044e145d500192 local:/10.10.170.113:56548
remoteserver:pinot-zookeeper/10.96.109.137:2181 lastZxid:4294990692 xid:3728
sent:3737 recv:3773 queuedpkts:0 pendingresp:0 queuedevents:0
Session: 0x2044e145d500192 closed
EventThread shut down for session: 0x2044e145d500192
MBean
HelixZkClient:Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,Type=CONTROLLER
has been un-registered.
MBean
HelixZkClient:Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,PATH=Controller,Type=CONTROLLER
has been un-registered.
MBean
HelixZkClient:Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,PATH=ExternalView,Type=CONTROLLER
has been un-registered.
MBean
HelixZkClient:Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,PATH=LiveInstances,Type=CONTROLLER
has been un-registered.
MBean
HelixZkClient:Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,PATH=IdealStates,Type=CONTROLLER
has been un-registered.
MBean
HelixZkClient:Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,PATH=Configs,Type=CONTROLLER
has been un-registered.
MBean
HelixZkClient:Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,PATH=PropertyStore,Type=CONTROLLER
has been un-registered.
MBean
HelixZkClient:Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,PATH=CurrentStates,Type=CONTROLLER
has been un-registered.
MBean
HelixZkClient:Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,PATH=Root,Type=CONTROLLER
has been un-registered.
MBean
HelixZkClient:Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,PATH=Instances,Type=CONTROLLER
has been un-registered.
MBean
HelixZkClient:Key=pinot.pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000,PATH=Messages,Type=CONTROLLER
has been un-registered.
Closed zkclient with uid:5
Cluster manager:
pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
disconnected
Deregistering service status handler
Shutdown Controller Metrics Registry
Finish shutting down Pinot controller for
Controller_pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000
Pinot [CONTROLLER] Instance
[Controller_pinot-controller-0.pinot-controller-headless.pinot.svc.cluster.local_9000]
is Stopped...
Shutting down Pinot Service Manager admin application...
Deregistering service status handler
```
The pinot-controller log shows:
```
Callback handler 1 received event in wrong order.
```
But I didn't find in pinot source code where it can generate logs like
this(using keywords like 'in wrong order' 'Callback handler ' to search in
source code)
I have no idea why pinot controller keep crashing and restarting...
pinot code git version: 1b4d6b6b0a27422c1552ea1a936ad145056f7033
And.. This is the pinot controller part yaml file rendered by helmfile
template command:
```
apiVersion: apps/v1
kind: StatefulSet
metadata:
labels:
app: pinot
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/version: 0.2.5-SNAPSHOT
chart: pinot
component: controller
helm.sh/chart: pinot
heritage: Helm
release: pinot
name: pinot-controller
spec:
podManagementPolicy: Parallel
replicas: 1
selector:
matchLabels:
app: pinot
chart: pinot
component: controller
release: pinot
serviceName: pinot-controller-headless
template:
metadata:
annotations: {}
labels:
app: pinot
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/version: 0.2.5-SNAPSHOT
chart: pinot
component: controller
helm.sh/chart: pinot
heritage: Helm
release: pinot
spec:
affinity: {}
containers:
- args:
- StartController
- -configFileName
- /var/pinot/controller/config/pinot-controller.conf
env:
- name: JAVA_OPTS
value: -Xms256M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200
-Xlog:gc*:file=/opt/pinot/gc-pinot-controller.log
-Dlog4j2.configurationFile=/opt/pinot/conf/log4j2.xml
-Dplugins.dir=/opt/pinot/plugins
envFrom:
- secretRef:
name: pinot-aws-creds
image: pinot:1.2.7391
imagePullPolicy: Always
livenessProbe:
httpGet:
path: /health
port: 9000
initialDelaySeconds: 60
periodSeconds: 10
name: controller
ports:
- containerPort: 9000
name: controller
protocol: TCP
readinessProbe:
httpGet:
path: /health
port: 9000
initialDelaySeconds: 60
periodSeconds: 10
resources: {}
securityContext: {}
volumeMounts:
- mountPath: /var/pinot/controller/config
name: config
- mountPath: /var/pinot/controller/data
name: data
nodeSelector:
onprem-dedicated: service
restartPolicy: Always
securityContext: {}
serviceAccountName: pinot
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoSchedule
key: onprem-dedicated
operator: Equal
value: service
volumes:
- configMap:
name: pinot-controller-config
name: config
updateStrategy:
type: RollingUpdate
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1G
storageClassName: csi-super-high-perf
---
```
And this is pinot controller's config file:
```
apiVersion: v1
data:
pinot-controller.conf: |-
controller.helix.cluster.name=pinot
controller.port=9000
controller.disable.ingestion.groovy=false
controller.data.dir=s3://s3-bucket/pinot/data
controller.zk.str=pinot-zookeeper:2181
controller.local.temp.dir=/tmp/pinot-tmp-data/
pinot.set.instance.id.to.hostname=true
controller.task.scheduler.enabled=true
pinot.controller.storage.factory.class.s3=org.apache.pinot.plugin.filesystem.S3PinotFS
pinot.controller.storage.factory.s3.endpoint=s3-endpoint
pinot.controller.storage.factory.s3.region=s3-region
pinot.controller.segment.fetcher.protocols=file,http,s3
pinot.controller.segment.fetcher.s3.class=org.apache.pinot.common.utils.fetcher.PinotFSSegmentFetcher
pinot.controller.storage.factory.s3.disableAcl=false
kind: ConfigMap
metadata:
name: pinot-controller-config
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
