kumarpritam863 commented on code in PR #12372:
URL: https://github.com/apache/iceberg/pull/12372#discussion_r1985440317
##########
kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/channel/CommitterImpl.java:
##########
@@ -56,51 +74,57 @@ public int compare(TopicPartition o1, TopicPartition o2) {
}
}
- @Override
- public void start(Catalog catalog, IcebergSinkConfig config, SinkTaskContext
context) {
- KafkaClientFactory clientFactory = new
KafkaClientFactory(config.kafkaProps());
+ public boolean hasLeaderPartitions(Collection<TopicPartition>
currentAssignedPartitions) {
ConsumerGroupDescription groupDesc;
try (Admin admin = clientFactory.createAdmin()) {
groupDesc = KafkaUtils.consumerGroupDescription(config.connectGroupId(),
admin);
}
-
- if (groupDesc.state() == ConsumerGroupState.STABLE) {
+ if (groupDesc.state() == ConsumerGroupState.STABLE) {
Collection<MemberDescription> members = groupDesc.members();
- Set<TopicPartition> partitions = context.assignment();
- if (isLeader(members, partitions)) {
- LOG.info("Task elected leader, starting commit coordinator");
- Coordinator coordinator = new Coordinator(catalog, config, members,
clientFactory, context);
- coordinatorThread = new CoordinatorThread(coordinator);
- coordinatorThread.start();
+ if (isLeader(members, currentAssignedPartitions)) {
+ membersWhenWorkerIsCoordinator = members;
+ return true;
}
}
+ return false;
+ }
- LOG.info("Starting commit worker");
- SinkWriter sinkWriter = new SinkWriter(catalog, config);
- worker = new Worker(config, clientFactory, sinkWriter, context);
- worker.start();
+ @Override
+ public void start(Catalog catalog, IcebergSinkConfig config, SinkTaskContext
context) {
+ // No-Op
}
@Override
- public void save(Collection<SinkRecord> sinkRecords) {
- if (sinkRecords != null && !sinkRecords.isEmpty()) {
- worker.save(sinkRecords);
+ public void start(Catalog catalog, IcebergSinkConfig config, SinkTaskContext
context, Collection<TopicPartition> addedPartitions) {
+ if (hasLeaderPartitions(addedPartitions)) {
+ LOG.info("Committer received leader partition. Starting Coordinator.");
+ startCoordinator();
}
- processControlEvents();
}
@Override
public void stop() {
- if (worker != null) {
- worker.stop();
- worker = null;
+ // No-Op
Review Comment:
Yeah actually thought of this but some some kafka code which adopted this so
took this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
