mbwaheed commented on a change in pull request #1188: SOLR-14044: Support
collection and shard deletion in shared storage
URL: https://github.com/apache/lucene-solr/pull/1188#discussion_r374908649
##########
File path:
solr/core/src/java/org/apache/solr/store/blob/process/BlobDeleteManager.java
##########
@@ -18,40 +18,77 @@
package org.apache.solr.store.blob.process;
import java.lang.invoke.MethodHandles;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.lucene.util.NamedThreadFactory;
-import org.apache.solr.common.util.ExecutorUtil.MDCAwareThreadPoolExecutor;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.store.blob.client.CoreStorageClient;
import org.apache.solr.store.blob.metadata.CorePushPull;
+import org.apache.solr.store.blob.metadata.ServerSideMetadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
+
/**
- * Manager of blobs (files) to delete, putting them in a queue (if space left
on the queue) then consumed and processed
- * by {@link BlobDeleterTask}
+ * This class manages the deletion machinery required by shared storage
enabled collections. Its responsibilities
+ * include the allocation and management of bounded deletion task queues and
their consumers.
+ *
+ * Deletion of blob files from shared store happen on two paths:
+ * 1. In the indexing path, the local {@link SolrCore}'s index files
represented by an instance of a
+ * {@link ServerSideMetadata} object is resolved against the blob store's
core.metadata file, or the
+ * the source of truth for what index files a {@link SolrCore} should have.
As the difference between
+ * these two metadata instances are resolved, we add files to be deleted to
the BlobDeleteManager which
+ * enqueues a {@link BlobDeleterTask} for asynchronous processing.
+ * 2. In the collection admin API, we may delete a collection or collection
shard. In the former, all index
+ * files belonging to the specified collection on shared storage should be
deleted while in the latter
+ * all index files belonging to a particular collection/shard pair should be
deleted.
+ *
+ * Shard leaders are the only replicas receiving indexing traffic and pushing
to shared store in a shared collection
+ * so all Solr nodes in a cluster may be sending deletion requests to the
shared storage provider at a given moment.
+ * Collection commands are only processed by the Overseer and therefore only
the Overseer should be deleting entire
+ * collections or shard files from shared storage.
+ *
+ * The BlobDeleteManager maintains two queues to prevent any potential
starvation, one for the incremental indexing
+ * deletion path that is always initiated when a Solr node with shared
collections starts up and one that is only
+ * used when the current node is Overseer and handles Overseer specific
actions.
*/
public class BlobDeleteManager {
private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ /**
+ * Identifier for a BlobDeleteProcessor that runs on all Solr nodes
containing shared collections
+ */
+ public static final String BLOB_FILE_DELETER = "BlobFileDeleter";
+
+ /**
+ * Identifier for a BlobDeleteProcessor that runs on the Overseer if the
Solr cluster contains
+ * any shared collection
+ */
+ public static final String OVERSEER_BLOB_FILE_DELETER =
"OverseerBlobFileDeleter";
+
/**
* Limit to the number of blob files to delete accepted on the delete queue
(and lost in case of server crash). When
* the queue reaches that size, no more deletes are accepted (will be
retried later for a core, next time it is pushed).
* (note that tests in searchserver.blobstore.metadata.CorePushTest trigger
a merge that enqueues more than 100 files to
Review comment:
I don't see CorePushTest anymore. If that is the case, then we should delete
this note. And if some test needs a non-default value, it should override.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
