romseygeek commented on a change in pull request #679:
URL: https://github.com/apache/lucene/pull/679#discussion_r811130545



##########
File path: 
lucene/monitor/src/java/org/apache/lucene/monitor/ReadonlyQueryIndex.java
##########
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.monitor;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
+
+class ReadonlyQueryIndex extends QueryIndex {
+
+  public ReadonlyQueryIndex(MonitorConfiguration configuration) throws 
IOException {
+    if (configuration.getDirectoryProvider() == null) {
+      throw new IllegalStateException(
+          "You must specify a Directory when configuring a Monitor as 
read-only.");
+    }
+    Directory directory = configuration.getDirectoryProvider().get();
+    this.manager = new SearcherManager(directory, new 
TermsHashBuilder(termFilters));
+    this.decomposer = configuration.getQueryDecomposer();
+    this.serializer = configuration.getQuerySerializer();
+    this.populateQueryCache(serializer, decomposer);
+  }
+
+  @Override
+  public void commit(List<MonitorQuery> updates) throws IOException {
+    throw new IllegalStateException("Monitor is readOnly cannot commit");
+  }
+
+  @Override
+  long search(final Query query, QueryCollector matcher) throws IOException {
+    QueryBuilder builder = termFilter -> query;
+    return search(builder, matcher);
+  }
+
+  @Override
+  public long search(QueryBuilder queryBuilder, QueryCollector matcher) throws 
IOException {
+    IndexSearcher searcher = null;
+    try {
+      searcher = manager.acquire();
+      return searchInMemory(queryBuilder, matcher, searcher, this.queries);
+    } finally {
+      if (searcher != null) {
+        manager.release(searcher);
+      }
+    }
+  }
+
+  @Override
+  void purgeCache(CachePopulator populator) throws IOException {
+    final ConcurrentMap<String, QueryCacheEntry> newCache = new 
ConcurrentHashMap<>();

Review comment:
       I don't think this needs to be a concurrent hash map in this case, as 
we're not updating it elsewhere.

##########
File path: 
lucene/monitor/src/java/org/apache/lucene/monitor/ReadonlyQueryIndex.java
##########
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.monitor;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
+
+class ReadonlyQueryIndex extends QueryIndex {
+
+  public ReadonlyQueryIndex(MonitorConfiguration configuration) throws 
IOException {
+    if (configuration.getDirectoryProvider() == null) {
+      throw new IllegalStateException(
+          "You must specify a Directory when configuring a Monitor as 
read-only.");
+    }
+    Directory directory = configuration.getDirectoryProvider().get();
+    this.manager = new SearcherManager(directory, new 
TermsHashBuilder(termFilters));
+    this.decomposer = configuration.getQueryDecomposer();
+    this.serializer = configuration.getQuerySerializer();
+    this.populateQueryCache(serializer, decomposer);

Review comment:
       So this will still only repopulate the cache when you explicitly call 
`purgeCache()` on the parent Monitor I think?  That's probably OK but we should 
document that clearly.

##########
File path: 
lucene/monitor/src/test/org/apache/lucene/monitor/TestMonitorReadonly.java
##########
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.monitor;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Collections;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexNotFoundException;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.FSDirectory;
+import org.junit.Test;
+
+public class TestMonitorReadonly extends MonitorTestBase {
+  private static final Analyzer ANALYZER = new WhitespaceAnalyzer();
+
+  @Test
+  public void testReadonlyMonitorThrowsOnInexistentIndex() {
+    Path indexDirectory = createTempDir();
+    MonitorConfiguration config =
+        new MonitorConfiguration()
+            .setDirectoryProvider(
+                () -> FSDirectory.open(indexDirectory),
+                MonitorQuerySerializer.fromParser(MonitorTestBase::parse),
+                true);
+    assertThrows(
+        IndexNotFoundException.class,
+        () -> {
+          new Monitor(ANALYZER, config);
+        });
+  }
+
+  @Test
+  public void testReadonlyMonitorThrowsWhenCallingWriteRequests() throws 
IOException {
+    Path indexDirectory = createTempDir();
+    MonitorConfiguration writeConfig =
+        new MonitorConfiguration()
+            .setIndexPath(
+                indexDirectory, 
MonitorQuerySerializer.fromParser(MonitorTestBase::parse));
+
+    // this will create the index
+    Monitor writeMonitor = new Monitor(ANALYZER, writeConfig);
+    writeMonitor.close();
+
+    MonitorConfiguration config =
+        new MonitorConfiguration()
+            .setDirectoryProvider(
+                () -> FSDirectory.open(indexDirectory),
+                MonitorQuerySerializer.fromParser(MonitorTestBase::parse),
+                true);
+    try (Monitor monitor = new Monitor(ANALYZER, config)) {
+      assertThrows(
+          IllegalStateException.class,
+          () -> {
+            TermQuery query = new TermQuery(new Term(FIELD, "test"));
+            monitor.register(
+                new MonitorQuery("query1", query, query.toString(), 
Collections.emptyMap()));
+          });
+
+      assertThrows(
+          IllegalStateException.class,
+          () -> {
+            monitor.deleteById("query1");
+          });
+
+      assertThrows(
+          IllegalStateException.class,
+          () -> {
+            monitor.clear();
+          });
+    }
+  }
+
+  @Test
+  public void testSettingCustomDirectory() throws IOException {
+    Path indexDirectory = createTempDir();
+    Document doc = new Document();
+    doc.add(newTextField(FIELD, "This is a Foobar test document", 
Field.Store.NO));
+
+    MonitorConfiguration writeConfig =
+        new MonitorConfiguration()
+            .setDirectoryProvider(
+                () -> FSDirectory.open(indexDirectory),
+                MonitorQuerySerializer.fromParser(MonitorTestBase::parse));
+
+    try (Monitor writeMonitor = new Monitor(ANALYZER, writeConfig)) {
+      TermQuery query = new TermQuery(new Term(FIELD, "test"));
+      writeMonitor.register(
+          new MonitorQuery("query1", query, query.toString(), 
Collections.emptyMap()));
+      TermQuery query2 = new TermQuery(new Term(FIELD, "Foobar"));
+      writeMonitor.register(
+          new MonitorQuery("query2", query2, query.toString(), 
Collections.emptyMap()));
+      MatchingQueries<QueryMatch> matches = writeMonitor.match(doc, 
QueryMatch.SIMPLE_MATCHER);
+      assertNotNull(matches.getMatches());
+      assertEquals(2, matches.getMatchCount());
+      assertNotNull(matches.matches("query2"));
+    }
+  }
+
+  public void testMonitorReadOnlyCouldReadOnTheSameIndex() throws IOException {
+    Path indexDirectory = createTempDir();
+    Document doc = new Document();
+    doc.add(newTextField(FIELD, "This is a test document", Field.Store.NO));
+
+    MonitorConfiguration writeConfig =
+        new MonitorConfiguration()
+            .setDirectoryProvider(
+                () -> FSDirectory.open(indexDirectory),
+                MonitorQuerySerializer.fromParser(MonitorTestBase::parse));
+
+    try (Monitor writeMonitor = new Monitor(ANALYZER, writeConfig)) {
+      TermQuery query = new TermQuery(new Term(FIELD, "test"));
+      writeMonitor.register(
+          new MonitorQuery("query1", query, query.toString(), 
Collections.emptyMap()));
+    }
+
+    MonitorConfiguration readConfig =
+        new MonitorConfiguration()
+            .setDirectoryProvider(
+                () -> FSDirectory.open(indexDirectory),
+                MonitorQuerySerializer.fromParser(MonitorTestBase::parse),
+                true);
+
+    try (Monitor readMonitor1 = new Monitor(ANALYZER, readConfig)) {
+      MatchingQueries<QueryMatch> matches = readMonitor1.match(doc, 
QueryMatch.SIMPLE_MATCHER);
+      assertNotNull(matches.getMatches());
+      assertEquals(1, matches.getMatchCount());
+      assertNotNull(matches.matches("query1"));
+    }
+
+    try (Monitor readMonitor2 = new Monitor(ANALYZER, readConfig)) {
+      MatchingQueries<QueryMatch> matches = readMonitor2.match(doc, 
QueryMatch.SIMPLE_MATCHER);
+      assertNotNull(matches.getMatches());
+      assertEquals(1, matches.getMatchCount());
+      assertNotNull(matches.matches("query1"));
+
+      assertThrows(
+          IllegalStateException.class,
+          () -> {
+            TermQuery query = new TermQuery(new Term(FIELD, "test"));
+            readMonitor2.register(
+                new MonitorQuery("query1", query, query.toString(), 
Collections.emptyMap()));
+          });
+    }
+  }

Review comment:
       Can you add a test for updating the write monitor? We should see no 
change in the read monitor until we call purgeCache.

##########
File path: 
lucene/monitor/src/java/org/apache/lucene/monitor/WritableQueryIndex.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.monitor;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.document.*;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.NamedThreadFactory;
+
+class WritableQueryIndex extends QueryIndex {
+
+  private final IndexWriter writer;
+  private final Presearcher presearcher;
+
+  /* Used to cache updates while a purge is ongoing */
+  private volatile Map<String, QueryCacheEntry> purgeCache = null;
+
+  /* Used to lock around the creation of the purgeCache */
+  private final ReadWriteLock purgeLock = new ReentrantReadWriteLock();
+  private final Object commitLock = new Object();
+
+  private final ScheduledExecutorService purgeExecutor;
+  private final List<MonitorUpdateListener> listeners = new ArrayList<>();
+
+  WritableQueryIndex(MonitorConfiguration configuration, Presearcher 
presearcher)
+      throws IOException {
+
+    this.writer = configuration.buildIndexWriter();
+    this.manager = new SearcherManager(writer, true, true, new 
TermsHashBuilder(termFilters));
+    this.decomposer = configuration.getQueryDecomposer();
+    this.serializer = configuration.getQuerySerializer();
+    this.presearcher = presearcher;
+    populateQueryCache(serializer, decomposer);
+
+    long purgeFrequency = configuration.getPurgeFrequency();
+    this.purgeExecutor =
+        Executors.newSingleThreadScheduledExecutor(new 
NamedThreadFactory("cache-purge"));
+    this.purgeExecutor.scheduleAtFixedRate(
+        () -> {
+          try {
+            purgeCache();
+          } catch (Throwable e) {
+            listeners.forEach(l -> l.onPurgeError(e));
+          }
+        },
+        purgeFrequency,
+        purgeFrequency,
+        configuration.getPurgeFrequencyUnits());
+  }
+
+  @Override
+  public void addListener(MonitorUpdateListener listener) {
+    listeners.add(listener);
+  }
+
+  @Override
+  public void commit(List<MonitorQuery> updates) throws IOException {
+    commitWithoutNotify(updates);
+    listeners.forEach(l -> l.afterUpdate(updates));
+  }
+
+  private void commitWithoutNotify(List<MonitorQuery> updates) throws 
IOException {
+    List<Indexable> indexables = buildIndexables(updates);
+    synchronized (commitLock) {
+      purgeLock.readLock().lock();
+      try {
+        if (indexables.size() > 0) {
+          Set<String> ids = new HashSet<>();
+          for (Indexable update : indexables) {
+            ids.add(update.queryCacheEntry.queryId);
+          }
+          for (String id : ids) {
+            writer.deleteDocuments(new Term(FIELDS.query_id, id));
+          }
+          for (Indexable update : indexables) {
+            this.queries.put(update.queryCacheEntry.cacheId, 
update.queryCacheEntry);
+            writer.addDocument(update.document);
+            if (purgeCache != null)
+              purgeCache.put(update.queryCacheEntry.cacheId, 
update.queryCacheEntry);
+          }
+        }
+        writer.commit();
+        manager.maybeRefresh();
+      } finally {
+        purgeLock.readLock().unlock();
+      }
+    }
+  }
+
+  private static class Indexable {
+    final QueryCacheEntry queryCacheEntry;
+    final Document document;
+
+    private Indexable(QueryCacheEntry queryCacheEntry, Document document) {
+      this.queryCacheEntry = queryCacheEntry;
+      this.document = document;
+    }
+  }
+
+  private static final BytesRef EMPTY = new BytesRef();
+
+  private List<Indexable> buildIndexables(List<MonitorQuery> updates) {
+    List<Indexable> indexables = new ArrayList<>();
+    for (MonitorQuery mq : updates) {
+      if (serializer != null && mq.getQueryString() == null) {
+        throw new IllegalArgumentException(
+            "Cannot add a MonitorQuery with a null string representation to a 
non-ephemeral Monitor");
+      }
+      BytesRef serialized = serializer == null ? EMPTY : 
serializer.serialize(mq);
+      for (QueryCacheEntry qce : QueryCacheEntry.decompose(mq, decomposer)) {
+        Document doc = presearcher.indexQuery(qce.matchQuery, 
mq.getMetadata());
+        doc.add(new StringField(FIELDS.query_id, qce.queryId, Field.Store.NO));
+        doc.add(new SortedDocValuesField(FIELDS.cache_id, new 
BytesRef(qce.cacheId)));
+        doc.add(new SortedDocValuesField(FIELDS.query_id, new 
BytesRef(qce.queryId)));
+        doc.add(new BinaryDocValuesField(FIELDS.mq, serialized));
+        indexables.add(new Indexable(qce, doc));
+      }
+    }
+    return indexables;
+  }
+
+  @Override
+  public long search(QueryBuilder queryBuilder, QueryCollector matcher) throws 
IOException {
+    IndexSearcher searcher = null;
+    try {
+      Map<String, QueryCacheEntry> queries;
+
+      purgeLock.readLock().lock();
+      try {
+        searcher = manager.acquire();
+        queries = this.queries;
+      } finally {
+        purgeLock.readLock().unlock();
+      }
+
+      return searchInMemory(queryBuilder, matcher, searcher, queries);
+    } finally {
+      if (searcher != null) {
+        manager.release(searcher);
+      }
+    }
+  }
+
+  @Override
+  public void purgeCache() throws IOException {
+    super.purgeCache();
+    listeners.forEach(MonitorUpdateListener::onPurge);
+  }
+
+  @Override
+  /**
+   * Remove unused queries from the query cache.
+   *
+   * <p>This is normally called from a background thread at a rate set by 
configurePurgeFrequency().
+   *
+   * @throws IOException on IO errors
+   */
+  synchronized void purgeCache(CachePopulator populator) throws IOException {
+
+    // Note on implementation
+
+    // The purge works by scanning the query index and creating a new query 
cache populated
+    // for each query in the index.  When the scan is complete, the old query 
cache is swapped
+    // for the new, allowing it to be garbage-collected.
+
+    // In order to not drop cached queries that have been added while a purge 
is ongoing,
+    // we use a ReadWriteLock to guard the creation and removal of an register 
log.  Commits take
+    // the read lock.  If the register log has been created, then a purge is 
ongoing, and queries
+    // are added to the register log within the read lock guard.
+
+    // The purge takes the write lock when creating the register log, and then 
when swapping out
+    // the old query cache.  Within the second write lock guard, the contents 
of the register log
+    // are added to the new query cache, and the register log itself is 
removed.
+
+    final ConcurrentMap<String, QueryCacheEntry> newCache = new 
ConcurrentHashMap<>();
+
+    purgeLock.writeLock().lock();
+    try {
+      purgeCache = new ConcurrentHashMap<>();
+    } finally {
+      purgeLock.writeLock().unlock();
+    }
+
+    populator.populateCacheWithIndex(newCache);
+
+    purgeLock.writeLock().lock();
+    try {
+      newCache.putAll(purgeCache);
+      purgeCache = null;
+      queries = newCache;
+    } finally {
+      purgeLock.writeLock().unlock();
+    }
+  }
+
+  // ---------------------------------------------
+  //  Proxy trivial operations...
+  // ---------------------------------------------
+
+  @Override
+  public void close() throws IOException {
+    IOUtils.close(manager, writer, writer.getDirectory());
+    purgeExecutor.shutdown();

Review comment:
       Let's close the executor first to ensure we don't get spurious 
exceptions from a cache refresh operating on a closed directory.

##########
File path: 
lucene/monitor/src/test/org/apache/lucene/monitor/TestCachePurging.java
##########
@@ -155,7 +155,12 @@ public void testBackgroundPurges() throws IOException, 
InterruptedException {
             @Override
             public void onPurge() {
               // It can sometimes take a couple of purge runs to get 
everything in sync
-              if (monitor.getQueryCacheStats().cachedQueries == 99) 
latch.countDown();
+              try {
+                if (monitor.getQueryCacheStats().cachedQueries == 99) 
latch.countDown();
+              } catch (IOException e) {
+                // Ignore
+                System.out.println(e);

Review comment:
       Instead of a println can you rethrow this as a RuntimeException?




-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]



---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to