romseygeek commented on a change in pull request #679: URL: https://github.com/apache/lucene/pull/679#discussion_r811130545
########## File path: lucene/monitor/src/java/org/apache/lucene/monitor/ReadonlyQueryIndex.java ########## @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.monitor; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SearcherManager; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IOUtils; + +class ReadonlyQueryIndex extends QueryIndex { + + public ReadonlyQueryIndex(MonitorConfiguration configuration) throws IOException { + if (configuration.getDirectoryProvider() == null) { + throw new IllegalStateException( + "You must specify a Directory when configuring a Monitor as read-only."); + } + Directory directory = configuration.getDirectoryProvider().get(); + this.manager = new SearcherManager(directory, new TermsHashBuilder(termFilters)); + this.decomposer = configuration.getQueryDecomposer(); + this.serializer = configuration.getQuerySerializer(); + this.populateQueryCache(serializer, decomposer); + } + + @Override + public void commit(List<MonitorQuery> updates) throws IOException { + throw new IllegalStateException("Monitor is readOnly cannot commit"); + } + + @Override + long search(final Query query, QueryCollector matcher) throws IOException { + QueryBuilder builder = termFilter -> query; + return search(builder, matcher); + } + + @Override + public long search(QueryBuilder queryBuilder, QueryCollector matcher) throws IOException { + IndexSearcher searcher = null; + try { + searcher = manager.acquire(); + return searchInMemory(queryBuilder, matcher, searcher, this.queries); + } finally { + if (searcher != null) { + manager.release(searcher); + } + } + } + + @Override + void purgeCache(CachePopulator populator) throws IOException { + final ConcurrentMap<String, QueryCacheEntry> newCache = new ConcurrentHashMap<>(); Review comment: I don't think this needs to be a concurrent hash map in this case, as we're not updating it elsewhere. ########## File path: lucene/monitor/src/java/org/apache/lucene/monitor/ReadonlyQueryIndex.java ########## @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.monitor; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SearcherManager; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IOUtils; + +class ReadonlyQueryIndex extends QueryIndex { + + public ReadonlyQueryIndex(MonitorConfiguration configuration) throws IOException { + if (configuration.getDirectoryProvider() == null) { + throw new IllegalStateException( + "You must specify a Directory when configuring a Monitor as read-only."); + } + Directory directory = configuration.getDirectoryProvider().get(); + this.manager = new SearcherManager(directory, new TermsHashBuilder(termFilters)); + this.decomposer = configuration.getQueryDecomposer(); + this.serializer = configuration.getQuerySerializer(); + this.populateQueryCache(serializer, decomposer); Review comment: So this will still only repopulate the cache when you explicitly call `purgeCache()` on the parent Monitor I think? That's probably OK but we should document that clearly. ########## File path: lucene/monitor/src/test/org/apache/lucene/monitor/TestMonitorReadonly.java ########## @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.monitor; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collections; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexNotFoundException; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.FSDirectory; +import org.junit.Test; + +public class TestMonitorReadonly extends MonitorTestBase { + private static final Analyzer ANALYZER = new WhitespaceAnalyzer(); + + @Test + public void testReadonlyMonitorThrowsOnInexistentIndex() { + Path indexDirectory = createTempDir(); + MonitorConfiguration config = + new MonitorConfiguration() + .setDirectoryProvider( + () -> FSDirectory.open(indexDirectory), + MonitorQuerySerializer.fromParser(MonitorTestBase::parse), + true); + assertThrows( + IndexNotFoundException.class, + () -> { + new Monitor(ANALYZER, config); + }); + } + + @Test + public void testReadonlyMonitorThrowsWhenCallingWriteRequests() throws IOException { + Path indexDirectory = createTempDir(); + MonitorConfiguration writeConfig = + new MonitorConfiguration() + .setIndexPath( + indexDirectory, MonitorQuerySerializer.fromParser(MonitorTestBase::parse)); + + // this will create the index + Monitor writeMonitor = new Monitor(ANALYZER, writeConfig); + writeMonitor.close(); + + MonitorConfiguration config = + new MonitorConfiguration() + .setDirectoryProvider( + () -> FSDirectory.open(indexDirectory), + MonitorQuerySerializer.fromParser(MonitorTestBase::parse), + true); + try (Monitor monitor = new Monitor(ANALYZER, config)) { + assertThrows( + IllegalStateException.class, + () -> { + TermQuery query = new TermQuery(new Term(FIELD, "test")); + monitor.register( + new MonitorQuery("query1", query, query.toString(), Collections.emptyMap())); + }); + + assertThrows( + IllegalStateException.class, + () -> { + monitor.deleteById("query1"); + }); + + assertThrows( + IllegalStateException.class, + () -> { + monitor.clear(); + }); + } + } + + @Test + public void testSettingCustomDirectory() throws IOException { + Path indexDirectory = createTempDir(); + Document doc = new Document(); + doc.add(newTextField(FIELD, "This is a Foobar test document", Field.Store.NO)); + + MonitorConfiguration writeConfig = + new MonitorConfiguration() + .setDirectoryProvider( + () -> FSDirectory.open(indexDirectory), + MonitorQuerySerializer.fromParser(MonitorTestBase::parse)); + + try (Monitor writeMonitor = new Monitor(ANALYZER, writeConfig)) { + TermQuery query = new TermQuery(new Term(FIELD, "test")); + writeMonitor.register( + new MonitorQuery("query1", query, query.toString(), Collections.emptyMap())); + TermQuery query2 = new TermQuery(new Term(FIELD, "Foobar")); + writeMonitor.register( + new MonitorQuery("query2", query2, query.toString(), Collections.emptyMap())); + MatchingQueries<QueryMatch> matches = writeMonitor.match(doc, QueryMatch.SIMPLE_MATCHER); + assertNotNull(matches.getMatches()); + assertEquals(2, matches.getMatchCount()); + assertNotNull(matches.matches("query2")); + } + } + + public void testMonitorReadOnlyCouldReadOnTheSameIndex() throws IOException { + Path indexDirectory = createTempDir(); + Document doc = new Document(); + doc.add(newTextField(FIELD, "This is a test document", Field.Store.NO)); + + MonitorConfiguration writeConfig = + new MonitorConfiguration() + .setDirectoryProvider( + () -> FSDirectory.open(indexDirectory), + MonitorQuerySerializer.fromParser(MonitorTestBase::parse)); + + try (Monitor writeMonitor = new Monitor(ANALYZER, writeConfig)) { + TermQuery query = new TermQuery(new Term(FIELD, "test")); + writeMonitor.register( + new MonitorQuery("query1", query, query.toString(), Collections.emptyMap())); + } + + MonitorConfiguration readConfig = + new MonitorConfiguration() + .setDirectoryProvider( + () -> FSDirectory.open(indexDirectory), + MonitorQuerySerializer.fromParser(MonitorTestBase::parse), + true); + + try (Monitor readMonitor1 = new Monitor(ANALYZER, readConfig)) { + MatchingQueries<QueryMatch> matches = readMonitor1.match(doc, QueryMatch.SIMPLE_MATCHER); + assertNotNull(matches.getMatches()); + assertEquals(1, matches.getMatchCount()); + assertNotNull(matches.matches("query1")); + } + + try (Monitor readMonitor2 = new Monitor(ANALYZER, readConfig)) { + MatchingQueries<QueryMatch> matches = readMonitor2.match(doc, QueryMatch.SIMPLE_MATCHER); + assertNotNull(matches.getMatches()); + assertEquals(1, matches.getMatchCount()); + assertNotNull(matches.matches("query1")); + + assertThrows( + IllegalStateException.class, + () -> { + TermQuery query = new TermQuery(new Term(FIELD, "test")); + readMonitor2.register( + new MonitorQuery("query1", query, query.toString(), Collections.emptyMap())); + }); + } + } Review comment: Can you add a test for updating the write monitor? We should see no change in the read monitor until we call purgeCache. ########## File path: lucene/monitor/src/java/org/apache/lucene/monitor/WritableQueryIndex.java ########## @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.monitor; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.lucene.document.*; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.SearcherManager; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.NamedThreadFactory; + +class WritableQueryIndex extends QueryIndex { + + private final IndexWriter writer; + private final Presearcher presearcher; + + /* Used to cache updates while a purge is ongoing */ + private volatile Map<String, QueryCacheEntry> purgeCache = null; + + /* Used to lock around the creation of the purgeCache */ + private final ReadWriteLock purgeLock = new ReentrantReadWriteLock(); + private final Object commitLock = new Object(); + + private final ScheduledExecutorService purgeExecutor; + private final List<MonitorUpdateListener> listeners = new ArrayList<>(); + + WritableQueryIndex(MonitorConfiguration configuration, Presearcher presearcher) + throws IOException { + + this.writer = configuration.buildIndexWriter(); + this.manager = new SearcherManager(writer, true, true, new TermsHashBuilder(termFilters)); + this.decomposer = configuration.getQueryDecomposer(); + this.serializer = configuration.getQuerySerializer(); + this.presearcher = presearcher; + populateQueryCache(serializer, decomposer); + + long purgeFrequency = configuration.getPurgeFrequency(); + this.purgeExecutor = + Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("cache-purge")); + this.purgeExecutor.scheduleAtFixedRate( + () -> { + try { + purgeCache(); + } catch (Throwable e) { + listeners.forEach(l -> l.onPurgeError(e)); + } + }, + purgeFrequency, + purgeFrequency, + configuration.getPurgeFrequencyUnits()); + } + + @Override + public void addListener(MonitorUpdateListener listener) { + listeners.add(listener); + } + + @Override + public void commit(List<MonitorQuery> updates) throws IOException { + commitWithoutNotify(updates); + listeners.forEach(l -> l.afterUpdate(updates)); + } + + private void commitWithoutNotify(List<MonitorQuery> updates) throws IOException { + List<Indexable> indexables = buildIndexables(updates); + synchronized (commitLock) { + purgeLock.readLock().lock(); + try { + if (indexables.size() > 0) { + Set<String> ids = new HashSet<>(); + for (Indexable update : indexables) { + ids.add(update.queryCacheEntry.queryId); + } + for (String id : ids) { + writer.deleteDocuments(new Term(FIELDS.query_id, id)); + } + for (Indexable update : indexables) { + this.queries.put(update.queryCacheEntry.cacheId, update.queryCacheEntry); + writer.addDocument(update.document); + if (purgeCache != null) + purgeCache.put(update.queryCacheEntry.cacheId, update.queryCacheEntry); + } + } + writer.commit(); + manager.maybeRefresh(); + } finally { + purgeLock.readLock().unlock(); + } + } + } + + private static class Indexable { + final QueryCacheEntry queryCacheEntry; + final Document document; + + private Indexable(QueryCacheEntry queryCacheEntry, Document document) { + this.queryCacheEntry = queryCacheEntry; + this.document = document; + } + } + + private static final BytesRef EMPTY = new BytesRef(); + + private List<Indexable> buildIndexables(List<MonitorQuery> updates) { + List<Indexable> indexables = new ArrayList<>(); + for (MonitorQuery mq : updates) { + if (serializer != null && mq.getQueryString() == null) { + throw new IllegalArgumentException( + "Cannot add a MonitorQuery with a null string representation to a non-ephemeral Monitor"); + } + BytesRef serialized = serializer == null ? EMPTY : serializer.serialize(mq); + for (QueryCacheEntry qce : QueryCacheEntry.decompose(mq, decomposer)) { + Document doc = presearcher.indexQuery(qce.matchQuery, mq.getMetadata()); + doc.add(new StringField(FIELDS.query_id, qce.queryId, Field.Store.NO)); + doc.add(new SortedDocValuesField(FIELDS.cache_id, new BytesRef(qce.cacheId))); + doc.add(new SortedDocValuesField(FIELDS.query_id, new BytesRef(qce.queryId))); + doc.add(new BinaryDocValuesField(FIELDS.mq, serialized)); + indexables.add(new Indexable(qce, doc)); + } + } + return indexables; + } + + @Override + public long search(QueryBuilder queryBuilder, QueryCollector matcher) throws IOException { + IndexSearcher searcher = null; + try { + Map<String, QueryCacheEntry> queries; + + purgeLock.readLock().lock(); + try { + searcher = manager.acquire(); + queries = this.queries; + } finally { + purgeLock.readLock().unlock(); + } + + return searchInMemory(queryBuilder, matcher, searcher, queries); + } finally { + if (searcher != null) { + manager.release(searcher); + } + } + } + + @Override + public void purgeCache() throws IOException { + super.purgeCache(); + listeners.forEach(MonitorUpdateListener::onPurge); + } + + @Override + /** + * Remove unused queries from the query cache. + * + * <p>This is normally called from a background thread at a rate set by configurePurgeFrequency(). + * + * @throws IOException on IO errors + */ + synchronized void purgeCache(CachePopulator populator) throws IOException { + + // Note on implementation + + // The purge works by scanning the query index and creating a new query cache populated + // for each query in the index. When the scan is complete, the old query cache is swapped + // for the new, allowing it to be garbage-collected. + + // In order to not drop cached queries that have been added while a purge is ongoing, + // we use a ReadWriteLock to guard the creation and removal of an register log. Commits take + // the read lock. If the register log has been created, then a purge is ongoing, and queries + // are added to the register log within the read lock guard. + + // The purge takes the write lock when creating the register log, and then when swapping out + // the old query cache. Within the second write lock guard, the contents of the register log + // are added to the new query cache, and the register log itself is removed. + + final ConcurrentMap<String, QueryCacheEntry> newCache = new ConcurrentHashMap<>(); + + purgeLock.writeLock().lock(); + try { + purgeCache = new ConcurrentHashMap<>(); + } finally { + purgeLock.writeLock().unlock(); + } + + populator.populateCacheWithIndex(newCache); + + purgeLock.writeLock().lock(); + try { + newCache.putAll(purgeCache); + purgeCache = null; + queries = newCache; + } finally { + purgeLock.writeLock().unlock(); + } + } + + // --------------------------------------------- + // Proxy trivial operations... + // --------------------------------------------- + + @Override + public void close() throws IOException { + IOUtils.close(manager, writer, writer.getDirectory()); + purgeExecutor.shutdown(); Review comment: Let's close the executor first to ensure we don't get spurious exceptions from a cache refresh operating on a closed directory. ########## File path: lucene/monitor/src/test/org/apache/lucene/monitor/TestCachePurging.java ########## @@ -155,7 +155,12 @@ public void testBackgroundPurges() throws IOException, InterruptedException { @Override public void onPurge() { // It can sometimes take a couple of purge runs to get everything in sync - if (monitor.getQueryCacheStats().cachedQueries == 99) latch.countDown(); + try { + if (monitor.getQueryCacheStats().cachedQueries == 99) latch.countDown(); + } catch (IOException e) { + // Ignore + System.out.println(e); Review comment: Instead of a println can you rethrow this as a RuntimeException? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
