bruno-roustant commented on a change in pull request #1301: LUCENE-9254:
UniformSplit supports FST off-heap.
URL: https://github.com/apache/lucene-solr/pull/1301#discussion_r385820065
##########
File path:
lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitTermsReader.java
##########
@@ -122,12 +159,51 @@ protected UniformSplitTermsReader(PostingsReaderBase
postingsReader, SegmentRead
}
}
- protected void fillFieldMap(PostingsReaderBase postingsReader, BlockDecoder
blockDecoder,
- IndexInput dictionaryInput, IndexInput blockInput,
- Collection<FieldMetadata> fieldMetadataCollection,
FieldInfos fieldInfos) throws IOException {
+ protected void fillFieldMap(PostingsReaderBase postingsReader,
SegmentReadState state, BlockDecoder blockDecoder,
+ DictionaryLoadMode defaultLoadMode, IndexInput
dictionaryInput, IndexInput blockInput,
+ Collection<FieldMetadata>
fieldMetadataCollection, FieldInfos fieldInfos) throws IOException {
+ DictionaryLoadMode loadMode = getLoadMode(state.readerAttributes,
DICTIONARY_LOAD_MODE_KEY, defaultLoadMode);
for (FieldMetadata fieldMetadata : fieldMetadataCollection) {
+ DictionaryLoadMode fieldLoadMode = getLoadMode(state.readerAttributes,
DICTIONARY_LOAD_MODE_KEY + "." + fieldMetadata.getFieldInfo().name, loadMode);
+ boolean isDictionaryOffHeap = isDictionaryOffHeap(state, fieldLoadMode,
fieldMetadata, dictionaryInput);
+ IndexDictionary.BrowserSupplier dictionaryBrowserSupplier =
createDictionaryBrowserSupplier(dictionaryInput, fieldMetadata, blockDecoder,
isDictionaryOffHeap);
fieldToTermsMap.put(fieldMetadata.getFieldInfo().name,
- new UniformSplitTerms(dictionaryInput, blockInput, fieldMetadata,
postingsReader, blockDecoder));
+ new UniformSplitTerms(dictionaryInput, blockInput, fieldMetadata,
postingsReader, blockDecoder, dictionaryBrowserSupplier));
+ }
+ }
+
+ protected IndexDictionary.BrowserSupplier
createDictionaryBrowserSupplier(IndexInput dictionaryInput, FieldMetadata
fieldMetadata,
+
BlockDecoder blockDecoder, boolean isDictionaryOffHeap) throws IOException {
+ return new FSTDictionary.BrowserSupplier(dictionaryInput,
fieldMetadata.getDictionaryStartFP(), blockDecoder, isDictionaryOffHeap);
+ }
+
+ protected DictionaryLoadMode getLoadMode(Map<String, String> attributes,
String key, DictionaryLoadMode defaultLoadMode) {
+ String value = attributes.get(key);
+ if (value == null) {
+ return defaultLoadMode;
+ }
+ try {
+ return DictionaryLoadMode.valueOf(value);
+ } catch (IllegalArgumentException ex) {
+ throw new IllegalArgumentException("Invalid value for " + key + "
expected one of: "
+ + Arrays.toString(DictionaryLoadMode.values()) + " but was: " +
value, ex);
+ }
+ }
+
+ protected boolean isDictionaryOffHeap(SegmentReadState state,
DictionaryLoadMode loadMode, FieldMetadata fieldMetadata,
Review comment:
Same logic as in BlockTree FieldReader.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
