chenboat commented on code in PR #11739:
URL: https://github.com/apache/pinot/pull/11739#discussion_r1355488928
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/json/ImmutableJsonIndexReader.java:
##########
@@ -300,6 +305,92 @@ private int getDocId(int flattenedDocId) {
return _docIdMapping.getInt((long) flattenedDocId << 2);
}
+ /**
+ * ImmutableJsonIndexReaderContext holds a cache that is used to accelerate
following reads. The cache is specific
+ * to a key, and therefore should NOT be reused for other keys. For each
key, ImmutableJsonIndexReader.createContext()
+ * should be invoked to create a fresh context.
+ */
+ public static class ImmutableJsonIndexReaderContext implements
JsonIndexReaderContext {
+ private final Int2ObjectOpenHashMap<RoaringBitmap> _cache;
+
+ public ImmutableJsonIndexReaderContext() {
+ _cache = new Int2ObjectOpenHashMap<>();
+ }
+
+ public Int2ObjectOpenHashMap<RoaringBitmap> getCache() {
+ return _cache;
+ }
+ }
+
+ @Override
+ public ImmutableJsonIndexReaderContext createContext() {
+ return new ImmutableJsonIndexReaderContext();
+ }
+
+ @Override
+ public String[] getValuesForKeyAndDocs(String key, int[] docIds,
JsonIndexReaderContext context) {
+ ImmutableJsonIndexReaderContext immutableContext =
(ImmutableJsonIndexReaderContext) context;
+ Int2ObjectOpenHashMap<RoaringBitmap> cache = immutableContext.getCache();
+
+ RoaringBitmap docIdMask = RoaringBitmap.bitmapOf(docIds);
+ int[] dictIds = getDictIdsForKey(key);
+ Int2ObjectOpenHashMap<String> docIdToValues = new
Int2ObjectOpenHashMap<>(docIds.length);
+
+ if (cache.isEmpty()) {
+ for (int dictId = dictIds[0]; dictId < dictIds[1]; dictId++) {
+ // get docIds from posting list, convert these to the actual docIds
+ ImmutableRoaringBitmap flattenedDocIds =
_invertedIndex.getDocIds(dictId);
+ PeekableIntIterator it = flattenedDocIds.getIntIterator();
+ MutableRoaringBitmap realDocIds = new MutableRoaringBitmap();
+ while (it.hasNext()) {
+ realDocIds.add(getDocId(it.next()));
+ }
+ cache.put(dictId, realDocIds.toRoaringBitmap());
+ }
+ }
+
+ for (int dictId = dictIds[0]; dictId < dictIds[1]; dictId++) {
+ RoaringBitmap intersection = RoaringBitmap.and(cache.get(dictId),
docIdMask);
+ if (intersection.isEmpty()) {
+ continue;
+ }
+ // dictionary value lookup, stripping the path prefix
+ String val = _dictionary.getStringValue(dictId).substring(key.length() +
1);
+ for (int docId : intersection) {
+ docIdToValues.put(docId, val);
+ }
+ }
+
+ String[] values = new String[docIds.length];
+ for (int i = 0; i < docIds.length; i++) {
+ values[i] = docIdToValues.get(docIds[i]);
+ }
+ return values;
+ }
+
+ /**
+ * For a JSON key path, returns an int array of [min, max] of all values of
the JSON key path
Review Comment:
should the return values be the dictionary ids rather than "values"?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
