[
https://issues.apache.org/jira/browse/LUCENE-10159?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17430492#comment-17430492
]
ASF subversion and git services commented on LUCENE-10159:
----------------------------------------------------------
Commit 8b68bf60c9871ecb200f64c64bf55eb6ac456c0e in lucene's branch
refs/heads/main from Nhat Nguyen
[ https://gitbox.apache.org/repos/asf?p=lucene.git;h=8b68bf6 ]
LUCENE-10159: Fix invalid access in sorted set dv (#389)
We introduced invalid accesses for sorted set doc values in LUCENE-9613.
However, the issue has been unnoticed because the ordinals in doc values
tests aren't complex enough to use high packed bits, and the 3 padding
bytes make these invalid accesses perfectly fine. To reproduce this
issue, we need to use at least 20 bits per value for the ordinals.
> Index corruption: IndexOutOfBoundsException for doc values
> ----------------------------------------------------------
>
> Key: LUCENE-10159
> URL: https://issues.apache.org/jira/browse/LUCENE-10159
> Project: Lucene - Core
> Issue Type: Improvement
> Reporter: Adrien Grand
> Priority: Blocker
> Time Spent: 2h 50m
> Remaining Estimate: 0h
>
> Since we upgraded Elasticsearch to a Lucene 9 snaspshot, we have seen test
> failures with the following stack trace. This looks like an issue with the
> Lucene90 DocValuesFormat.
> {noformat}
> org.apache.lucene.index.MergePolicy$MergeException:
> java.lang.IndexOutOfBoundsException
> at
> org.elasticsearch.index.engine.InternalEngine$EngineMergeScheduler$2.doRun(InternalEngine.java:2340)
> ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT]
> at
> org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:737)
> ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT]
> at
> org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
> ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
> [?:?]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
> [?:?]
> at java.lang.Thread.run(Thread.java:833) [?:?]
> Caused by: java.lang.IndexOutOfBoundsException
> at java.nio.Buffer.checkIndex(Buffer.java:749) ~[?:?]
> at java.nio.DirectByteBuffer.getInt(DirectByteBuffer.java:692) ~[?:?]
> at
> org.apache.lucene.store.ByteBufferGuard.getInt(ByteBufferGuard.java:128)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at
> org.apache.lucene.store.ByteBufferIndexInput$SingleBufferImpl.readInt(ByteBufferIndexInput.java:591)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at
> org.apache.lucene.util.packed.DirectReader$DirectPackedReader20.get(DirectReader.java:222)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at
> org.apache.lucene.util.packed.DirectMonotonicReader.get(DirectMonotonicReader.java:149)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at
> org.apache.lucene.codecs.lucene90.Lucene90DocValuesProducer$24.set(Lucene90DocValuesProducer.java:1356)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at
> org.apache.lucene.codecs.lucene90.Lucene90DocValuesProducer$24.docValueCount(Lucene90DocValuesProducer.java:1348)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at
> org.apache.lucene.codecs.lucene90.Lucene90DocValuesProducer$25.nextDoc(Lucene90DocValuesProducer.java:1405)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at
> org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:837)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at
> org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:148)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at
> org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:154)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at
> org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:168)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at
> org.apache.lucene.index.SegmentMerger.lambda$merge$2(SegmentMerger.java:139)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at
> org.apache.lucene.index.SegmentMerger.mergeWithLogging(SegmentMerger.java:273)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4964)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4500)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at
> org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:6252)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:636)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> at
> org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:113)
> ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT]
> at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:697)
> ~[lucene-core-9.0.0-snapshot-cb366d04d4a.jar:9.0.0-snapshot-cb366d04d4a
> cb366d04d4a611faf90b36e49aa4bd9b48a6e83d - romseygeek - 2021-10-01 09:50:47]
> {noformat}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
