aokolnychyi commented on code in PR #9251:
URL: https://github.com/apache/iceberg/pull/9251#discussion_r1425372419
##########
core/src/main/java/org/apache/iceberg/DeleteFileIndex.java:
##########
@@ -582,93 +513,187 @@ private
Iterable<CloseableIterable<ManifestEntry<DeleteFile>>> deleteManifestRea
}
}
- // a group of indexed delete files sorted by the sequence number they apply
to
- private static class DeleteFileGroup {
- private final long[] seqs;
- private final IndexedDeleteFile[] files;
-
- DeleteFileGroup(IndexedDeleteFile[] files) {
- this.seqs =
Arrays.stream(files).mapToLong(IndexedDeleteFile::applySequenceNumber).toArray();
- this.files = files;
+ private static int findStartIndex(long[] seqs, long seq) {
+ int pos = Arrays.binarySearch(seqs, seq);
+ int start;
+ if (pos < 0) {
+ // the sequence number was not found, where it would be inserted is
-(pos + 1)
+ start = -(pos + 1);
+ } else {
+ // the sequence number was found, but may not be the first
+ // find the first delete file with the given sequence number by
decrementing the position
+ start = pos;
+ while (start > 0 && seqs[start - 1] >= seq) {
+ start -= 1;
+ }
}
- DeleteFileGroup(long[] seqs, IndexedDeleteFile[] files) {
- this.seqs = seqs;
- this.files = files;
+ return start;
+ }
+
+ private static DeleteFile[] concat(DeleteFile[]... deletes) {
+ return ArrayUtil.concat(DeleteFile.class, deletes);
+ }
+
+ // a group of position delete files sorted by the sequence number they apply
to
+ private static class PositionDeletes {
+ private static final Comparator<DeleteFile> SEQ_COMPARATOR =
+ Comparator.comparingLong(DeleteFile::dataSequenceNumber);
+
+ private long[] seqs = null;
+ private DeleteFile[] files = null;
+ private volatile List<DeleteFile> buffer = Lists.newArrayList();
+
+ public void add(DeleteFile file) {
+ buffer.add(file);
}
public DeleteFile[] filter(long seq) {
- int start = findStartIndex(seq);
+ indexIfNeeded();
+
+ int start = findStartIndex(seqs, seq);
if (start >= files.length) {
return NO_DELETES;
}
- DeleteFile[] matchingFiles = new DeleteFile[files.length - start];
-
- for (int index = start; index < files.length; index++) {
- matchingFiles[index - start] = files[index].wrapped();
+ if (start == 0) {
+ return files;
}
+ int matchingFilesCount = files.length - start;
+ DeleteFile[] matchingFiles = new DeleteFile[matchingFilesCount];
+ System.arraycopy(files, start, matchingFiles, 0, matchingFilesCount);
return matchingFiles;
}
- public Stream<IndexedDeleteFile> limit(long seq) {
- int start = findStartIndex(seq);
- return Arrays.stream(files, start, files.length);
+ public Iterable<DeleteFile> referencedDeleteFiles() {
+ indexIfNeeded();
+ return Arrays.asList(files);
}
- private int findStartIndex(long seq) {
- int pos = Arrays.binarySearch(seqs, seq);
- int start;
- if (pos < 0) {
- // the sequence number was not found, where it would be inserted is
-(pos + 1)
- start = -(pos + 1);
- } else {
- // the sequence number was found, but may not be the first
- // find the first delete file with the given sequence number by
decrementing the position
- start = pos;
- while (start > 0 && seqs[start - 1] >= seq) {
- start -= 1;
+ public boolean isEmpty() {
+ indexIfNeeded();
+ return files.length == 0;
+ }
+
+ private void indexIfNeeded() {
+ if (buffer != null) {
+ synchronized (this) {
+ if (buffer != null) {
+ this.files = indexFiles(buffer);
+ this.seqs = indexSeqs(files);
+ this.buffer = null;
+ }
+ }
+ }
+ }
+
+ private static DeleteFile[] indexFiles(List<DeleteFile> list) {
+ DeleteFile[] array = list.toArray(NO_DELETES);
Review Comment:
I wish we could use that but it is not allowed in Java 8. I renamed the var
to `EMPTY_DELETES`. Let me know what you think. I don't want to use a new array
on each call.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
