uschindler commented on code in PR #14843: URL: https://github.com/apache/lucene/pull/14843#discussion_r2194855413
########## lucene/sandbox/src/java21/org/apache/lucene/sandbox/codecs/faiss/LibFaissC.java: ########## @@ -0,0 +1,636 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.codecs.faiss; + +import static java.lang.foreign.ValueLayout.ADDRESS; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_FLOAT; +import static java.lang.foreign.ValueLayout.JAVA_INT; +import static java.lang.foreign.ValueLayout.JAVA_LONG; +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemoryLayout; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SymbolLookup; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.Locale; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.KnnVectorValues; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.hnsw.IntToIntFunction; + +/** + * Utility class to wrap necessary functions of the native <a + * href="https://github.com/facebookresearch/faiss/blob/v1.11.0/c_api/INSTALL.md";>C API</a> of Faiss + * using <a href="https://openjdk.org/projects/panama";>Project Panama</a>. + * + * @lucene.experimental + */ +final class LibFaissC { + // TODO: Use vectorized version where available + public static final String LIBRARY_NAME = "faiss_c"; + public static final String LIBRARY_VERSION = "1.11.0"; + + // See flags defined in c_api/index_io_c.h + static final int FAISS_IO_FLAG_MMAP = 1; + static final int FAISS_IO_FLAG_READ_ONLY = 2; + + private static final int BUFFER_SIZE = 256 * 1024 * 1024; // 256 MB + + // Begin support utilities for JDK21 -- wrappers for functions that were renamed! + private static final MethodHandle GET_STRING_DELEGATE; + + private static String getStringDelegate(MemorySegment segment, long offset) { + return call(GET_STRING_DELEGATE.bindTo(segment), offset); + } + + private static final MethodHandle ALLOCATE_FROM_STRING_DELEGATE; + + private static MemorySegment allocateFromStringDelegate(Arena arena, String string) { + return call(ALLOCATE_FROM_STRING_DELEGATE.bindTo(arena), string); + } + + private static final MethodHandle ALLOCATE_FROM_FLOAT_ARRAY_DELEGATE; + + private static MemorySegment allocateFromFloatArrayDelegate( + Arena arena, ValueLayout.OfFloat ofFloat, float... floats) { + return call(ALLOCATE_FROM_FLOAT_ARRAY_DELEGATE.bindTo(arena), ofFloat, floats); + } + + private static final MethodHandle ALLOCATE_FROM_LONG_ARRAY_DELEGATE; + + private static MemorySegment allocateFromLongArrayDelegate( + Arena arena, ValueLayout.OfLong ofLong, long... longs) { + return call(ALLOCATE_FROM_LONG_ARRAY_DELEGATE.bindTo(arena), ofLong, longs); + } + + private static final MethodHandle ALLOCATE_ARRAY_DELEGATE; + + private static MemorySegment allocateArrayDelegate(Arena arena, MemoryLayout layout, long count) { + return call(ALLOCATE_ARRAY_DELEGATE.bindTo(arena), layout, count); + } + + static { + int runtimeVersion = Runtime.version().feature(); + assert runtimeVersion >= 21; + + String getStringFunctionName; + String allocateFromStringFunctionName; + String allocateFromFloatArrayFunctionName; + String allocateFromLongArrayFunctionName; + String allocateArrayFunctionName; + if (runtimeVersion == 21) { + getStringFunctionName = "getUtf8String"; + allocateFromStringFunctionName = "allocateUtf8String"; + allocateFromFloatArrayFunctionName = "allocateArray"; + allocateFromLongArrayFunctionName = "allocateArray"; + allocateArrayFunctionName = "allocateArray"; + } else { + getStringFunctionName = "getString"; + allocateFromStringFunctionName = "allocateFrom"; + allocateFromFloatArrayFunctionName = "allocateFrom"; + allocateFromLongArrayFunctionName = "allocateFrom"; + allocateArrayFunctionName = "allocate"; + } + + try { + GET_STRING_DELEGATE = + MethodHandles.lookup() + .findVirtual( + MemorySegment.class, + getStringFunctionName, + MethodType.methodType(String.class, long.class)); + + ALLOCATE_FROM_STRING_DELEGATE = + MethodHandles.lookup() + .findVirtual( + Arena.class, + allocateFromStringFunctionName, + MethodType.methodType(MemorySegment.class, String.class)); + + ALLOCATE_FROM_FLOAT_ARRAY_DELEGATE = + MethodHandles.lookup() + .findVirtual( + Arena.class, + allocateFromFloatArrayFunctionName, + MethodType.methodType( + MemorySegment.class, ValueLayout.OfFloat.class, float[].class)); + + ALLOCATE_FROM_LONG_ARRAY_DELEGATE = + MethodHandles.lookup() + .findVirtual( + Arena.class, + allocateFromLongArrayFunctionName, + MethodType.methodType( + MemorySegment.class, ValueLayout.OfLong.class, long[].class)); + + ALLOCATE_ARRAY_DELEGATE = + MethodHandles.lookup() + .findVirtual( + Arena.class, + allocateArrayFunctionName, + MethodType.methodType(MemorySegment.class, MemoryLayout.class, long.class)); + } catch (IllegalAccessException | NoSuchMethodException e) { + throw new RuntimeException(e); Review Comment: I'd prefer to add a jdk-22 extract of all APIs in core package. But then we should find a way to reuse the apijar files in sandbox and not have duplicates. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org For additional commands, e-mail: issues-h...@lucene.apache.org
