jackluo923 commented on code in PR #14241: URL: https://github.com/apache/pinot/pull/14241#discussion_r1802042000
########## pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/CLPMutableForwardIndexV2.java: ########## @@ -0,0 +1,468 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.segment.local.realtime.impl.forward; + +import com.yscope.clp.compressorfrontend.BuiltInVariableHandlingRuleVersions; +import com.yscope.clp.compressorfrontend.EmptyArrayUtils; +import com.yscope.clp.compressorfrontend.EncodedMessage; +import com.yscope.clp.compressorfrontend.FlattenedByteArray; +import com.yscope.clp.compressorfrontend.FlattenedByteArrayFactory; +import com.yscope.clp.compressorfrontend.MessageDecoder; +import com.yscope.clp.compressorfrontend.MessageEncoder; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import javax.validation.constraints.NotNull; +import org.apache.pinot.segment.local.realtime.impl.dictionary.BytesOffHeapMutableDictionary; +import org.apache.pinot.segment.spi.index.mutable.MutableForwardIndex; +import org.apache.pinot.segment.spi.memory.PinotDataBufferMemoryManager; +import org.apache.pinot.spi.data.FieldSpec; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * This mutable forward index implements a composite index for string-typed columns with dynamic encoding options: + * <ol> + * <li>Pure CLP dictionary encoding when the dictionary cardinality is below a configurable threshold.</li> + * <li>CLP dictionary encoding combined with a raw string forward index when the dictionary cardinality exceeds + * the threshold.</li> + * </ol> + * <p> + * Initially, CLP encoding transforms a high-cardinality log message string into three data columns: + * <ul> + * <li>Logtype (very low cardinality) - essentially an inferred format string of the log</li> + * <li>Dictionary variables (medium cardinality) - variables with both alphabets and numbers</li> + * <li>Encoded variables (high cardinality) - pure fixed point and floating point numbers</li> + * </ul> + * The logtype and dictionary variables are dictionary-encoded, while the encoded variables are stored as longs. + * Notably, both {@code encodedVarIds} and {@code encodedVars} are multi-valued, but they are stored using a + * flattened single-value mutable forward index, along with a separate forward index to capture the end offsets + * for each multi-valued document. This approach is necessary because the maximum number of values per document + * is unknown during ingestion, unlike the existing multi-value forward index, which requires this information + * upfront. During the conversion from mutable to immutable forward index, the two single-value mutable indices + * are merged into a single immutable multi-valued forward index, as the max length is known at conversion time. + * <p> + * During ingestion, if the cardinality of either the {@code logtypeId} or {@code dictVarID} exceeds a predefined + * threshold, the ingestion mode switches to a raw bytes forward index for subsequent documents. Maintaining very + * large dictionaries is inefficient in Pinot due to memory and I/O constraints (memory-mapped). Switching to a + * raw bytes forward index helps avoid these issues. During reads, if the requested {@code docId} is in the raw + * forward index, the raw bytes are returned. Otherwise, the log type, dictionary variables, and encoded variables + * are decoded using the CLP decoder to return the original log message's bytes. + * + * <p><b>Note on Write and Read Operations:</b> Writes are strictly sequential, while reads can be performed + * randomly. The supported append operations are:</p> + * <ul> + * <li>{@code setString(int docId, String value)} - Encodes the log message using CLP and invokes + * {@code appendEncodedMessage(@NotNull EncodedMessage clpEncodedMessage)}.</li> + * <li>{@code appendEncodedMessage(@NotNull EncodedMessage clpEncodedMessage)}</li> + * </ul> + * + * <p><b>Limitations:</b> The current CLP mutable forward index does not achieve the same compression ratio as the + * original standalone CLP implementation, primarily due to design differences between Pinot and CLP. While Pinot + * is optimized for fast random access, CLP is designed for single-pass streaming compression and search. As a result, + * Pinot sacrifices some compression efficiency for these primary reasons: + * <ul> + * <li>Pinot implementation uses block compression compared to CLP’s standalone streaming compression, which achieves + * much better compression ratio at the cost of random access performance.</li> + * <li>Pinot implementation uses uncompressed dictionaries for random lookups of log types and dictionary variables, + * whereas CLP employs compressed dictionaries suited only for single-pass streaming queries.</li> + * <li>Pinot stores additional offsets and length metadata for log types, dictionary variables, and encoded + * variables:</li> + * <ul> + * <li>Streaming forward indices used by CLP's standalone implementation, do not need to store document start or + * end markers because the boundary of one document naturally aligns with the next.</li> + * <li>CLP avoids storing the number of {@code dictVars} and {@code encodedVars}, as this information is already + * embedded in the log type and available during decoding. Pinot, however, needs to store this metadata, + * which can sometimes take up more space than the data itself when compressed.</li> + * </ul> + * </ul> + * Additionally, CLP standalone binaries can perform search without decompressing the data into plain text. The most + * common query type on log data is partial matching. Frequently, searches can be completed by scanning only the logtype + * dictionaries for partial matches. Searches can also be executed on {@code logtypeId} and {@code dictVarId} directly + * by first performing lookup on the dictionaries to get a subset of dictionary ids to filter, whereas in Pinot, + * each dictionary id must be first converted back to strings, followed by a brute-force search on the corresponding + * string value. For these reasons, direct searching on CLP columns in Pinot is not yet implemented but may be + * included in future updates.</p> + */ +public class CLPMutableForwardIndexV2 implements MutableForwardIndex { + protected static final Logger LOGGER = LoggerFactory.getLogger(CLPMutableForwardIndexV2.class); + public final String _columnName; + + protected final EncodedMessage _clpEncodedMessage; + protected final MessageEncoder _clpMessageEncoder; + protected final MessageDecoder _clpMessageDecoder; + + protected int _nextDocId = 0; + protected int _nextDictVarDocId = 0; + protected int _nextEncodedVarId = 0; + protected int _bytesRawFwdIndexDocIdStartOffset = Integer.MAX_VALUE; Review Comment: The value is updated at the end of `appendEncodedMessage` method call. I will add a comment to make this clear. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
