BiteTheDDDDt commented on code in PR #52967:
URL: https://github.com/apache/doris/pull/52967#discussion_r2203788668
##########
be/src/util/frame_of_reference_coding.cpp:
##########
@@ -421,28 +422,141 @@ bool ForDecoder<T>::init() {
}
// todo(kks): improve this method by SIMD instructions
+
+template <typename T>
+template <typename U>
+void ForDecoder<T>::bit_unpack_optimize(const uint8_t* input, uint8_t in_num,
int bit_width,
+ T* output) {
+ static_assert(std::is_same<U, int64_t>::value || std::is_same<U,
__int128_t>::value,
+ "bit_unpack_optimize only supports U = int64_t or
__int128_t");
+ constexpr int u_size = sizeof(U); // Size of U
+ constexpr int u_size_shift = (u_size == 8) ? 3 : 4; // log2(u_size)
+ int valid_bit = 0; // How many valid bits
+ int need_bit = 0; // still need
+ size_t input_size = (in_num * bit_width + 7) >> 3; // input's size
+ int full_batch_size = (input_size >> u_size_shift)
+ << u_size_shift; // Adjust input_size to a
multiple of u_size
+ int tail_count = input_size & (u_size - 1); // The remainder of input_size
modulo u_size.
+ // The number of bits in input to adjust to multiples of 8 and thus more
+ int more_bit = (input_size << 3) - (in_num * bit_width);
+
+ // to ensure that only bit_width bits are valid
+ T output_mask;
+ if (bit_width >= static_cast<int>(sizeof(T) * 8)) {
+ output_mask = static_cast<T>(~T(0));
+ } else {
+ output_mask = static_cast<T>((static_cast<T>(1) << bit_width) - 1);
+ }
+
+ U s = 0; // Temporary buffer for bitstream: aggregates input bytes into a
large integer for unpacking
+
+ for (int i = 0; i < full_batch_size; i += u_size) {
+ s = 0;
+
+ if constexpr (u_size == 8) {
+ if constexpr (std::endian::native == std::endian::little) {
Review Comment:
seems judge is useless, to_endian will do this
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
