wumeibanfa commented on code in PR #52829:
URL: https://github.com/apache/doris/pull/52829#discussion_r2188968210
##########
be/src/util/frame_of_reference_coding.cpp:
##########
@@ -72,6 +72,168 @@ void ForEncoder<T>::put_batch(const T* in_data, size_t
count) {
// todo(kks): improve this method by SIMD instructions
+template <typename T>
+void ForEncoder<T>::bit_pack_8(const T* input, uint8_t in_num, int bit_width,
uint8_t* output) {
+ int64_t s = 0;
+ uint8_t output_mask = 255;
+ int tail_count = in_num & 7;
+ int full_batch_size = (in_num >> 3) << 3;
+
+ for (int i = 0; i < full_batch_size; i += 8) {
+ s |= static_cast<int64_t>(input[i + 7]);
+ s |= (static_cast<int64_t>(input[i + 6])) << bit_width;
+ s |= (static_cast<int64_t>(input[i + 5])) << (2 * bit_width);
+ s |= (static_cast<int64_t>(input[i + 4])) << (3 * bit_width);
+ s |= (static_cast<int64_t>(input[i + 3])) << (4 * bit_width);
+ s |= (static_cast<int64_t>(input[i + 2])) << (5 * bit_width);
+ s |= (static_cast<int64_t>(input[i + 1])) << (6 * bit_width);
+ s |= (static_cast<int64_t>(input[i])) << (7 * bit_width);
+
+ for (int j = 0; j < bit_width; j++) {
+ output[j] = (s >> ((bit_width - j - 1) << 3)) & output_mask;
+ }
+ output += bit_width;
+ s = 0;
+ }
+
+ // remainder
+ int byte = tail_count * bit_width;
+ int bytes = (byte + 7) >> 3;
+
+ for (int i = 0; i < tail_count; i++) {
+ s |= (static_cast<int64_t>(input[i + full_batch_size]))
+ << ((tail_count - i - 1) * bit_width);
+ }
+
+ s <<= (bytes << 3) - byte;
+
+ for (int i = 0; i < bytes; i++) {
+ output[i] = (s >> ((bytes - i - 1) << 3)) & output_mask;
+ }
+}
+
+template <typename T>
+void ForEncoder<T>::bit_pack_16(const T* input, uint8_t in_num, int bit_width,
uint8_t* output) {
+ int64_t s = 0;
+ uint8_t output_mask = 255;
+ int tail_count = in_num & 3;
+ int full_batch_size = (in_num >> 2) << 2;
+ int output_size = 0; // How many outputs can be processed at a time
+ int bit_width_remainder =
+ (bit_width << 2) & 7; // How many bits will be left after
processing 4 numbers at a time
+ int extra_bit = 0; // Extra bits after each process
+
+ for (int i = 0; i < full_batch_size; i += 4) {
+ s <<= bit_width;
+ s |= (static_cast<int64_t>(input[i]));
+ s <<= bit_width;
+ s |= (static_cast<int64_t>(input[i + 1]));
+ s <<= bit_width;
+ s |= (static_cast<int64_t>(input[i + 2]));
+ s <<= bit_width;
+ s |= (static_cast<int64_t>(input[i + 3]));
+
+ output_size = (bit_width * 4 + extra_bit) >> 3;
Review Comment:
Explain why calculated it this way?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
