amorynan commented on code in PR #18819:
URL: https://github.com/apache/doris/pull/18819#discussion_r1172021202
##########
be/src/vec/data_types/serde/data_type_number_serde.h:
##########
@@ -157,5 +162,62 @@ Status DataTypeNumberSerDe<T>::write_column_to_pb(const
IColumn& column, PValues
return Status::OK();
}
+template <typename T>
+Status DataTypeNumberSerDe<T>::read_column_from_jsonb(IColumn& column,
+ const JsonbValue* arg)
const {
+ if constexpr (std::is_same_v<T, Int8>) {
+ column.insert(static_cast<const JsonbInt8Val*>(arg)->val());
Review Comment:
emm origin code insert(Field), maybe I can use insert_value
##########
be/src/vec/jsonb/serialize.cpp:
##########
@@ -17,268 +17,265 @@
#include "vec/jsonb/serialize.h"
-#include "olap/hll.h"
#include "olap/tablet_schema.h"
-#include "runtime/jsonb_value.h"
#include "util/jsonb_stream.h"
#include "util/jsonb_writer.h"
#include "vec/common/arena.h"
-#include "vec/core/types.h"
namespace doris::vectorized {
-static inline bool is_column_null_at(int row, const IColumn* column, const
doris::FieldType& type,
- const StringRef& data_ref) {
- if (type != FieldType::OLAP_FIELD_TYPE_ARRAY) {
- return data_ref.data == nullptr;
- } else {
- Field array;
- column->get(row, array);
- return array.is_null();
- }
-}
-
-static bool is_jsonb_blob_type(FieldType type) {
- return type == FieldType::OLAP_FIELD_TYPE_VARCHAR || type ==
FieldType::OLAP_FIELD_TYPE_CHAR ||
- type == FieldType::OLAP_FIELD_TYPE_STRING || type ==
FieldType::OLAP_FIELD_TYPE_STRUCT ||
- type == FieldType::OLAP_FIELD_TYPE_ARRAY || type ==
FieldType::OLAP_FIELD_TYPE_MAP ||
- type == FieldType::OLAP_FIELD_TYPE_HLL || type ==
FieldType::OLAP_FIELD_TYPE_OBJECT ||
- type == FieldType::OLAP_FIELD_TYPE_JSONB;
-}
-
-// jsonb -> column value
-static void deserialize_column(PrimitiveType type, JsonbValue* slot_value,
MutableColumnPtr& dst) {
- if (type == TYPE_ARRAY) {
- assert(slot_value->isBinary());
- auto blob = static_cast<JsonbBlobVal*>(slot_value);
- dst->deserialize_and_insert_from_arena(blob->getBlob());
- } else if (type == TYPE_OBJECT) {
- assert(slot_value->isBinary());
- auto blob = static_cast<JsonbBlobVal*>(slot_value);
- BitmapValue bitmap_value;
- bitmap_value.deserialize(blob->getBlob());
- dst->insert_data(reinterpret_cast<const char*>(&bitmap_value),
sizeof(BitmapValue));
- } else if (type == TYPE_HLL) {
- assert(slot_value->isBinary());
- auto blob = static_cast<JsonbBlobVal*>(slot_value);
- HyperLogLog hyper_log_log;
- Slice data {blob->getBlob(), blob->getBlobLen()};
- hyper_log_log.deserialize(data);
- dst->insert_data(reinterpret_cast<const char*>(&hyper_log_log),
sizeof(HyperLogLog));
- } else if (is_string_type(type)) {
- assert(slot_value->isBinary());
- auto blob = static_cast<JsonbBlobVal*>(slot_value);
- dst->insert_data(blob->getBlob(), blob->getBlobLen());
- } else {
- switch (type) {
- case TYPE_BOOLEAN: {
- assert(slot_value->isInt8());
- dst->insert(static_cast<JsonbInt8Val*>(slot_value)->val());
- break;
- }
- case TYPE_TINYINT: {
- assert(slot_value->isInt8());
- dst->insert(static_cast<JsonbInt8Val*>(slot_value)->val());
- break;
- }
- case TYPE_SMALLINT: {
- assert(slot_value->isInt16());
- dst->insert(static_cast<JsonbInt16Val*>(slot_value)->val());
- break;
- }
- case TYPE_INT: {
- assert(slot_value->isInt32());
- dst->insert(static_cast<JsonbInt32Val*>(slot_value)->val());
- break;
- }
- case TYPE_BIGINT: {
- assert(slot_value->isInt64());
- dst->insert(static_cast<JsonbInt64Val*>(slot_value)->val());
- break;
- }
- case TYPE_LARGEINT: {
- assert(slot_value->isInt128());
- dst->insert(static_cast<JsonbInt128Val*>(slot_value)->val());
- break;
- }
- case TYPE_FLOAT: {
- dst->insert(static_cast<JsonbFloatVal*>(slot_value)->val());
- break;
- }
- case TYPE_DOUBLE: {
- dst->insert(static_cast<JsonbDoubleVal*>(slot_value)->val());
- break;
- }
- case TYPE_DATE: {
- assert(slot_value->isInt32());
- int32_t val = static_cast<JsonbInt32Val*>(slot_value)->val();
- dst->insert_many_fix_len_data(reinterpret_cast<const char*>(&val),
1);
- break;
- }
- case TYPE_DATETIME: {
- assert(slot_value->isInt64());
- int64_t val = static_cast<JsonbInt64Val*>(slot_value)->val();
- dst->insert_many_fix_len_data(reinterpret_cast<const char*>(&val),
1);
- break;
- }
- case TYPE_DATEV2: {
- assert(slot_value->isInt32());
- dst->insert(static_cast<JsonbInt32Val*>(slot_value)->val());
- break;
- }
- case TYPE_DATETIMEV2: {
- assert(slot_value->isInt64());
- dst->insert(static_cast<JsonbInt64Val*>(slot_value)->val());
- break;
- }
- case TYPE_DECIMAL32: {
- assert(slot_value->isInt32());
- dst->insert(static_cast<JsonbInt32Val*>(slot_value)->val());
- break;
- }
- case TYPE_DECIMAL64: {
- assert(slot_value->isInt64());
- dst->insert(static_cast<JsonbInt64Val*>(slot_value)->val());
- break;
- }
- case TYPE_DECIMAL128I: {
- assert(slot_value->isInt128());
- dst->insert(static_cast<JsonbInt128Val*>(slot_value)->val());
- break;
- }
- default:
- LOG(FATAL) << "unknow type " << type;
- break;
- }
- }
-}
-
-// column value -> jsonb
-static void serialize_column(Arena* mem_pool, const TabletColumn&
tablet_column,
- const IColumn* column, const StringRef& data_ref,
int row,
- JsonbWriterT<JsonbOutStream>& jsonb_writer) {
- if (is_column_null_at(row, column, tablet_column.type(), data_ref)) {
- // Do nothing
- return;
- }
- jsonb_writer.writeKey(tablet_column.unique_id());
- if (tablet_column.is_array_type()) {
- const char* begin = nullptr;
- StringRef value = column->serialize_value_into_arena(row, *mem_pool,
begin);
- jsonb_writer.writeStartBinary();
- jsonb_writer.writeBinary(value.data, value.size);
- jsonb_writer.writeEndBinary();
- } else if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_OBJECT) {
- auto bitmap_value = (BitmapValue*)(data_ref.data);
- auto size = bitmap_value->getSizeInBytes();
- // serialize the content of string
- auto ptr = mem_pool->alloc(size);
- bitmap_value->write_to(reinterpret_cast<char*>(ptr));
- jsonb_writer.writeStartBinary();
- jsonb_writer.writeBinary(reinterpret_cast<const char*>(ptr), size);
- jsonb_writer.writeEndBinary();
- } else if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_HLL) {
- auto hll_value = (HyperLogLog*)(data_ref.data);
- auto size = hll_value->max_serialized_size();
- auto ptr = reinterpret_cast<char*>(mem_pool->alloc(size));
- size_t actual_size = hll_value->serialize((uint8_t*)ptr);
- jsonb_writer.writeStartBinary();
- jsonb_writer.writeBinary(reinterpret_cast<const char*>(ptr),
actual_size);
- jsonb_writer.writeEndBinary();
- } else if (is_jsonb_blob_type(tablet_column.type())) {
- jsonb_writer.writeStartBinary();
- jsonb_writer.writeBinary(reinterpret_cast<const char*>(data_ref.data),
data_ref.size);
- jsonb_writer.writeEndBinary();
- } else {
- switch (tablet_column.type()) {
- case FieldType::OLAP_FIELD_TYPE_BOOL: {
- int8_t val = *reinterpret_cast<const int8_t*>(data_ref.data);
- jsonb_writer.writeInt8(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_TINYINT: {
- int8_t val = *reinterpret_cast<const int8_t*>(data_ref.data);
- jsonb_writer.writeInt8(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_SMALLINT: {
- int16_t val = *reinterpret_cast<const int16_t*>(data_ref.data);
- jsonb_writer.writeInt16(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_INT: {
- int32_t val = *reinterpret_cast<const int32_t*>(data_ref.data);
- jsonb_writer.writeInt32(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_BIGINT: {
- int64_t val = *reinterpret_cast<const int64_t*>(data_ref.data);
- jsonb_writer.writeInt64(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_LARGEINT: {
- __int128_t val = *reinterpret_cast<const
__int128_t*>(data_ref.data);
- jsonb_writer.writeInt128(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_FLOAT: {
- float val = *reinterpret_cast<const float*>(data_ref.data);
- jsonb_writer.writeFloat(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DOUBLE: {
- double val = *reinterpret_cast<const double*>(data_ref.data);
- jsonb_writer.writeDouble(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DATE: {
- const auto* datetime_cur = reinterpret_cast<const
VecDateTimeValue*>(data_ref.data);
- jsonb_writer.writeInt32(datetime_cur->to_olap_date());
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DATETIME: {
- const auto* datetime_cur = reinterpret_cast<const
VecDateTimeValue*>(data_ref.data);
- jsonb_writer.writeInt64(datetime_cur->to_olap_datetime());
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DATEV2: {
- uint32_t val = *reinterpret_cast<const uint32_t*>(data_ref.data);
- jsonb_writer.writeInt32(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: {
- uint64_t val = *reinterpret_cast<const uint64_t*>(data_ref.data);
- jsonb_writer.writeInt64(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DECIMAL32: {
- Decimal32::NativeType val =
- *reinterpret_cast<const
Decimal32::NativeType*>(data_ref.data);
- jsonb_writer.writeInt32(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DECIMAL64: {
- Decimal64::NativeType val =
- *reinterpret_cast<const
Decimal64::NativeType*>(data_ref.data);
- jsonb_writer.writeInt64(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DECIMAL128I: {
- Decimal128I::NativeType val =
- *reinterpret_cast<const
Decimal128I::NativeType*>(data_ref.data);
- jsonb_writer.writeInt128(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DECIMAL:
- LOG(FATAL)
- << "FieldType::OLAP_FIELD_TYPE_DECIMAL not implemented use
DecimalV3 instead";
- break;
- default:
- LOG(FATAL) << "unknow type " << int(tablet_column.type());
- break;
- }
- }
-}
+//static inline bool is_column_null_at(int row, const IColumn* column, const
doris::FieldType& type,
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
