imay commented on a change in pull request #3637: URL: https://github.com/apache/incubator-doris/pull/3637#discussion_r428414008
########## File path: be/src/olap/memory/partial_row_batch.h ########## @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "olap/memory/common.h" +#include "olap/memory/schema.h" + +namespace doris { +namespace memory { + +// A chunk of memory that stores a batch of serialized partial rows +// +// Serialization format for a batch: +// 4 byte len | serialized partial row +// 4 byte len | serialized partial row +// ... +// 4 byte len | serialized partial row +// +// Serialization format for a partial row +// bit vector(se + null) byte size (2 byte) | +// bit vector mark set cells | +// bit vector mark nullable cells' null value | +// 8bit padding +// serialized not null cells +// +// Note: currently only fixed length column types are supported. All length and scalar types store +// in native byte order(little endian in x86-64). +// +// Note: The serialization format is simple, it only provides basic functionalities +// so we can quickly complete the whole create/read/write pipeline. The format may change +// as the project evolves. +class PartialRowBatch { +public: + static const size_t DEFAULT_BYTE_CAPACITY = 1 << 20; + static const size_t DEFAULT_ROW_CAPACIT = 1 << 16; + + PartialRowBatch(scoped_refptr<Schema>* schema, size_t byte_capacity = DEFAULT_BYTE_CAPACITY, + size_t row_capacity = DEFAULT_ROW_CAPACIT); + ~PartialRowBatch(); + + const Schema& schema() const { return *_schema.get(); } + + size_t row_size() const { return _row_offsets.size(); } + size_t row_capacity() const { return _row_capacity; } + size_t byte_size() const { return _bsize; } + size_t byte_capacity() const { return _byte_capacity; } + + const uint8_t* get_row(size_t idx) const; + +private: + friend class PartialRowWriter; + friend class PartialRowReader; + scoped_refptr<Schema> _schema; + vector<uint32_t> _row_offsets; + uint8_t* _data; + size_t _bsize; + size_t _byte_capacity; + size_t _row_capacity; +}; + +// Writer for PartialRowBatch Review comment: Give an example about how to use this class ########## File path: be/src/olap/memory/mem_sub_tablet.h ########## @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "olap/memory/common.h" +#include "olap/memory/schema.h" + +namespace doris { +namespace memory { + +class HashIndex; +class ColumnReader; +class PartialRowReader; +class Column; +class ColumnWriter; + +// A MemTablet can contain multiple MemSubTablets (currently only one). +// MemSubTablet hold a HashIndex and a collection of columns. +// It supports single-writer multi-reader concurrently. +class MemSubTablet { Review comment: Better to give a example about how to use this class. ########## File path: be/src/olap/memory/partial_row_batch.cpp ########## @@ -0,0 +1,267 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "olap/memory/partial_row_batch.h" + +#include "util/bitmap.h" + +namespace doris { +namespace memory { + +// Methods for PartialRowBatch + +PartialRowBatch::PartialRowBatch(scoped_refptr<Schema>* schema, size_t byte_capacity, + size_t row_capacity) + : _schema(*schema) { + _data = reinterpret_cast<uint8_t*>(aligned_malloc(byte_capacity, 64)); + _bsize = 0; + if (!_data) { + _byte_capacity = 0; + _row_capacity = 0; + } + _byte_capacity = byte_capacity; + _row_capacity = row_capacity; +} + +PartialRowBatch::~PartialRowBatch() { + if (_data) { + free(_data); + _data = nullptr; + } + _bsize = 0; + _byte_capacity = 0; + _row_capacity = 0; +} + +const uint8_t* PartialRowBatch::get_row(size_t idx) const { + if (!_data || idx >= _row_offsets.size()) { + return nullptr; + } + return _data + _row_offsets[idx] + 4; +} + +// Methods for PartialRowWriter + +PartialRowWriter::PartialRowWriter(const Schema& schema) + : _schema(schema), _bit_set_size(_schema.cid_size()), _bit_null_size(0) { + _temp_cells.resize(_schema.cid_size()); +} + +PartialRowWriter::~PartialRowWriter() {} + +void PartialRowWriter::start_row() { + _bit_null_size = 0; + memset(&(_temp_cells[0]), 0, sizeof(CellInfo) * _temp_cells.size()); +} + +Status PartialRowWriter::write_row_to_batch(PartialRowBatch* batch) { + if (batch->_row_offsets.size() >= batch->row_capacity()) { + return Status::InvalidArgument("over capacity"); + } + size_t row_byte_size = byte_size(); + if (batch->byte_size() + row_byte_size + 4 > batch->byte_capacity()) { + return Status::InvalidArgument("over capacity"); + } + *reinterpret_cast<uint32_t*>(batch->_data + batch->_bsize) = row_byte_size; + uint8_t* pos = batch->_data + batch->_bsize + 4; + RETURN_IF_ERROR(write(&pos)); + batch->_row_offsets.push_back(batch->_bsize); + batch->_bsize = pos - batch->_data; + return Status::OK(); +} + +Status PartialRowWriter::set(const string& col, const void* data) { + auto cs = _schema.get_by_name(col); + if (!cs) { + return Status::NotFound("col name not found"); + } + if (cs->type() == ColumnType::OLAP_FIELD_TYPE_VARCHAR) { + return Status::NotSupported("var length type not supported"); + } + uint32_t cid = cs->cid(); + if (cs->is_nullable() || data) { + if (cs->is_nullable() && !_temp_cells[cid].isnullable) { + _bit_null_size++; + } + _temp_cells[cid].isnullable = cs->is_nullable(); + _temp_cells[cid].isset = 1; + _temp_cells[cid].data = reinterpret_cast<const uint8_t*>(data); + } else { + return Status::InvalidArgument("not nullable"); + } + return Status::OK(); +} + +Status PartialRowWriter::set(uint32_t cid, const void* data) { + auto cs = _schema.get_by_cid(cid); + if (!cs) { + return Status::NotFound("cid not found"); + } + if (cs->is_nullable() || data) { + if (cs->is_nullable() && !_temp_cells[cid].isnullable) { + _bit_null_size++; + } + _temp_cells[cid].isnullable = cs->is_nullable(); + _temp_cells[cid].isset = 1; + _temp_cells[cid].data = reinterpret_cast<const uint8_t*>(data); + } else { + return Status::InvalidArgument("not nullable column set to null"); + } + return Status::OK(); +} + +Status PartialRowWriter::set_delete() { + // TODO: support delete + // _temp_cells[0].isset = 1; + return Status::NotSupported("delete not supported"); +} + +size_t PartialRowWriter::byte_size() const { + // TODO: support delete + size_t bit_all_size = num_block(_bit_set_size + _bit_null_size, 8); + size_t data_size = 2 + bit_all_size; + for (size_t i = 1; i < _temp_cells.size(); i++) { + if (_temp_cells[i].data) { + size_t bsize = _schema.get_column_byte_size(i); Review comment: seems this can be done in constructor. ########## File path: be/src/olap/memory/partial_row_batch.cpp ########## @@ -0,0 +1,267 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "olap/memory/partial_row_batch.h" + +#include "util/bitmap.h" + +namespace doris { +namespace memory { + +// Methods for PartialRowBatch + +PartialRowBatch::PartialRowBatch(scoped_refptr<Schema>* schema, size_t byte_capacity, + size_t row_capacity) + : _schema(*schema) { + _data = reinterpret_cast<uint8_t*>(aligned_malloc(byte_capacity, 64)); + _bsize = 0; + if (!_data) { + _byte_capacity = 0; + _row_capacity = 0; + } + _byte_capacity = byte_capacity; + _row_capacity = row_capacity; +} + +PartialRowBatch::~PartialRowBatch() { + if (_data) { + free(_data); + _data = nullptr; + } + _bsize = 0; + _byte_capacity = 0; + _row_capacity = 0; +} + +const uint8_t* PartialRowBatch::get_row(size_t idx) const { + if (!_data || idx >= _row_offsets.size()) { + return nullptr; + } + return _data + _row_offsets[idx] + 4; +} + +// Methods for PartialRowWriter + +PartialRowWriter::PartialRowWriter(const Schema& schema) + : _schema(schema), _bit_set_size(_schema.cid_size()), _bit_null_size(0) { + _temp_cells.resize(_schema.cid_size()); +} + +PartialRowWriter::~PartialRowWriter() {} + +void PartialRowWriter::start_row() { + _bit_null_size = 0; + memset(&(_temp_cells[0]), 0, sizeof(CellInfo) * _temp_cells.size()); +} + +Status PartialRowWriter::write_row_to_batch(PartialRowBatch* batch) { + if (batch->_row_offsets.size() >= batch->row_capacity()) { + return Status::InvalidArgument("over capacity"); + } + size_t row_byte_size = byte_size(); + if (batch->byte_size() + row_byte_size + 4 > batch->byte_capacity()) { + return Status::InvalidArgument("over capacity"); + } + *reinterpret_cast<uint32_t*>(batch->_data + batch->_bsize) = row_byte_size; + uint8_t* pos = batch->_data + batch->_bsize + 4; + RETURN_IF_ERROR(write(&pos)); + batch->_row_offsets.push_back(batch->_bsize); + batch->_bsize = pos - batch->_data; + return Status::OK(); +} + +Status PartialRowWriter::set(const string& col, const void* data) { + auto cs = _schema.get_by_name(col); + if (!cs) { + return Status::NotFound("col name not found"); + } + if (cs->type() == ColumnType::OLAP_FIELD_TYPE_VARCHAR) { + return Status::NotSupported("var length type not supported"); + } + uint32_t cid = cs->cid(); + if (cs->is_nullable() || data) { + if (cs->is_nullable() && !_temp_cells[cid].isnullable) { + _bit_null_size++; + } + _temp_cells[cid].isnullable = cs->is_nullable(); + _temp_cells[cid].isset = 1; + _temp_cells[cid].data = reinterpret_cast<const uint8_t*>(data); + } else { + return Status::InvalidArgument("not nullable"); + } + return Status::OK(); +} + +Status PartialRowWriter::set(uint32_t cid, const void* data) { + auto cs = _schema.get_by_cid(cid); + if (!cs) { + return Status::NotFound("cid not found"); + } + if (cs->is_nullable() || data) { + if (cs->is_nullable() && !_temp_cells[cid].isnullable) { + _bit_null_size++; + } + _temp_cells[cid].isnullable = cs->is_nullable(); + _temp_cells[cid].isset = 1; + _temp_cells[cid].data = reinterpret_cast<const uint8_t*>(data); + } else { + return Status::InvalidArgument("not nullable column set to null"); + } Review comment: Seems the same with one in above function. ########## File path: be/src/olap/memory/partial_row_batch.cpp ########## @@ -0,0 +1,267 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "olap/memory/partial_row_batch.h" + +#include "util/bitmap.h" + +namespace doris { +namespace memory { + +// Methods for PartialRowBatch + +PartialRowBatch::PartialRowBatch(scoped_refptr<Schema>* schema, size_t byte_capacity, + size_t row_capacity) + : _schema(*schema) { + _data = reinterpret_cast<uint8_t*>(aligned_malloc(byte_capacity, 64)); + _bsize = 0; + if (!_data) { + _byte_capacity = 0; + _row_capacity = 0; + } + _byte_capacity = byte_capacity; + _row_capacity = row_capacity; +} + +PartialRowBatch::~PartialRowBatch() { + if (_data) { + free(_data); + _data = nullptr; + } + _bsize = 0; + _byte_capacity = 0; + _row_capacity = 0; +} + +const uint8_t* PartialRowBatch::get_row(size_t idx) const { + if (!_data || idx >= _row_offsets.size()) { + return nullptr; + } + return _data + _row_offsets[idx] + 4; +} + +// Methods for PartialRowWriter + +PartialRowWriter::PartialRowWriter(const Schema& schema) + : _schema(schema), _bit_set_size(_schema.cid_size()), _bit_null_size(0) { + _temp_cells.resize(_schema.cid_size()); +} + +PartialRowWriter::~PartialRowWriter() {} + +void PartialRowWriter::start_row() { + _bit_null_size = 0; + memset(&(_temp_cells[0]), 0, sizeof(CellInfo) * _temp_cells.size()); +} + +Status PartialRowWriter::write_row_to_batch(PartialRowBatch* batch) { + if (batch->_row_offsets.size() >= batch->row_capacity()) { + return Status::InvalidArgument("over capacity"); + } + size_t row_byte_size = byte_size(); + if (batch->byte_size() + row_byte_size + 4 > batch->byte_capacity()) { + return Status::InvalidArgument("over capacity"); + } + *reinterpret_cast<uint32_t*>(batch->_data + batch->_bsize) = row_byte_size; + uint8_t* pos = batch->_data + batch->_bsize + 4; + RETURN_IF_ERROR(write(&pos)); + batch->_row_offsets.push_back(batch->_bsize); + batch->_bsize = pos - batch->_data; + return Status::OK(); +} + +Status PartialRowWriter::set(const string& col, const void* data) { + auto cs = _schema.get_by_name(col); + if (!cs) { + return Status::NotFound("col name not found"); + } + if (cs->type() == ColumnType::OLAP_FIELD_TYPE_VARCHAR) { + return Status::NotSupported("var length type not supported"); + } + uint32_t cid = cs->cid(); + if (cs->is_nullable() || data) { + if (cs->is_nullable() && !_temp_cells[cid].isnullable) { + _bit_null_size++; + } + _temp_cells[cid].isnullable = cs->is_nullable(); + _temp_cells[cid].isset = 1; + _temp_cells[cid].data = reinterpret_cast<const uint8_t*>(data); + } else { + return Status::InvalidArgument("not nullable"); + } + return Status::OK(); +} + +Status PartialRowWriter::set(uint32_t cid, const void* data) { + auto cs = _schema.get_by_cid(cid); + if (!cs) { + return Status::NotFound("cid not found"); + } + if (cs->is_nullable() || data) { + if (cs->is_nullable() && !_temp_cells[cid].isnullable) { + _bit_null_size++; + } + _temp_cells[cid].isnullable = cs->is_nullable(); + _temp_cells[cid].isset = 1; + _temp_cells[cid].data = reinterpret_cast<const uint8_t*>(data); + } else { + return Status::InvalidArgument("not nullable column set to null"); + } + return Status::OK(); +} + +Status PartialRowWriter::set_delete() { + // TODO: support delete + // _temp_cells[0].isset = 1; + return Status::NotSupported("delete not supported"); +} + +size_t PartialRowWriter::byte_size() const { + // TODO: support delete + size_t bit_all_size = num_block(_bit_set_size + _bit_null_size, 8); + size_t data_size = 2 + bit_all_size; + for (size_t i = 1; i < _temp_cells.size(); i++) { + if (_temp_cells[i].data) { Review comment: ```suggestion if (_temp_cells[i].data != nullptr) { ``` prefer `ptr != nullptr` for pointer type ########## File path: be/src/olap/memory/partial_row_batch.cpp ########## @@ -0,0 +1,267 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "olap/memory/partial_row_batch.h" + +#include "util/bitmap.h" + +namespace doris { +namespace memory { + +// Methods for PartialRowBatch + +PartialRowBatch::PartialRowBatch(scoped_refptr<Schema>* schema, size_t byte_capacity, + size_t row_capacity) + : _schema(*schema) { + _data = reinterpret_cast<uint8_t*>(aligned_malloc(byte_capacity, 64)); + _bsize = 0; + if (!_data) { + _byte_capacity = 0; + _row_capacity = 0; + } + _byte_capacity = byte_capacity; + _row_capacity = row_capacity; +} + +PartialRowBatch::~PartialRowBatch() { + if (_data) { + free(_data); + _data = nullptr; + } + _bsize = 0; + _byte_capacity = 0; + _row_capacity = 0; +} + +const uint8_t* PartialRowBatch::get_row(size_t idx) const { + if (!_data || idx >= _row_offsets.size()) { + return nullptr; + } + return _data + _row_offsets[idx] + 4; +} + +// Methods for PartialRowWriter + +PartialRowWriter::PartialRowWriter(const Schema& schema) + : _schema(schema), _bit_set_size(_schema.cid_size()), _bit_null_size(0) { + _temp_cells.resize(_schema.cid_size()); +} + +PartialRowWriter::~PartialRowWriter() {} + +void PartialRowWriter::start_row() { + _bit_null_size = 0; + memset(&(_temp_cells[0]), 0, sizeof(CellInfo) * _temp_cells.size()); +} + +Status PartialRowWriter::write_row_to_batch(PartialRowBatch* batch) { + if (batch->_row_offsets.size() >= batch->row_capacity()) { + return Status::InvalidArgument("over capacity"); + } + size_t row_byte_size = byte_size(); + if (batch->byte_size() + row_byte_size + 4 > batch->byte_capacity()) { + return Status::InvalidArgument("over capacity"); + } + *reinterpret_cast<uint32_t*>(batch->_data + batch->_bsize) = row_byte_size; + uint8_t* pos = batch->_data + batch->_bsize + 4; + RETURN_IF_ERROR(write(&pos)); + batch->_row_offsets.push_back(batch->_bsize); + batch->_bsize = pos - batch->_data; Review comment: Why not construct all items that the batch needs, and construct the batch. If then, the batch can be immutable and this class will not operate batch's private member. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
