This is an automated email from the ASF dual-hosted git repository.
gabriellee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1d5ece98c92 [refactor](minor) Remove unused Partitioner (#42147)
1d5ece98c92 is described below
commit 1d5ece98c920b91d6c72e9bfb49676ae92f8321d
Author: Gabriel <gabrielleeb...@gmail.com>
AuthorDate: Mon Oct 21 14:07:51 2024 +0800
[refactor](minor) Remove unused Partitioner (#42147)
---
be/src/vec/runtime/partitioner.cpp | 41 ++++++-----------------------
be/src/vec/runtime/partitioner.h | 49 ++++++++++-------------------------
be/src/vec/sink/vdata_stream_sender.h | 7 -----
3 files changed, 22 insertions(+), 75 deletions(-)
diff --git a/be/src/vec/runtime/partitioner.cpp
b/be/src/vec/runtime/partitioner.cpp
index 89656a74508..660ffe51a83 100644
--- a/be/src/vec/runtime/partitioner.cpp
+++ b/be/src/vec/runtime/partitioner.cpp
@@ -24,9 +24,8 @@
namespace doris::vectorized {
-template <typename HashValueType, typename ChannelIds>
-Status Partitioner<HashValueType, ChannelIds>::do_partitioning(RuntimeState*
state,
- Block* block)
const {
+template <typename ChannelIds>
+Status Crc32HashPartitioner<ChannelIds>::do_partitioning(RuntimeState* state,
Block* block) const {
int rows = block->rows();
if (rows > 0) {
@@ -55,47 +54,23 @@ Status Partitioner<HashValueType,
ChannelIds>::do_partitioning(RuntimeState* sta
template <typename ChannelIds>
void Crc32HashPartitioner<ChannelIds>::_do_hash(const ColumnPtr& column,
uint32_t* __restrict result,
int idx) const {
- column->update_crcs_with_value(result,
Base::_partition_expr_ctxs[idx]->root()->type().type,
+ column->update_crcs_with_value(result,
_partition_expr_ctxs[idx]->root()->type().type,
column->size());
}
-template <typename ChannelIds>
-void XXHashPartitioner<ChannelIds>::_do_hash(const ColumnPtr& column,
uint64_t* __restrict result,
- int /*idx*/) const {
- column->update_hashes_with_value(result);
-}
-
-template <typename ChannelIds>
-Status XXHashPartitioner<ChannelIds>::clone(RuntimeState* state,
- std::unique_ptr<PartitionerBase>&
partitioner) {
- auto* new_partitioner = new XXHashPartitioner(Base::_partition_count);
- partitioner.reset(new_partitioner);
-
new_partitioner->_partition_expr_ctxs.resize(Base::_partition_expr_ctxs.size());
- for (size_t i = 0; i < Base::_partition_expr_ctxs.size(); i++) {
- RETURN_IF_ERROR(Base::_partition_expr_ctxs[i]->clone(
- state, new_partitioner->_partition_expr_ctxs[i]));
- }
- return Status::OK();
-}
-
template <typename ChannelIds>
Status Crc32HashPartitioner<ChannelIds>::clone(RuntimeState* state,
std::unique_ptr<PartitionerBase>& partitioner) {
- auto* new_partitioner = new Crc32HashPartitioner(Base::_partition_count);
+ auto* new_partitioner = new
Crc32HashPartitioner<ChannelIds>(_partition_count);
partitioner.reset(new_partitioner);
-
new_partitioner->_partition_expr_ctxs.resize(Base::_partition_expr_ctxs.size());
- for (size_t i = 0; i < Base::_partition_expr_ctxs.size(); i++) {
- RETURN_IF_ERROR(Base::_partition_expr_ctxs[i]->clone(
- state, new_partitioner->_partition_expr_ctxs[i]));
+ new_partitioner->_partition_expr_ctxs.resize(_partition_expr_ctxs.size());
+ for (size_t i = 0; i < _partition_expr_ctxs.size(); i++) {
+ RETURN_IF_ERROR(
+ _partition_expr_ctxs[i]->clone(state,
new_partitioner->_partition_expr_ctxs[i]));
}
return Status::OK();
}
-template class Partitioner<size_t, pipeline::LocalExchangeChannelIds>;
-template class XXHashPartitioner<pipeline::LocalExchangeChannelIds>;
-template class Partitioner<size_t, ShuffleChannelIds>;
-template class XXHashPartitioner<ShuffleChannelIds>;
-template class Partitioner<uint32_t, ShuffleChannelIds>;
template class Crc32HashPartitioner<ShuffleChannelIds>;
template class Crc32HashPartitioner<SpillPartitionChannelIds>;
diff --git a/be/src/vec/runtime/partitioner.h b/be/src/vec/runtime/partitioner.h
index 5607a83327b..e8feb74335a 100644
--- a/be/src/vec/runtime/partitioner.h
+++ b/be/src/vec/runtime/partitioner.h
@@ -58,11 +58,11 @@ protected:
const size_t _partition_count;
};
-template <typename HashValueType, typename ChannelIds>
-class Partitioner : public PartitionerBase {
+template <typename ChannelIds>
+class Crc32HashPartitioner : public PartitionerBase {
public:
- Partitioner(int partition_count) : PartitionerBase(partition_count) {}
- ~Partitioner() override = default;
+ Crc32HashPartitioner(int partition_count) :
PartitionerBase(partition_count) {}
+ ~Crc32HashPartitioner() override = default;
Status init(const std::vector<TExpr>& texprs) override {
return VExpr::create_expr_trees(texprs, _partition_expr_ctxs);
@@ -76,9 +76,9 @@ public:
Status do_partitioning(RuntimeState* state, Block* block) const override;
- ChannelField get_channel_ids() const override {
- return {_hash_vals.data(), sizeof(HashValueType)};
- }
+ ChannelField get_channel_ids() const override { return {_hash_vals.data(),
sizeof(uint32_t)}; }
+
+ Status clone(RuntimeState* state, std::unique_ptr<PartitionerBase>&
partitioner) override;
protected:
Status _get_partition_column_result(Block* block, std::vector<int>&
result) const {
@@ -89,38 +89,17 @@ protected:
return Status::OK();
}
- virtual void _do_hash(const ColumnPtr& column, HashValueType* __restrict
result,
- int idx) const = 0;
+ void _do_hash(const ColumnPtr& column, uint32_t* __restrict result, int
idx) const;
VExprContextSPtrs _partition_expr_ctxs;
- mutable std::vector<HashValueType> _hash_vals;
+ mutable std::vector<uint32_t> _hash_vals;
};
-template <typename ChannelIds>
-class XXHashPartitioner final : public Partitioner<uint64_t, ChannelIds> {
-public:
- using Base = Partitioner<uint64_t, ChannelIds>;
- XXHashPartitioner(int partition_count) : Partitioner<uint64_t,
ChannelIds>(partition_count) {}
- ~XXHashPartitioner() override = default;
-
- Status clone(RuntimeState* state, std::unique_ptr<PartitionerBase>&
partitioner) override;
-
-private:
- void _do_hash(const ColumnPtr& column, uint64_t* __restrict result, int
idx) const override;
-};
-
-template <typename ChannelIds>
-class Crc32HashPartitioner final : public Partitioner<uint32_t, ChannelIds> {
-public:
- using Base = Partitioner<uint32_t, ChannelIds>;
- Crc32HashPartitioner(int partition_count)
- : Partitioner<uint32_t, ChannelIds>(partition_count) {}
- ~Crc32HashPartitioner() override = default;
-
- Status clone(RuntimeState* state, std::unique_ptr<PartitionerBase>&
partitioner) override;
-
-private:
- void _do_hash(const ColumnPtr& column, uint32_t* __restrict result, int
idx) const override;
+struct ShuffleChannelIds {
+ template <typename HashValueType>
+ HashValueType operator()(HashValueType l, size_t r) {
+ return l % r;
+ }
};
struct SpillPartitionChannelIds {
diff --git a/be/src/vec/sink/vdata_stream_sender.h
b/be/src/vec/sink/vdata_stream_sender.h
index b0b0e0dc182..f59a39e4969 100644
--- a/be/src/vec/sink/vdata_stream_sender.h
+++ b/be/src/vec/sink/vdata_stream_sender.h
@@ -98,13 +98,6 @@ private:
const int _batch_size;
};
-struct ShuffleChannelIds {
- template <typename HashValueType>
- HashValueType operator()(HashValueType l, size_t r) {
- return l % r;
- }
-};
-
template <typename Parent>
class Channel {
public:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
