Re: [PR] [fix](pipeline) only sub_running_sink_operators in close [doris]
Mryange commented on PR #43500: URL: https://github.com/apache/doris/pull/43500#issuecomment-2466137450 run cloud_p0 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-3.0: [chore](log) Use correct error type of uneven user behaviour [doris]
yiguolei closed pull request #43493: branch-3.0: [chore](log) Use correct error type of uneven user behaviour URL: https://github.com/apache/doris/pull/43493 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](arrow-flight-sql) Arrow flight server supports data forwarding when BE uses public vip [doris]
yiguolei commented on code in PR #43281:
URL: https://github.com/apache/doris/pull/43281#discussion_r1835327667
##
be/src/runtime/buffer_control_block.cpp:
##
@@ -191,56 +269,115 @@ void BufferControlBlock::get_batch(GetResultBatchCtx*
ctx) {
ctx->on_data(result, _packet_num);
_packet_num++;
-_update_dependency();
return;
}
if (_is_close) {
ctx->on_close(_packet_num, _query_statistics.get());
-_update_dependency();
return;
}
// no ready data, push ctx to waiting list
_waiting_rpc.push_back(ctx);
-_update_dependency();
}
-Status
BufferControlBlock::get_arrow_batch(std::shared_ptr*
result) {
+Status BufferControlBlock::get_arrow_batch(std::shared_ptr*
result,
+ cctz::time_zone& timezone_obj) {
std::unique_lock l(_lock);
+Defer defer {[&]() { _update_dependency(); }};
if (!_status.ok()) {
return _status;
}
if (_is_cancelled) {
return Status::Cancelled("Cancelled");
}
-while (_arrow_flight_batch_queue.empty() && !_is_cancelled && !_is_close) {
-_arrow_data_arrival.wait_for(l, std::chrono::seconds(1));
+while (_arrow_flight_result_batch_queue.empty() && !_is_cancelled &&
!_is_close) {
+_arrow_data_arrival.wait_for(l, std::chrono::milliseconds(20));
}
if (_is_cancelled) {
return Status::Cancelled("Cancelled");
}
-if (!_arrow_flight_batch_queue.empty()) {
-*result = std::move(_arrow_flight_batch_queue.front());
-_arrow_flight_batch_queue.pop_front();
+if (!_arrow_flight_result_batch_queue.empty()) {
+*result = std::move(_arrow_flight_result_batch_queue.front());
+_arrow_flight_result_batch_queue.pop_front();
+timezone_obj = _timezone_obj;
+
for (auto it : _instance_rows_in_queue.front()) {
_instance_rows[it.first] -= it.second;
}
_instance_rows_in_queue.pop_front();
_packet_num++;
-_update_dependency();
return Status::OK();
}
// normal path end
if (_is_close) {
-_update_dependency();
+#ifndef NDEBUG
+std::stringstream ss;
+_profile.pretty_print(&ss);
+LOG(INFO) << fmt::format(
Review Comment:
使用LOG DEBUG
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](arrow-flight-sql) Arrow flight server supports data forwarding when BE uses public vip [doris]
yiguolei commented on code in PR #43281:
URL: https://github.com/apache/doris/pull/43281#discussion_r1835327728
##
be/src/runtime/buffer_control_block.cpp:
##
@@ -191,56 +269,115 @@ void BufferControlBlock::get_batch(GetResultBatchCtx*
ctx) {
ctx->on_data(result, _packet_num);
_packet_num++;
-_update_dependency();
return;
}
if (_is_close) {
ctx->on_close(_packet_num, _query_statistics.get());
-_update_dependency();
return;
}
// no ready data, push ctx to waiting list
_waiting_rpc.push_back(ctx);
-_update_dependency();
}
-Status
BufferControlBlock::get_arrow_batch(std::shared_ptr*
result) {
+Status BufferControlBlock::get_arrow_batch(std::shared_ptr*
result,
+ cctz::time_zone& timezone_obj) {
std::unique_lock l(_lock);
+Defer defer {[&]() { _update_dependency(); }};
if (!_status.ok()) {
return _status;
}
if (_is_cancelled) {
return Status::Cancelled("Cancelled");
}
-while (_arrow_flight_batch_queue.empty() && !_is_cancelled && !_is_close) {
-_arrow_data_arrival.wait_for(l, std::chrono::seconds(1));
+while (_arrow_flight_result_batch_queue.empty() && !_is_cancelled &&
!_is_close) {
+_arrow_data_arrival.wait_for(l, std::chrono::milliseconds(20));
}
if (_is_cancelled) {
return Status::Cancelled("Cancelled");
}
-if (!_arrow_flight_batch_queue.empty()) {
-*result = std::move(_arrow_flight_batch_queue.front());
-_arrow_flight_batch_queue.pop_front();
+if (!_arrow_flight_result_batch_queue.empty()) {
+*result = std::move(_arrow_flight_result_batch_queue.front());
+_arrow_flight_result_batch_queue.pop_front();
+timezone_obj = _timezone_obj;
+
for (auto it : _instance_rows_in_queue.front()) {
_instance_rows[it.first] -= it.second;
}
_instance_rows_in_queue.pop_front();
_packet_num++;
-_update_dependency();
return Status::OK();
}
// normal path end
if (_is_close) {
-_update_dependency();
+#ifndef NDEBUG
+std::stringstream ss;
+_profile.pretty_print(&ss);
+LOG(INFO) << fmt::format(
+"BufferControlBlock finished, fragment_id={}, is_close={},
is_cancelled={}, "
+"packet_num={}, peak_memory_usage={}, profile={}",
+print_id(_fragment_id), _is_close, _is_cancelled, _packet_num,
+_mem_tracker->peak_consumption(), ss.str());
+#endif
return Status::OK();
}
return Status::InternalError("Get Arrow Batch Abnormal Ending");
}
+void BufferControlBlock::get_arrow_batch(GetArrowResultBatchCtx* ctx) {
+std::unique_lock l(_lock);
+SCOPED_ATTACH_TASK(_mem_tracker);
+Defer defer {[&]() { _update_dependency(); }};
+if (!_status.ok()) {
+ctx->on_failure(_status);
+return;
+}
+if (_is_cancelled) {
+ctx->on_failure(Status::Cancelled("Cancelled"));
+return;
+}
+
+if (!_arrow_flight_result_batch_queue.empty()) {
+auto block = _arrow_flight_result_batch_queue.front();
+_arrow_flight_result_batch_queue.pop_front();
+for (auto it : _instance_rows_in_queue.front()) {
+_instance_rows[it.first] -= it.second;
+}
+_instance_rows_in_queue.pop_front();
+
+ctx->on_data(block, _packet_num, _be_exec_version,
_fragement_transmission_compression_type,
+ _timezone, _arrow_schema_field_names,
_serialize_batch_ns_timer,
+ _uncompressed_bytes_counter, _compressed_bytes_counter);
+_packet_num++;
+return;
+}
+
+// normal path end
+if (_is_close) {
+ctx->on_close(_packet_num);
+#ifndef NDEBUG
+std::stringstream ss;
Review Comment:
使用LOG DEBUG
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](arrow-flight-sql) Arrow flight server supports data forwarding when BE uses public vip [doris]
yiguolei commented on code in PR #43281:
URL: https://github.com/apache/doris/pull/43281#discussion_r1835327628
##
be/src/service/arrow_flight/arrow_flight_batch_reader.cpp:
##
@@ -18,52 +18,278 @@
#include "service/arrow_flight/arrow_flight_batch_reader.h"
#include
+#include
+#include
+
+#include
-#include "arrow/builder.h"
#include "runtime/exec_env.h"
+#include "runtime/memory/mem_tracker_limiter.h"
#include "runtime/result_buffer_mgr.h"
+#include "runtime/thread_context.h"
+#include "service/backend_options.h"
+#include "util/arrow/block_convertor.h"
#include "util/arrow/row_batch.h"
#include "util/arrow/utils.h"
+#include "util/brpc_client_cache.h"
+#include "util/ref_count_closure.h"
+#include "util/string_util.h"
+#include "vec/core/block.h"
+
+namespace doris::flight {
+
+constexpr size_t BRPC_CONTROLLER_TIMEOUT_MS = 60 * 1000;
+
+ArrowFlightBatchReaderBase::ArrowFlightBatchReaderBase(
+const std::shared_ptr& statement)
+: _statement(statement) {}
+
+std::shared_ptr ArrowFlightBatchReaderBase::schema() const {
+return _schema;
+}
-namespace doris {
-namespace flight {
+arrow::Status ArrowFlightBatchReaderBase::_return_invalid_status(const
std::string& msg) {
+std::string status_msg =
+fmt::format("ArrowFlightBatchReader {}, packet_seq={},
result={}:{}, finistId={}", msg,
+_packet_seq, _statement->result_addr.hostname,
_statement->result_addr.port,
+print_id(_statement->query_id));
+LOG(WARNING) << status_msg;
+return arrow::Status::Invalid(status_msg);
+}
-std::shared_ptr ArrowFlightBatchReader::schema() const {
-return schema_;
+ArrowFlightBatchReaderBase::~ArrowFlightBatchReaderBase() {
+#ifndef NDEBUG
+LOG(INFO) << fmt::format(
+"ArrowFlightBatchReader finished, packet_seq={},
result_addr={}:{}, finistId={}, "
Review Comment:
这么写不太好，直接LOG(DEBUG) 这样线上又问题，也不用重新编译版本。
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-3.0: [chore](log) Use correct error type of uneven user behaviour [doris]
github-actions[bot] commented on PR #43493: URL: https://github.com/apache/doris/pull/43493#issuecomment-2466144936 clang-tidy review says "All clean, LGTM! :+1:" -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](arrow-flight-sql) Arrow flight server supports data forwarding when BE uses public vip [doris]
yiguolei commented on code in PR #43281:
URL: https://github.com/apache/doris/pull/43281#discussion_r1835328163
##
be/src/vec/sink/varrow_flight_result_writer.cpp:
##
@@ -67,29 +60,31 @@ Status VArrowFlightResultWriter::write(RuntimeState* state,
Block& input_block)
RETURN_IF_ERROR(VExprContext::get_output_block_after_execute_exprs(_output_vexpr_ctxs,
input_block, &block));
-// convert one batch
-std::shared_ptr result;
-auto num_rows = block.rows();
-// arrow::RecordBatch without `nbytes()` in C++
-uint64_t bytes_sent = block.bytes();
{
-SCOPED_TIMER(_convert_tuple_timer);
-RETURN_IF_ERROR(convert_to_arrow_batch(block, _arrow_schema,
arrow::default_memory_pool(),
- &result, _timezone_obj));
-}
-{
-SCOPED_TIMER(_result_send_timer);
-// If this is a dry run task, no need to send data block
-if (!_is_dry_run) {
-status = _sinker->add_arrow_batch(state, result);
-}
-if (status.ok()) {
-_written_rows += num_rows;
+SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_sinker->mem_tracker());
Review Comment:
这里为啥要switch
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](arrow-flight-sql) Arrow flight server supports data forwarding when BE uses public vip [doris]
yiguolei commented on code in PR #43281:
URL: https://github.com/apache/doris/pull/43281#discussion_r1835328145
##
gensrc/proto/internal_service.proto:
##
@@ -283,6 +283,21 @@ message PFetchDataResult {
optional bool empty_batch = 6;
};
+message PFetchArrowDataRequest {
+optional PUniqueId finst_id = 1;
+};
+
+message PFetchArrowDataResult {
+required PStatus status = 1;
Review Comment:
never use required
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Enhancement] replace table options to keep original dropped table in recycle bin [doris]
Vallishp commented on PR #43538: URL: https://github.com/apache/doris/pull/43538#issuecomment-2466149486 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](column_complex) wrong type of Field returned by ColumnComplex [doris]
mrhhsg commented on PR #43515: URL: https://github.com/apache/doris/pull/43515#issuecomment-2466149438 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [branch-3.0](binlog) Support drop view binlog (#39781) [doris]
doris-robot commented on PR #43403: URL: https://github.com/apache/doris/pull/43403#issuecomment-2466112364 TPC-H: Total hot run time: 40779 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools Tpch sf100 test result on commit b7b25cb2b3d3c2a0db1ff9041a34040939644dcb, data reload: false -- Round 1 -- q1 17746 748973417341 q2 2091174 177 174 q3 10648 107411741074 q4 10562 749 819 749 q5 7747279527902790 q6 240 146 146 146 q7 964 620 601 601 q8 9383194820411948 q9 6603641363956395 q10 6953231723072307 q11 460 254 260 254 q12 397 213 212 212 q13 17789 299429922992 q14 232 211 217 211 q15 556 538 500 500 q16 660 598 590 590 q17 973 488 574 488 q18 7368664065886588 q19 39901085982 982 q20 487 199 197 197 q21 3909329732593259 q22 10901013981 981 Total cold run time: 110848 ms Total hot run time: 40779 ms - Round 2, with runtime_filter_mode=off - q1 7483730372907290 q2 392 314 308 308 q3 3055294529532945 q4 2054178617431743 q5 5686577657275727 q6 224 144 144 144 q7 2232177217691769 q8 3403348734543454 q9 8945883188288828 q10 3548351035203510 q11 592 485 501 485 q12 828 613 584 584 q13 16773 313331813133 q14 302 262 277 262 q15 604 568 555 555 q16 723 661 671 661 q17 1868166116571657 q18 8213784275287528 q19 4790162215871587 q20 2058186318101810 q21 5312529952385238 q22 1127102710251025 Total cold run time: 80212 ms Total hot run time: 60243 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [branch-3.0](binlog) Support drop view binlog (#39781) [doris]
doris-robot commented on PR #43403: URL: https://github.com/apache/doris/pull/43403#issuecomment-2466115604 TPC-DS: Total hot run time: 192748 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools TPC-DS sf100 test result on commit b7b25cb2b3d3c2a0db1ff9041a34040939644dcb, data reload: false query1 2509224522242224 query2 6854206819991999 query3 14432 10588 255 255 query4 33743 23502 23331 23331 query5 4963465 437 437 query6 277 187 183 183 query7 3968304 307 304 query8 294 229 216 216 query9 9638266726802667 query10 507 260 264 260 query11 18105 15165 15309 15165 query12 157 101 98 98 query13 1545430 403 403 query14 9795690172776901 query15 231 187 169 169 query16 7606503 568 503 query17 1443587 591 587 query18 1781612 636 612 query19 230 174 196 174 query20 126 113 112 112 query21 213 103 109 103 query22 4761406741614067 query23 34617 34034 33752 33752 query24 7636335534113355 query25 569 426 418 418 query26 750 183 192 183 query27 1807291 296 291 query28 4077251825022502 query29 701 459 436 436 query30 367 307 317 307 query31 1030799 774 774 query32 81 56 55 55 query33 517 297 280 280 query34 888 503 519 503 query35 857 738 726 726 query36 1072937 935 935 query37 116 72 74 72 query38 3993386838503850 query39 1537142014131413 query40 201 104 100 100 query41 53 53 53 53 query42 105 96 100 96 query43 551 499 483 483 query44 1149785 791 785 query45 182 165 166 165 query46 1119703 731 703 query47 1924179518471795 query48 446 370 366 366 query49 752 393 389 389 query50 800 403 413 403 query51 7269729469996999 query52 99 98 87 87 query53 251 187 184 184 query54 562 451 450 450 query55 79 76 76 76 query56 254 261 237 237 query57 1195109710951095 query58 215 200 209 200 query59 3132297328312831 query60 282 250 248 248 query61 114 109 112 109 query62 777 668 657 657 query63 199 192 201 192 query64 3231646 646 646 query65 3234315931913159 query66 783 312 301 301 query67 16107 15343 15445 15343 query68 4531581 566 566 query69 428 252 265 252 query70 1169107911361079 query71 373 256 269 256 query72 6302391340863913 query73 754 337 348 337 query74 10226 901490679014 query75 3311264226412641 query76 2342958 858 858 query77 437 277 293 277 query78 10662 973094629462 query79 8450591 585 585 query80 2049411 421 411 query81 558 252 240 240 query82 1680119 111 111 query83 281 158 155 155 query84 292 82 78 78 query85 1726308 295 295 query86 476 309 296 296 query87 4425420542194205 query88 5470235323832353 query89 583 286 291 286 query90 2250182 186 182 query91 177 141 142 141 query92 68 46 48 46 query93 7939542 527 527 query94 895 266 301 266 query95 351 252 254 252 query96 633 286 285 285 query97 3403312131593121 query98 212 198 205 198 query99 1618128613201286 Total cold run time: 312929 ms Total hot run time: 192748 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to
Re: [PR] [Enhancement] (nereids)implement alterRoleCommand in nereids [doris]
Vallishp commented on PR #43238: URL: https://github.com/apache/doris/pull/43238#issuecomment-2466135274 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [feat](restore) Support compressed snapshot meta and job info [doris]
924060929 commented on code in PR #43516:
URL: https://github.com/apache/doris/pull/43516#discussion_r1835335349
##
fe/fe-common/src/main/java/org/apache/doris/common/GZIPUtils.java:
##
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.common;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+public class GZIPUtils {
+public static boolean isGZIPCompressed(byte[] data) {
+// From RFC 1952: 3.2. Members with a deflate compressed data stream
(ID1 = 8, ID2 = 8)
+return data.length >= 2 && data[0] == (byte) 0x1F && data[1] == (byte)
0x8B;
+}
+
+public static byte[] compress(byte[] data) throws IOException {
+ByteArrayOutputStream bytesStream = new ByteArrayOutputStream();
+try (GZIPOutputStream gzipStream = new GZIPOutputStream(bytesStream)) {
+gzipStream.write(data);
+}
+return bytesStream.toByteArray();
+}
+
+public static byte[] decompress(byte[] data) throws IOException {
+ByteArrayInputStream bytesStream = new ByteArrayInputStream(data);
+try (GZIPInputStream gzipStream = new GZIPInputStream(bytesStream)) {
+return gzipStream.readAllBytes();
+}
+}
Review Comment:
This grammar not support java8.
I think you should change to
```java
import org.apache.commons.io.IOUtils;
public static byte[] decompress(byte[] data) throws IOException {
ByteArrayInputStream bytesStream = new ByteArrayInputStream(data);
try (GZIPInputStream gzipStream = new GZIPInputStream(bytesStream)) {
return IOUtils.toByteArray(gzipStream);
}
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](nereids)Solve the problem of pruning wrong partitions in multi-column partition pruning [doris]
feiniaofeiafei commented on PR #43332: URL: https://github.com/apache/doris/pull/43332#issuecomment-2466162905 run p0 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [improvement](paimon)Using table serialization on the jni side #43167 [doris]
morningman merged PR #43474: URL: https://github.com/apache/doris/pull/43474 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-3.0 updated: [improvement](paimon)Using table serialization on the jni side #43167 (#43474)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new d32d2603bbc [improvement](paimon)Using table serialization on the jni
side #43167 (#43474)
d32d2603bbc is described below
commit d32d2603bbc07d247a2b04ef12bfd04dde699198
Author: Mingyu Chen (Rayner)
AuthorDate: Sat Nov 9 18:58:40 2024 +0800
[improvement](paimon)Using table serialization on the jni side #43167
(#43474)
cherry pick from #43167
Co-authored-by: wuwenchi
Co-authored-by: wuwenchi
---
be/src/vec/exec/format/table/paimon_jni_reader.cpp | 3 ++
.../org/apache/doris/paimon/PaimonJniScanner.java | 32 --
.../datasource/paimon/source/PaimonScanNode.java | 1 +
gensrc/thrift/PlanNodes.thrift | 17 ++--
4 files changed, 36 insertions(+), 17 deletions(-)
diff --git a/be/src/vec/exec/format/table/paimon_jni_reader.cpp
b/be/src/vec/exec/format/table/paimon_jni_reader.cpp
index a9ec243cf46..30358eace1a 100644
--- a/be/src/vec/exec/format/table/paimon_jni_reader.cpp
+++ b/be/src/vec/exec/format/table/paimon_jni_reader.cpp
@@ -61,6 +61,9 @@ PaimonJniReader::PaimonJniReader(const
std::vector& file_slot_d
std::to_string(range.table_format_params.paimon_params.last_update_time);
params["required_fields"] = join(column_names, ",");
params["columns_types"] = join(column_types, "#");
+if (range.table_format_params.paimon_params.__isset.paimon_table) {
+params["paimon_table"] =
range.table_format_params.paimon_params.paimon_table;
+}
// Used to create paimon option
for (auto& kv : range.table_format_params.paimon_params.paimon_options) {
diff --git
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
index 7bd9fa631c8..e85d465f663 100644
---
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
+++
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
@@ -42,13 +42,19 @@ import java.util.stream.Collectors;
public class PaimonJniScanner extends JniScanner {
private static final Logger LOG =
LoggerFactory.getLogger(PaimonJniScanner.class);
+@Deprecated
private static final String PAIMON_OPTION_PREFIX = "paimon.";
+@Deprecated
private static final String HADOOP_OPTION_PREFIX = "hadoop.";
private final Map params;
+@Deprecated
private final Map paimonOptionParams;
+@Deprecated
private final Map hadoopOptionParams;
+@Deprecated
private final String dbName;
+@Deprecated
private final String tblName;
private final String paimonSplit;
private final String paimonPredicate;
@@ -58,9 +64,13 @@ public class PaimonJniScanner extends JniScanner {
private List paimonAllFieldNames;
private List paimonDataTypeList;
+@Deprecated
private long ctlId;
+@Deprecated
private long dbId;
+@Deprecated
private long tblId;
+@Deprecated
private long lastUpdateTime;
private RecordReader.RecordIterator recordIterator = null;
private final ClassLoader classLoader;
@@ -214,16 +224,20 @@ public class PaimonJniScanner extends JniScanner {
}
private void initTable() {
-PaimonTableCacheKey key = new PaimonTableCacheKey(ctlId, dbId, tblId,
-paimonOptionParams, hadoopOptionParams, dbName, tblName);
-TableExt tableExt = PaimonTableCache.getTable(key);
-if (tableExt.getCreateTime() < lastUpdateTime) {
-LOG.warn("invalidate cache table:{}, localTime:{}, remoteTime:{}",
key, tableExt.getCreateTime(),
-lastUpdateTime);
-PaimonTableCache.invalidateTableCache(key);
-tableExt = PaimonTableCache.getTable(key);
+if (params.containsKey("paimon_table")) {
+table = PaimonUtils.deserialize(params.get("paimon_table"));
+} else {
+PaimonTableCacheKey key = new PaimonTableCacheKey(ctlId, dbId,
tblId,
+paimonOptionParams, hadoopOptionParams, dbName, tblName);
+TableExt tableExt = PaimonTableCache.getTable(key);
+if (tableExt.getCreateTime() < lastUpdateTime) {
+LOG.warn("invalidate cache table:{}, localTime:{},
remoteTime:{}", key, tableExt.getCreateTime(),
+lastUpdateTime);
+PaimonTableCache.invalidateTableCache(key);
+tableExt = PaimonTableCache.getTable(key);
+}
+this.table = tableExt.getTable();
}
-this.table = tableExt.getTable();
paimonAllFieldNames = PaimonUtils.getFieldNames(this.table.
Re: [PR] [fix](scan) Avoid memory allocated by buffered_reader from being traced (#41921) [doris]
morningman merged PR #43478: URL: https://github.com/apache/doris/pull/43478 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-3.0 updated: [fix](scan) Avoid memory allocated by buffered_reader from being traced (#41921) (#43478)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 9f28b4fdcf2 [fix](scan) Avoid memory allocated by buffered_reader from
being traced (#41921) (#43478)
9f28b4fdcf2 is described below
commit 9f28b4fdcf229aca01ad4e04f9d1f7d4fc8df20c
Author: Jerry Hu
AuthorDate: Sat Nov 9 19:01:56 2024 +0800
[fix](scan) Avoid memory allocated by buffered_reader from being traced
(#41921) (#43478)
Pick #41921
---
be/src/io/fs/buffered_reader.cpp | 19 +++
be/src/io/fs/buffered_reader.h | 11 +++
be/src/util/slice.h | 6 ++
3 files changed, 20 insertions(+), 16 deletions(-)
diff --git a/be/src/io/fs/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp
index 62c0a9c7a0c..20d5684734e 100644
--- a/be/src/io/fs/buffered_reader.cpp
+++ b/be/src/io/fs/buffered_reader.cpp
@@ -23,6 +23,7 @@
#include
#include
+#include
#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/config.h"
@@ -31,6 +32,7 @@
#include "runtime/thread_context.h"
#include "runtime/workload_management/io_throttle.h"
#include "util/runtime_profile.h"
+#include "util/slice.h"
#include "util/threadpool.h"
namespace doris {
@@ -270,7 +272,7 @@ void MergeRangeFileReader::_read_in_box(RangeCachedData&
cached_data, size_t off
}
if (copy_out != nullptr) {
memcpy(copy_out + to_handle - remaining,
- _boxes[box_index] + cached_data.box_start_offset[i],
box_to_handle);
+ _boxes[box_index].data() +
cached_data.box_start_offset[i], box_to_handle);
}
remaining -= box_to_handle;
cached_data.box_start_offset[i] += box_to_handle;
@@ -307,14 +309,15 @@ void MergeRangeFileReader::_read_in_box(RangeCachedData&
cached_data, size_t off
Status MergeRangeFileReader::_fill_box(int range_index, size_t start_offset,
size_t to_read,
size_t* bytes_read, const IOContext*
io_ctx) {
-if (_read_slice == nullptr) {
-_read_slice = new char[READ_SLICE_SIZE];
+if (!_read_slice) {
+_read_slice = std::make_unique(READ_SLICE_SIZE);
}
+
*bytes_read = 0;
{
SCOPED_RAW_TIMER(&_statistics.read_time);
-RETURN_IF_ERROR(
-_reader->read_at(start_offset, Slice(_read_slice, to_read),
bytes_read, io_ctx));
+RETURN_IF_ERROR(_reader->read_at(start_offset,
Slice(_read_slice->data(), to_read),
+ bytes_read, io_ctx));
_statistics.merged_io++;
_statistics.merged_bytes += *bytes_read;
}
@@ -328,8 +331,8 @@ Status MergeRangeFileReader::_fill_box(int range_index,
size_t start_offset, siz
auto fill_box = [&](int16 fill_box_ref, uint32 box_usage, size_t
box_copy_end) {
size_t copy_size = std::min(box_copy_end - copy_start, BOX_SIZE -
box_usage);
-memcpy(_boxes[fill_box_ref] + box_usage, _read_slice + copy_start -
start_offset,
- copy_size);
+memcpy(_boxes[fill_box_ref].data() + box_usage,
+ _read_slice->data() + copy_start - start_offset, copy_size);
filled_boxes.emplace_back(fill_box_ref, box_usage, copy_start,
copy_start + copy_size);
copy_start += copy_size;
_last_box_ref = fill_box_ref;
@@ -367,7 +370,7 @@ Status MergeRangeFileReader::_fill_box(int range_index,
size_t start_offset, siz
}
// apply for new box to copy data
while (copy_start < range_copy_end && _boxes.size() < NUM_BOX) {
-_boxes.emplace_back(new char[BOX_SIZE]);
+_boxes.emplace_back(BOX_SIZE);
_box_ref.emplace_back(0);
fill_box(_boxes.size() - 1, 0, range_copy_end);
}
diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h
index 70c8445db23..907ea11b216 100644
--- a/be/src/io/fs/buffered_reader.h
+++ b/be/src/io/fs/buffered_reader.h
@@ -168,12 +168,7 @@ public:
}
}
-~MergeRangeFileReader() override {
-delete[] _read_slice;
-for (char* box : _boxes) {
-delete[] box;
-}
-}
+~MergeRangeFileReader() override = default;
Status close() override {
if (!_closed) {
@@ -244,8 +239,8 @@ private:
bool _closed = false;
size_t _remaining;
-char* _read_slice = nullptr;
-std::vector _boxes;
+std::unique_ptr _read_slice;
+std::vector _boxes;
int16 _last_box_ref = -1;
uint32 _last_box_usage = 0;
std::vector _box_ref;
diff --git a/be/src/util/slice.h b/be/src/util/slice.h
index b38b1147894..fd6bcf0adfb 100644
--- a/be/src/util/slice.h
+++ b/be/src/util/slice.h
@@ -344,6 +344,10 @@ class OwnedSlice : private Allocator(Allocator::alloc(length)
Re: [PR] [Opt](orc)Optimize the merge io when orc reader read multiple tiny stripes. (#42004) [doris]
morningman commented on PR #43467: URL: https://github.com/apache/doris/pull/43467#issuecomment-2466173123 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Refactor] (inverted index) Refactor Inverted index file writer (#41625) [doris]
csun5285 commented on PR #43528: URL: https://github.com/apache/doris/pull/43528#issuecomment-2466616989 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](arrow-flight-sql) Arrow flight server supports data forwarding when BE uses public vip [doris]
xinyiZzz commented on code in PR #43281:
URL: https://github.com/apache/doris/pull/43281#discussion_r1835605743
##
be/src/service/arrow_flight/arrow_flight_batch_reader.cpp:
##
@@ -18,52 +18,278 @@
#include "service/arrow_flight/arrow_flight_batch_reader.h"
#include
+#include
+#include
+
+#include
-#include "arrow/builder.h"
#include "runtime/exec_env.h"
+#include "runtime/memory/mem_tracker_limiter.h"
#include "runtime/result_buffer_mgr.h"
+#include "runtime/thread_context.h"
+#include "service/backend_options.h"
+#include "util/arrow/block_convertor.h"
#include "util/arrow/row_batch.h"
#include "util/arrow/utils.h"
+#include "util/brpc_client_cache.h"
+#include "util/ref_count_closure.h"
+#include "util/string_util.h"
+#include "vec/core/block.h"
+
+namespace doris::flight {
+
+constexpr size_t BRPC_CONTROLLER_TIMEOUT_MS = 60 * 1000;
+
+ArrowFlightBatchReaderBase::ArrowFlightBatchReaderBase(
+const std::shared_ptr& statement)
+: _statement(statement) {}
+
+std::shared_ptr ArrowFlightBatchReaderBase::schema() const {
+return _schema;
+}
-namespace doris {
-namespace flight {
+arrow::Status ArrowFlightBatchReaderBase::_return_invalid_status(const
std::string& msg) {
+std::string status_msg =
+fmt::format("ArrowFlightBatchReader {}, packet_seq={},
result={}:{}, finistId={}", msg,
+_packet_seq, _statement->result_addr.hostname,
_statement->result_addr.port,
+print_id(_statement->query_id));
+LOG(WARNING) << status_msg;
+return arrow::Status::Invalid(status_msg);
+}
-std::shared_ptr ArrowFlightBatchReader::schema() const {
-return schema_;
+ArrowFlightBatchReaderBase::~ArrowFlightBatchReaderBase() {
+#ifndef NDEBUG
+LOG(INFO) << fmt::format(
+"ArrowFlightBatchReader finished, packet_seq={},
result_addr={}:{}, finistId={}, "
Review Comment:
改成了 vlog
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](arrow-flight-sql) Arrow flight server supports data forwarding when BE uses public vip [doris]
xinyiZzz commented on PR #43281: URL: https://github.com/apache/doris/pull/43281#issuecomment-2466616332 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](arrow-flight-sql) Arrow flight server supports data forwarding when BE uses public vip [doris]
github-actions[bot] commented on code in PR #43281:
URL: https://github.com/apache/doris/pull/43281#discussion_r1835606862
##
be/src/service/arrow_flight/arrow_flight_batch_reader.cpp:
##
@@ -18,52 +18,276 @@
#include "service/arrow_flight/arrow_flight_batch_reader.h"
#include
+#include
+#include
+
+#include
-#include "arrow/builder.h"
#include "runtime/exec_env.h"
+#include "runtime/memory/mem_tracker_limiter.h"
#include "runtime/result_buffer_mgr.h"
+#include "runtime/thread_context.h"
+#include "service/backend_options.h"
+#include "util/arrow/block_convertor.h"
#include "util/arrow/row_batch.h"
#include "util/arrow/utils.h"
+#include "util/brpc_client_cache.h"
+#include "util/ref_count_closure.h"
+#include "util/string_util.h"
+#include "vec/core/block.h"
+
+namespace doris::flight {
+
+constexpr size_t BRPC_CONTROLLER_TIMEOUT_MS = 60 * 1000;
+
+ArrowFlightBatchReaderBase::ArrowFlightBatchReaderBase(
+const std::shared_ptr& statement)
+: _statement(statement) {}
+
+std::shared_ptr ArrowFlightBatchReaderBase::schema() const {
+return _schema;
+}
-namespace doris {
-namespace flight {
+arrow::Status ArrowFlightBatchReaderBase::_return_invalid_status(const
std::string& msg) {
+std::string status_msg =
+fmt::format("ArrowFlightBatchReader {}, packet_seq={},
result={}:{}, finistId={}", msg,
+_packet_seq, _statement->result_addr.hostname,
_statement->result_addr.port,
+print_id(_statement->query_id));
+LOG(WARNING) << status_msg;
+return arrow::Status::Invalid(status_msg);
+}
-std::shared_ptr ArrowFlightBatchReader::schema() const {
-return schema_;
+ArrowFlightBatchReaderBase::~ArrowFlightBatchReaderBase() {
+VLOG_NOTICE << fmt::format(
+"ArrowFlightBatchReader finished, packet_seq={},
result_addr={}:{}, finistId={}, "
+"convert_arrow_batch_timer={}, deserialize_block_timer={},
peak_memory_usage={}",
+_packet_seq, _statement->result_addr.hostname,
_statement->result_addr.port,
+print_id(_statement->query_id), _convert_arrow_batch_timer,
_deserialize_block_timer,
+_mem_tracker->peak_consumption());
}
-ArrowFlightBatchReader::ArrowFlightBatchReader(std::shared_ptr
statement,
- std::shared_ptr
schema)
-: statement_(std::move(statement)), schema_(std::move(schema)) {}
+ArrowFlightBatchLocalReader::ArrowFlightBatchLocalReader(
+const std::shared_ptr& statement,
+const std::shared_ptr& schema,
+const std::shared_ptr& mem_tracker)
+: ArrowFlightBatchReaderBase(statement) {
+_schema = schema;
+_mem_tracker = mem_tracker;
+}
-arrow::Result>
ArrowFlightBatchReader::Create(
-const std::shared_ptr& statement_) {
+arrow::Result>
ArrowFlightBatchLocalReader::Create(
+const std::shared_ptr& statement) {
+DCHECK(statement->result_addr.hostname == BackendOptions::get_localhost());
// Make sure that FE send the fragment to BE and creates the
BufferControlBlock before returning ticket
// to the ADBC client, so that the schema and control block can be found.
-auto schema =
ExecEnv::GetInstance()->result_mgr()->find_arrow_schema(statement_->query_id);
-if (schema == nullptr) {
-ARROW_RETURN_NOT_OK(arrow::Status::Invalid(fmt::format(
-"Client not found arrow flight schema, maybe query has been
canceled, queryid: {}",
-print_id(statement_->query_id;
+std::shared_ptr schema;
+RETURN_ARROW_STATUS_IF_ERROR(
+
ExecEnv::GetInstance()->result_mgr()->find_arrow_schema(statement->query_id,
&schema));
+std::shared_ptr mem_tracker;
+
RETURN_ARROW_STATUS_IF_ERROR(ExecEnv::GetInstance()->result_mgr()->find_mem_tracker(
+statement->query_id, &mem_tracker));
+
+std::shared_ptr result(
+new ArrowFlightBatchLocalReader(statement, schema, mem_tracker));
+return result;
+}
+
+arrow::Status
ArrowFlightBatchLocalReader::ReadNext(std::shared_ptr* out)
{
+// parameter *out not nullptr
+*out = nullptr;
+SCOPED_ATTACH_TASK(_mem_tracker);
+std::shared_ptr result;
+auto st =
ExecEnv::GetInstance()->result_mgr()->fetch_arrow_data(_statement->query_id,
&result,
+
_timezone_obj);
+st.prepend("ArrowFlightBatchLocalReader fetch arrow data failed");
+ARROW_RETURN_NOT_OK(to_arrow_status(st));
+if (result == nullptr) {
+// eof, normal path end
+return arrow::Status::OK();
+}
+
+{
+// convert one batch
+SCOPED_ATOMIC_TIMER(&_convert_arrow_batch_timer);
+st = convert_to_arrow_batch(*result, _schema,
arrow::default_memory_pool(), out,
+_timezone_obj);
+st.prepend("ArrowFlightBatchLocalReader convert block to arrow batch
faile
Re: [PR] [opt](http) enable auth token with BE http request [doris]
doris-robot commented on PR #41994: URL: https://github.com/apache/doris/pull/41994#issuecomment-2466621286 TeamCity be ut coverage result: Function Coverage: 37.91% (9868/26029) Line Coverage: 29.11% (82315/282803) Region Coverage: 28.25% (42359/149942) Branch Coverage: 24.80% (21451/86500) Coverage Report: http://coverage.selectdb-in.cc/coverage/8fe9ba30c4c51d151a4081261a22d46bc2caac5a_8fe9ba30c4c51d151a4081261a22d46bc2caac5a/report/index.html -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [improvement](build index)Optimize failed task check on same tablet [doris]
qidaye commented on PR #42295: URL: https://github.com/apache/doris/pull/42295#issuecomment-2466624661 run performance -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](http) enable auth token with BE http request [doris]
morningman commented on PR #41994: URL: https://github.com/apache/doris/pull/41994#issuecomment-2466594620 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](arrow-flight-sql) Arrow flight server supports data forwarding when BE uses public vip [doris]
xinyiZzz commented on code in PR #43281:
URL: https://github.com/apache/doris/pull/43281#discussion_r1835597650
##
be/src/service/arrow_flight/arrow_flight_batch_reader.cpp:
##
@@ -18,52 +18,278 @@
#include "service/arrow_flight/arrow_flight_batch_reader.h"
#include
+#include
+#include
+
+#include
-#include "arrow/builder.h"
#include "runtime/exec_env.h"
+#include "runtime/memory/mem_tracker_limiter.h"
#include "runtime/result_buffer_mgr.h"
+#include "runtime/thread_context.h"
+#include "service/backend_options.h"
+#include "util/arrow/block_convertor.h"
#include "util/arrow/row_batch.h"
#include "util/arrow/utils.h"
+#include "util/brpc_client_cache.h"
+#include "util/ref_count_closure.h"
+#include "util/string_util.h"
+#include "vec/core/block.h"
+
+namespace doris::flight {
+
+constexpr size_t BRPC_CONTROLLER_TIMEOUT_MS = 60 * 1000;
+
+ArrowFlightBatchReaderBase::ArrowFlightBatchReaderBase(
+const std::shared_ptr& statement)
+: _statement(statement) {}
+
+std::shared_ptr ArrowFlightBatchReaderBase::schema() const {
+return _schema;
+}
-namespace doris {
-namespace flight {
+arrow::Status ArrowFlightBatchReaderBase::_return_invalid_status(const
std::string& msg) {
+std::string status_msg =
+fmt::format("ArrowFlightBatchReader {}, packet_seq={},
result={}:{}, finistId={}", msg,
+_packet_seq, _statement->result_addr.hostname,
_statement->result_addr.port,
+print_id(_statement->query_id));
+LOG(WARNING) << status_msg;
+return arrow::Status::Invalid(status_msg);
+}
-std::shared_ptr ArrowFlightBatchReader::schema() const {
-return schema_;
+ArrowFlightBatchReaderBase::~ArrowFlightBatchReaderBase() {
+#ifndef NDEBUG
+LOG(INFO) << fmt::format(
+"ArrowFlightBatchReader finished, packet_seq={},
result_addr={}:{}, finistId={}, "
Review Comment:
没有 LOG(DEBUG) 这个用法，除非用
VLOG，但VLOG还要改conf。。预期是不在线上打印，避免压测时刷太多日志，但平时跑流水线能打印帮助定位问题
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](arrow-flight-sql) Arrow flight server supports data forwarding when BE uses public vip [doris]
xinyiZzz commented on code in PR #43281:
URL: https://github.com/apache/doris/pull/43281#discussion_r1835597583
##
be/src/vec/sink/varrow_flight_result_writer.cpp:
##
@@ -67,29 +60,31 @@ Status VArrowFlightResultWriter::write(RuntimeState* state,
Block& input_block)
RETURN_IF_ERROR(VExprContext::get_output_block_after_execute_exprs(_output_vexpr_ctxs,
input_block, &block));
-// convert one batch
-std::shared_ptr result;
-auto num_rows = block.rows();
-// arrow::RecordBatch without `nbytes()` in C++
-uint64_t bytes_sent = block.bytes();
{
-SCOPED_TIMER(_convert_tuple_timer);
-RETURN_IF_ERROR(convert_to_arrow_batch(block, _arrow_schema,
arrow::default_memory_pool(),
- &result, _timezone_obj));
-}
-{
-SCOPED_TIMER(_result_send_timer);
-// If this is a dry run task, no need to send data block
-if (!_is_dry_run) {
-status = _sinker->add_arrow_batch(state, result);
-}
-if (status.ok()) {
-_written_rows += num_rows;
+SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_sinker->mem_tracker());
Review Comment:
因为会拷贝一份Block放入 buffer control block 的 result queue 里，这个新 Block 的生命周期不属于
query，属于 buffer control block，会在 fetch queue 时释放
所以这里 swtich 了 buffer control block 的 mem tracker
##
gensrc/proto/internal_service.proto:
##
@@ -283,6 +283,21 @@ message PFetchDataResult {
optional bool empty_batch = 6;
};
+message PFetchArrowDataRequest {
+optional PUniqueId finst_id = 1;
+};
+
+message PFetchArrowDataResult {
+required PStatus status = 1;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [improvement](build index)Optimize failed task check on same tablet [doris]
qidaye commented on PR #42295: URL: https://github.com/apache/doris/pull/42295#issuecomment-2466590405 run performance -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris-website) branch asf-site updated (410988085bc -> 4072ab3af53)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/doris-website.git discard 410988085bc Automated deployment with doris branch @ 8392376723a884f6649d19ad0749b3191a66a71c new 4072ab3af53 Automated deployment with doris branch @ 8392376723a884f6649d19ad0749b3191a66a71c This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (410988085bc) \ N -- N -- N refs/heads/asf-site (4072ab3af53) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: docs/1.2/search-index.json | 2 +- docs/2.0/search-index.json | 2 +- docs/3.0/search-index.json | 2 +- docs/dev/search-index.json | 2 +- search-index.json| 2 +- zh-CN/docs/1.2/search-index.json | 2 +- zh-CN/docs/2.0/search-index.json | 2 +- zh-CN/docs/3.0/search-index.json | 2 +- zh-CN/docs/dev/search-index.json | 2 +- zh-CN/search-index.json | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[PR] [Enhancement] replace table options to keep original dropped table in recycle bin [doris-website]
Vallishp opened a new pull request, #1323: URL: https://github.com/apache/doris-website/pull/1323 … # Versions - [ ] dev // dev version - [ ] 3.0 - [ ] 2.1 - [ ] 2.0 # Languages - [ ] Chinese // need update - [ ] English // updated -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-2.1: [Improvement](LDAP Auth)Enhance LDAP authentication with a configurable group filter [doris]
yiguolei closed pull request #43293: branch-2.1: [Improvement](LDAP Auth)Enhance LDAP authentication with a configurable group filter URL: https://github.com/apache/doris/pull/43293 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-3.0: [enhance](mtmv)MTMV interface optimization [doris]
yiguolei merged PR #43328: URL: https://github.com/apache/doris/pull/43328 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-3.0 updated: branch-3.0: [enhance](mtmv)MTMV interface optimization (#43328)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 1a6a0a67008 branch-3.0: [enhance](mtmv)MTMV interface optimization
(#43328)
1a6a0a67008 is described below
commit 1a6a0a670088e259af213f253307c53695425926
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Nov 10 10:08:01 2024 +0800
branch-3.0: [enhance](mtmv)MTMV interface optimization (#43328)
Cherry-picked from #43086
Co-authored-by: zhangdong <493738...@qq.com>
---
.../java/org/apache/doris/catalog/OlapTable.java | 5 ---
.../doris/datasource/hive/HMSExternalTable.java| 22 +++--
.../apache/doris/job/extensions/mtmv/MTMVTask.java | 18 +--
.../org/apache/doris/mtmv/MTMVBaseTableIf.java | 36 ++
.../org/apache/doris/mtmv/MTMVRelatedTableIf.java | 16 --
5 files changed, 70 insertions(+), 27 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 3aa8f61e905..2f8f0eb7909 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -3164,11 +3164,6 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
return new MTMVVersionSnapshot(visibleVersion, id);
}
-@Override
-public boolean needAutoRefresh() {
-return true;
-}
-
@Override
public boolean isPartitionColumnAllowNull() {
return true;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index b48b47acf13..445aae09f0d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -20,6 +20,7 @@ package org.apache.doris.datasource.hive;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.ListPartitionItem;
+import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.PrimitiveType;
@@ -27,10 +28,12 @@ import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
+import org.apache.doris.common.DdlException;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.SchemaCacheValue;
import org.apache.doris.datasource.hudi.HudiUtils;
import org.apache.doris.datasource.iceberg.IcebergUtils;
+import org.apache.doris.mtmv.MTMVBaseTableIf;
import org.apache.doris.mtmv.MTMVMaxTimestampSnapshot;
import org.apache.doris.mtmv.MTMVRefreshContext;
import org.apache.doris.mtmv.MTMVRelatedTableIf;
@@ -87,7 +90,7 @@ import java.util.stream.Collectors;
/**
* Hive metastore external table.
*/
-public class HMSExternalTable extends ExternalTable implements
MTMVRelatedTableIf {
+public class HMSExternalTable extends ExternalTable implements
MTMVRelatedTableIf, MTMVBaseTableIf {
private static final Logger LOG =
LogManager.getLogger(HMSExternalTable.class);
public static final Set SUPPORTED_HIVE_FILE_FORMATS;
@@ -586,9 +589,9 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
case ICEBERG:
if (GlobalVariable.enableFetchIcebergStats) {
return StatisticsUtil.getIcebergColumnStats(colName,
-
Env.getCurrentEnv().getExtMetaCacheMgr().getIcebergMetadataCache().getIcebergTable(
-catalog, dbName, name
-));
+
Env.getCurrentEnv().getExtMetaCacheMgr().getIcebergMetadataCache().getIcebergTable(
+catalog, dbName, name
+));
} else {
break;
}
@@ -819,11 +822,6 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
return partition;
}
-@Override
-public boolean needAutoRefresh() {
-return true;
-}
-
@Override
public boolean isPartitionColumnAllowNull() {
return true;
@@ -958,4 +956,10 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
makeSureInitialized();
return !isView() && remoteTable.getPartitionKeysSize() > 0;
}
+
+@Override
+public void beforeMTMVRefresh(MTMV mtmv) throws DdlException {
+Env.ge
(doris) branch branch-2.1 updated: branch-2.1: [enhance](mtmv)MTMV interface optimization (#43329)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new ea67e3a6b47 branch-2.1: [enhance](mtmv)MTMV interface optimization
(#43329)
ea67e3a6b47 is described below
commit ea67e3a6b474d9f10ead11fd2ea1d6c99e32a187
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Nov 10 10:07:35 2024 +0800
branch-2.1: [enhance](mtmv)MTMV interface optimization (#43329)
Cherry-picked from #43086
Co-authored-by: zhangdong <493738...@qq.com>
---
.../java/org/apache/doris/catalog/OlapTable.java | 5 ---
.../doris/datasource/hive/HMSExternalTable.java| 22 +++--
.../apache/doris/job/extensions/mtmv/MTMVTask.java | 18 +--
.../org/apache/doris/mtmv/MTMVBaseTableIf.java | 36 ++
.../org/apache/doris/mtmv/MTMVRelatedTableIf.java | 16 --
5 files changed, 70 insertions(+), 27 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 123412827a1..37e5f265bd6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -3075,11 +3075,6 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf {
return new MTMVVersionSnapshot(visibleVersion, id);
}
-@Override
-public boolean needAutoRefresh() {
-return true;
-}
-
@Override
public boolean isPartitionColumnAllowNull() {
return true;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index 06f1635e2fb..aaa59d44144 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -20,6 +20,7 @@ package org.apache.doris.datasource.hive;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.ListPartitionItem;
+import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.PrimitiveType;
@@ -27,10 +28,12 @@ import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
+import org.apache.doris.common.DdlException;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.SchemaCacheValue;
import org.apache.doris.datasource.hudi.HudiUtils;
import org.apache.doris.datasource.iceberg.IcebergUtils;
+import org.apache.doris.mtmv.MTMVBaseTableIf;
import org.apache.doris.mtmv.MTMVMaxTimestampSnapshot;
import org.apache.doris.mtmv.MTMVRefreshContext;
import org.apache.doris.mtmv.MTMVRelatedTableIf;
@@ -87,7 +90,7 @@ import java.util.stream.Collectors;
/**
* Hive metastore external table.
*/
-public class HMSExternalTable extends ExternalTable implements
MTMVRelatedTableIf {
+public class HMSExternalTable extends ExternalTable implements
MTMVRelatedTableIf, MTMVBaseTableIf {
private static final Logger LOG =
LogManager.getLogger(HMSExternalTable.class);
public static final Set SUPPORTED_HIVE_FILE_FORMATS;
@@ -587,9 +590,9 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
case ICEBERG:
if (GlobalVariable.enableFetchIcebergStats) {
return StatisticsUtil.getIcebergColumnStats(colName,
-
Env.getCurrentEnv().getExtMetaCacheMgr().getIcebergMetadataCache().getIcebergTable(
-catalog, dbName, name
-));
+
Env.getCurrentEnv().getExtMetaCacheMgr().getIcebergMetadataCache().getIcebergTable(
+catalog, dbName, name
+));
} else {
break;
}
@@ -821,11 +824,6 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
return partition;
}
-@Override
-public boolean needAutoRefresh() {
-return true;
-}
-
@Override
public boolean isPartitionColumnAllowNull() {
return true;
@@ -960,4 +958,10 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
makeSureInitialized();
return !isView() && remoteTable.getPartitionKeysSize() > 0;
}
+
+@Override
+public void beforeMTMVRefresh(MTMV mtmv) throws DdlException {
+Env.getCurrentEnv(
(doris) branch branch-2.1 updated: branch-2.1: [Improvement](LDAP Auth)Enhance LDAP authentication with a configurable group filter (#43293)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 80fd76677ed branch-2.1: [Improvement](LDAP Auth)Enhance LDAP
authentication with a configurable group filter (#43293)
80fd76677ed is described below
commit 80fd76677ed72390a093ccb742921bca4a7ee317
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Nov 10 10:06:13 2024 +0800
branch-2.1: [Improvement](LDAP Auth)Enhance LDAP authentication with a
configurable group filter (#43293)
Cherry-picked from #42038
Co-authored-by: nsivarajan <117266407+nsivara...@users.noreply.github.com>
Co-authored-by: Sivarajan Narayanan
---
conf/ldap.conf | 1 +
.../main/java/org/apache/doris/common/LdapConfig.java | 6 ++
.../doris/mysql/authenticate/ldap/LdapClient.java | 18 +-
3 files changed, 24 insertions(+), 1 deletion(-)
diff --git a/conf/ldap.conf b/conf/ldap.conf
index f783c53ea96..b501a729d7e 100644
--- a/conf/ldap.conf
+++ b/conf/ldap.conf
@@ -30,6 +30,7 @@
# ldap_user_basedn - Search base for users.
# ldap_user_filter - User lookup filter, the placeholder {login} will be
replaced by the user supplied login.
# ldap_group_basedn - Search base for groups.
+# ldap_group_filter - Group lookup filter, the placeholder {login} will be
replaced by the user supplied login. example : "(&(memberUid={login}))"
## step2: Restart fe, and use root or admin account to log in to doris.
## step3: Execute sql statement to set ldap admin password:
# set ldap_admin_password = 'password';
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/LdapConfig.java
b/fe/fe-common/src/main/java/org/apache/doris/common/LdapConfig.java
index a6fb10f261d..f174a4ef663 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/LdapConfig.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/LdapConfig.java
@@ -66,6 +66,12 @@ public class LdapConfig extends ConfigBase {
@ConfigBase.ConfField
public static String ldap_group_basedn = "";
+/**
+ * Group lookup filter, the placeholder {login} will be replaced by the
user supplied login.
+ */
+@ConfigBase.ConfField
+public static String ldap_group_filter = "";
+
/**
* The user LDAP information cache time.
* After timeout, the user information will be retrieved from the LDAP
service again.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapClient.java
b/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapClient.java
index bbb8bf4d378..8d1304658ff 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapClient.java
@@ -159,9 +159,21 @@ public class LdapClient {
if (userDn == null) {
return groups;
}
-List groupDns =
getDn(org.springframework.ldap.query.LdapQueryBuilder.query()
+List groupDns;
+
+// Support Open Directory implementations
+// If no group filter is configured, it defaults to querying groups
based on the attribute 'member'
+// for standard LDAP implementations
+if (!LdapConfig.ldap_group_filter.isEmpty()) {
+groupDns =
getDn(org.springframework.ldap.query.LdapQueryBuilder.query()
+.base(LdapConfig.ldap_group_basedn)
+.filter(getGroupFilter(LdapConfig.ldap_group_filter,
userName)));
+} else {
+groupDns =
getDn(org.springframework.ldap.query.LdapQueryBuilder.query()
.base(LdapConfig.ldap_group_basedn)
.where("member").is(userDn));
+}
+
if (groupDns == null) {
return groups;
}
@@ -209,4 +221,8 @@ public class LdapClient {
private String getUserFilter(String userFilter, String userName) {
return userFilter.replaceAll("\\{login}", userName);
}
+
+private String getGroupFilter(String groupFilter, String userName) {
+return groupFilter.replaceAll("\\{login}", userName);
+}
}
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-2.1: [Improvement](LDAP Auth)Enhance LDAP authentication with a configurable group filter [doris]
yiguolei merged PR #43293: URL: https://github.com/apache/doris/pull/43293 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-2.1 updated: [fix](planner) NullLiteral should always having a correct Type and set to be analyzed (#43371)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 182f37f8371 [fix](planner) NullLiteral should always having a correct
Type and set to be analyzed (#43371)
182f37f8371 is described below
commit 182f37f83716e2e8f4b8c60f9a49eb60ab585660
Author: starocean999 <40539150+starocean...@users.noreply.github.com>
AuthorDate: Sun Nov 10 10:10:40 2024 +0800
[fix](planner) NullLiteral should always having a correct Type and set to
be analyzed (#43371)
---
.../org/apache/doris/analysis/NullLiteral.java | 1 +
.../org/apache/doris/analysis/StringLiteral.java | 2 +-
regression-test/data/insert_p0/test_insert_nan.out | 4 +++
.../suites/insert_p0/test_insert_nan.groovy| 34 ++
4 files changed, 40 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java
index 665b5dd87c5..1ba5a43b4ee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java
@@ -53,6 +53,7 @@ public class NullLiteral extends LiteralExpr {
public static NullLiteral create(Type type) {
NullLiteral l = new NullLiteral();
l.type = type;
+l.analysisDone();
return l;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
index 9ebde03c132..2284e9e547c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
@@ -275,7 +275,7 @@ public class StringLiteral extends LiteralExpr {
return new FloatLiteral(Double.valueOf(value),
targetType);
} catch (NumberFormatException e) {
// consistent with CastExpr's getResultValue() method
-return new NullLiteral();
+return NullLiteral.create(targetType);
}
case DECIMALV2:
case DECIMAL32:
diff --git a/regression-test/data/insert_p0/test_insert_nan.out
b/regression-test/data/insert_p0/test_insert_nan.out
new file mode 100644
index 000..f0d5ba9c2d6
--- /dev/null
+++ b/regression-test/data/insert_p0/test_insert_nan.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select --
+1 \N
+
diff --git a/regression-test/suites/insert_p0/test_insert_nan.groovy
b/regression-test/suites/insert_p0/test_insert_nan.groovy
new file mode 100644
index 000..8650c5d0cf1
--- /dev/null
+++ b/regression-test/suites/insert_p0/test_insert_nan.groovy
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_insert_nan") {
+sql """drop table if exists `nan_table`;"""
+
+sql """
+CREATE TABLE `nan_table` (
+`id` int NOT NULL ,
+`val` double,
+) ENGINE=OLAP
+UNIQUE KEY(`id`)
+DISTRIBUTED BY HASH(`id`) BUCKETS 3
+PROPERTIES("replication_num" = "1");
+"""
+
+sql """insert into nan_table values ('1', 'nan');"""
+
+qt_select """select * from nan_table;"""
+}
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-2.1 updated: [fix](auditlog) set isQuery to true when query is short circuited (#42647) (#43345)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 1d740ff8256 [fix](auditlog) set isQuery to true when query is short
circuited (#42647) (#43345)
1d740ff8256 is described below
commit 1d740ff8256599b5b6529325e92799f706e0b8d0
Author: lihangyu <15605149...@163.com>
AuthorDate: Sun Nov 10 10:08:56 2024 +0800
[fix](auditlog) set isQuery to true when query is short circuited (#42647)
(#43345)
(#42647)
---
fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java | 2 ++
1 file changed, 2 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 4cab4220eef..831696d9ebb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -1846,10 +1846,12 @@ public class StmtExecutor {
: new ShortCircuitQueryContext(planner,
(Queriable) parsedStmt);
coordBase = new PointQueryExecutor(shortCircuitQueryContext,
context.getSessionVariable().getMaxMsgSizeOfResultReceiver());
+context.getState().setIsQuery(true);
} else if (queryStmt instanceof SelectStmt && ((SelectStmt)
parsedStmt).isPointQueryShortCircuit()) {
// this branch is for legacy planner, to be removed
coordBase = new PointQueryExec(planner, analyzer,
context.getSessionVariable().getMaxMsgSizeOfResultReceiver());
+context.getState().setIsQuery(true);
} else {
coord = new Coordinator(context, analyzer, planner,
context.getStatsErrorEstimator());
profile.addExecutionProfile(coord.getExecutionProfile());
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-2.1 updated: branch-2.1: [opt](max-compute) avoid repeated location path creation (#43383)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 5ac3aee460c branch-2.1: [opt](max-compute) avoid repeated location
path creation (#43383)
5ac3aee460c is described below
commit 5ac3aee460c43c6b5cd704df0358e6749bfea28c
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Nov 10 10:11:37 2024 +0800
branch-2.1: [opt](max-compute) avoid repeated location path creation
(#43383)
Cherry-picked from #43355
Co-authored-by: Mingyu Chen (Rayner)
---
.../doris/datasource/maxcompute/source/MaxComputeScanNode.java| 8 +---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
index 4ec4319c228..e0b84b0860e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
@@ -81,8 +81,10 @@ import java.util.stream.Collectors;
public class MaxComputeScanNode extends FileQueryScanNode {
private final MaxComputeExternalTable table;
-TableBatchReadSession tableBatchReadSession;
+private TableBatchReadSession tableBatchReadSession;
private Predicate filterPredicate;
+private static final LocationPath ROW_OFFSET_PATH = new
LocationPath("/row_offset", Maps.newHashMap());
+private static final LocationPath BYTE_SIZE_PATH = new
LocationPath("/byte_size", Maps.newHashMap());
public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, boolean
needCheckColumnPriv) {
this(id, desc, "MCScanNode", StatisticalType.MAX_COMPUTE_SCAN_NODE,
needCheckColumnPriv);
@@ -441,7 +443,7 @@ public class MaxComputeScanNode extends FileQueryScanNode {
for (com.aliyun.odps.table.read.split.InputSplit split :
assigner.getAllSplits()) {
MaxComputeSplit maxComputeSplit =
-new MaxComputeSplit(new LocationPath("/byte_size",
Maps.newHashMap()),
+new MaxComputeSplit(BYTE_SIZE_PATH,
((IndexedInputSplit)
split).getSplitIndex(), -1,
mcCatalog.getSplitByteSize(),
modificationTime, null,
@@ -464,7 +466,7 @@ public class MaxComputeScanNode extends FileQueryScanNode {
assigner.getSplitByRowOffset(offset,
recordsPerSplit);
MaxComputeSplit maxComputeSplit =
-new MaxComputeSplit(new
LocationPath("/row_offset", Maps.newHashMap()),
+new MaxComputeSplit(ROW_OFFSET_PATH,
offset, recordsPerSplit, totalRowCount,
modificationTime, null,
Collections.emptyList());
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](planner) NullLiteral should always having a correct Type and set to be analyzed [doris]
yiguolei merged PR #43371: URL: https://github.com/apache/doris/pull/43371 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-2.1: [opt](max-compute) avoid repeated location path creation [doris]
yiguolei merged PR #43383: URL: https://github.com/apache/doris/pull/43383 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-2.1: [enhance](mtmv)MTMV interface optimization [doris]
yiguolei merged PR #43329: URL: https://github.com/apache/doris/pull/43329 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [improvement](build index)Optimize failed task check on same tablet [doris]
qidaye commented on PR #42295: URL: https://github.com/apache/doris/pull/42295#issuecomment-2466558281 run performance -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated: [opt](profile) Add WaitWorkerTime into MergedProfile of pipeline task (#43350)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9c66acdaff0 [opt](profile) Add WaitWorkerTime into MergedProfile of
pipeline task (#43350)
9c66acdaff0 is described below
commit 9c66acdaff07e3b561719a8b5ae7faedf8ce416d
Author: zhiqiang
AuthorDate: Sun Nov 10 10:19:35 2024 +0800
[opt](profile) Add WaitWorkerTime into MergedProfile of pipeline task
(#43350)
---
be/src/pipeline/pipeline_task.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/be/src/pipeline/pipeline_task.cpp
b/be/src/pipeline/pipeline_task.cpp
index 400495437d4..6f9e59c8291 100644
--- a/be/src/pipeline/pipeline_task.cpp
+++ b/be/src/pipeline/pipeline_task.cpp
@@ -181,7 +181,7 @@ void PipelineTask::_init_profile() {
_sink_timer = ADD_CHILD_TIMER(_task_profile, "SinkTime", exec_time);
_close_timer = ADD_CHILD_TIMER(_task_profile, "CloseTime", exec_time);
-_wait_worker_timer = ADD_TIMER(_task_profile, "WaitWorkerTime");
+_wait_worker_timer = ADD_TIMER_WITH_LEVEL(_task_profile, "WaitWorkerTime",
1);
_schedule_counts = ADD_COUNTER(_task_profile, "NumScheduleTimes",
TUnit::UNIT);
_yield_counts = ADD_COUNTER(_task_profile, "NumYieldTimes", TUnit::UNIT);
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](auditlog) set isQuery to true when query is short circuited (#42647) [doris]
yiguolei merged PR #43345: URL: https://github.com/apache/doris/pull/43345 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](profile) Add WaitWorkerTime into MergedProfile of pipeline task [doris]
yiguolei merged PR #43350: URL: https://github.com/apache/doris/pull/43350 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [chore](type cast) Fix some implicit cast [doris]
zhiqiang- commented on PR #43050: URL: https://github.com/apache/doris/pull/43050#issuecomment-2466551390 run external -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Enhancement] (nereids)implement showProcCommand in nereids [doris]
Vallishp commented on PR #43146: URL: https://github.com/apache/doris/pull/43146#issuecomment-2466187002 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix] add force in the tosql for drop table and drop database [doris]
github-actions[bot] commented on PR #43227: URL: https://github.com/apache/doris/pull/43227#issuecomment-2466233343 PR approved by anyone and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-3.0: [chore](log) Use correct error type of uneven user behaviour [doris]
yiguolei merged PR #43493: URL: https://github.com/apache/doris/pull/43493 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-3.0 updated: branch-3.0: [chore](log) Use correct error type of uneven user behaviour (#43493)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 803cf4dd0bb branch-3.0: [chore](log) Use correct error type of uneven
user behaviour (#43493)
803cf4dd0bb is described below
commit 803cf4dd0bbb7bb111377ac460c16ca514bec99b
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sat Nov 9 22:08:16 2024 +0800
branch-3.0: [chore](log) Use correct error type of uneven user behaviour
(#43493)
Cherry-picked from #43334
Co-authored-by: zclllhhjj
---
be/src/vec/columns/column_array.cpp | 5 +++--
be/src/vec/columns/column_map.cpp | 2 +-
be/src/vec/functions/array/function_array_with_constant.cpp | 4 ++--
3 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/be/src/vec/columns/column_array.cpp
b/be/src/vec/columns/column_array.cpp
index 8b0c8b1e8c2..05cb2d2addc 100644
--- a/be/src/vec/columns/column_array.cpp
+++ b/be/src/vec/columns/column_array.cpp
@@ -25,6 +25,7 @@
#include
#include
+#include "common/status.h"
#include "vec/columns/column_const.h"
#include "vec/columns/column_nullable.h"
#include "vec/columns/column_string.h"
@@ -130,7 +131,7 @@ Field ColumnArray::operator[](size_t n) const {
if (size > max_array_size_as_field)
throw doris::Exception(
-ErrorCode::INTERNAL_ERROR,
+ErrorCode::INVALID_ARGUMENT,
"Array of size {}, is too large to be manipulated as single
field, maximum size {}",
size, max_array_size_as_field);
@@ -147,7 +148,7 @@ void ColumnArray::get(size_t n, Field& res) const {
if (size > max_array_size_as_field)
throw doris::Exception(
-ErrorCode::INTERNAL_ERROR,
+ErrorCode::INVALID_ARGUMENT,
"Array of size {}, is too large to be manipulated as single
field, maximum size {}",
size, max_array_size_as_field);
diff --git a/be/src/vec/columns/column_map.cpp
b/be/src/vec/columns/column_map.cpp
index a85737b9e04..c00d1faba41 100644
--- a/be/src/vec/columns/column_map.cpp
+++ b/be/src/vec/columns/column_map.cpp
@@ -105,7 +105,7 @@ Field ColumnMap::operator[](size_t n) const {
size_t element_size = size_at(n);
if (element_size > max_array_size_as_field) {
-throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
+throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
"element size {} is too large to be manipulated
as single map "
"field, maximum size {}",
element_size, max_array_size_as_field);
diff --git a/be/src/vec/functions/array/function_array_with_constant.cpp
b/be/src/vec/functions/array/function_array_with_constant.cpp
index 16e3947714c..0fd20a436ea 100644
--- a/be/src/vec/functions/array/function_array_with_constant.cpp
+++ b/be/src/vec/functions/array/function_array_with_constant.cpp
@@ -92,8 +92,8 @@ public:
for (size_t i = 0; i < input_rows_count; ++i) {
auto array_size = num->get_int(i);
if (UNLIKELY(array_size < 0) || UNLIKELY(array_size >
max_array_size_as_field)) {
-return Status::RuntimeError("Array size should in range(0, {})
in function: {}",
-max_array_size_as_field,
get_name());
+return Status::InvalidArgument("Array size should in range(0,
{}) in function: {}",
+ max_array_size_as_field,
get_name());
}
offset += array_size;
offsets.push_back(offset);
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-2.1 updated: [pick](branch-2.1) pick #38215 (#43386)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 9d7bc5b7652 [pick](branch-2.1) pick #38215 (#43386)
9d7bc5b7652 is described below
commit 9d7bc5b765259f101da836126ce48cb8f7724b22
Author: Xinyi Zou
AuthorDate: Sat Nov 9 22:13:05 2024 +0800
[pick](branch-2.1) pick #38215 (#43386)
pick #38215
-
Co-authored-by: Zou Xinyi
---
be/src/pipeline/exec/result_sink_operator.cpp | 5 ++-
be/src/util/arrow/row_batch.cpp| 37 --
be/src/util/arrow/row_batch.h | 11 ---
be/src/vec/runtime/vparquet_transformer.cpp| 3 +-
be/src/vec/sink/vmemory_scratch_sink.cpp | 2 +-
be/src/vec/sink/vresult_sink.cpp | 4 ++-
.../serde/data_type_serde_arrow_test.cpp | 4 +--
7 files changed, 40 insertions(+), 26 deletions(-)
diff --git a/be/src/pipeline/exec/result_sink_operator.cpp
b/be/src/pipeline/exec/result_sink_operator.cpp
index 1114bdd312f..52daf425f55 100644
--- a/be/src/pipeline/exec/result_sink_operator.cpp
+++ b/be/src/pipeline/exec/result_sink_operator.cpp
@@ -17,6 +17,8 @@
#include "result_sink_operator.h"
+#include
+
#include
#include "common/config.h"
@@ -96,7 +98,8 @@ Status ResultSinkLocalState::open(RuntimeState* state) {
}
case TResultSinkType::ARROW_FLIGHT_PROTOCAL: {
std::shared_ptr arrow_schema;
-RETURN_IF_ERROR(convert_expr_ctxs_arrow_schema(_output_vexpr_ctxs,
&arrow_schema));
+RETURN_IF_ERROR(convert_expr_ctxs_arrow_schema(_output_vexpr_ctxs,
&arrow_schema,
+ state->timezone()));
state->exec_env()->result_mgr()->register_arrow_schema(state->fragment_instance_id(),
arrow_schema);
_writer.reset(new (std::nothrow) vectorized::VArrowFlightResultWriter(
diff --git a/be/src/util/arrow/row_batch.cpp b/be/src/util/arrow/row_batch.cpp
index cb4ccf15ba2..ba6f4adf6c6 100644
--- a/be/src/util/arrow/row_batch.cpp
+++ b/be/src/util/arrow/row_batch.cpp
@@ -46,7 +46,8 @@ namespace doris {
using strings::Substitute;
-Status convert_to_arrow_type(const TypeDescriptor& type,
std::shared_ptr* result) {
+Status convert_to_arrow_type(const TypeDescriptor& type,
std::shared_ptr* result,
+ const std::string& timezone) {
switch (type.type) {
case TYPE_NULL:
*result = arrow::null();
@@ -96,11 +97,11 @@ Status convert_to_arrow_type(const TypeDescriptor& type,
std::shared_ptr 3) {
-*result =
std::make_shared(arrow::TimeUnit::MICRO);
+*result =
std::make_shared(arrow::TimeUnit::MICRO, timezone);
} else if (type.scale > 0) {
-*result =
std::make_shared(arrow::TimeUnit::MILLI);
+*result =
std::make_shared(arrow::TimeUnit::MILLI, timezone);
} else {
-*result =
std::make_shared(arrow::TimeUnit::SECOND);
+*result =
std::make_shared(arrow::TimeUnit::SECOND, timezone);
}
break;
case TYPE_DECIMALV2:
@@ -120,7 +121,7 @@ Status convert_to_arrow_type(const TypeDescriptor& type,
std::shared_ptr item_type;
-static_cast(convert_to_arrow_type(type.children[0], &item_type));
+static_cast(convert_to_arrow_type(type.children[0], &item_type,
timezone));
*result = std::make_shared(item_type);
break;
}
@@ -128,8 +129,8 @@ Status convert_to_arrow_type(const TypeDescriptor& type,
std::shared_ptr key_type;
std::shared_ptr val_type;
-static_cast(convert_to_arrow_type(type.children[0], &key_type));
-static_cast(convert_to_arrow_type(type.children[1], &val_type));
+static_cast(convert_to_arrow_type(type.children[0], &key_type,
timezone));
+static_cast(convert_to_arrow_type(type.children[1], &val_type,
timezone));
*result = std::make_shared(key_type, val_type);
break;
}
@@ -138,7 +139,7 @@ Status convert_to_arrow_type(const TypeDescriptor& type,
std::shared_ptr> fields;
for (size_t i = 0; i < type.children.size(); i++) {
std::shared_ptr field_type;
-static_cast(convert_to_arrow_type(type.children[i],
&field_type));
+static_cast(convert_to_arrow_type(type.children[i],
&field_type, timezone));
fields.push_back(std::make_shared(type.field_names[i], field_type,
type.contains_nulls[i]));
}
@@ -156,20 +157,22 @@ Status convert_to_arrow_type(const TypeDescriptor& type,
std::shared_ptr* field) {
+Status convert_to_arrow_field(SlotDescriptor* desc,
std::shared_ptr* field,
+ const std::st
Re: [PR] [fix] add force in the tosql for drop table and drop database [doris]
github-actions[bot] commented on PR #43227: URL: https://github.com/apache/doris/pull/43227#issuecomment-246621 PR approved by at least one committer and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [pick](branch-2.1) pick #38215 [doris]
yiguolei merged PR #43386: URL: https://github.com/apache/doris/pull/43386 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-2.1 updated: [fix](profile) update_rpc_time when enable_verbose_profile = false (#43096)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 5195d61b6f6 [fix](profile) update_rpc_time when enable_verbose_profile
= false (#43096)
5195d61b6f6 is described below
commit 5195d61b6f6840a8073076095e6df04c00cd97c5
Author: Mryange <59914473+mrya...@users.noreply.github.com>
AuthorDate: Sat Nov 9 22:17:29 2024 +0800
[fix](profile) update_rpc_time when enable_verbose_profile = false (#43096)
### What problem does this PR solve?
If enable_verbose_profile is false, update_rpc_time will not be called,
and RPC count, max, and min statistics will not be recorded.
```
- RpcCount: 0
- RpcMaxTime: 0ns
- RpcMinTime: 0ns
- RpcSumTime: 0ns
```
Co-authored-by: Mryange
---
be/src/pipeline/exec/exchange_sink_buffer.cpp | 20 ++--
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/be/src/pipeline/exec/exchange_sink_buffer.cpp
b/be/src/pipeline/exec/exchange_sink_buffer.cpp
index 63b69713f0d..af66eedc1e0 100644
--- a/be/src/pipeline/exec/exchange_sink_buffer.cpp
+++ b/be/src/pipeline/exec/exchange_sink_buffer.cpp
@@ -300,10 +300,10 @@ Status ExchangeSinkBuffer::_send_rpc(InstanceLoId
id) {
}
// attach task for memory tracker and query id when core
SCOPED_ATTACH_TASK(_state);
-if (_state->enable_verbose_profile()) {
-auto end_rpc_time = GetCurrentTimeNanos();
-update_rpc_time(id, start_rpc_time, end_rpc_time);
-}
+
+auto end_rpc_time = GetCurrentTimeNanos();
+update_rpc_time(id, start_rpc_time, end_rpc_time);
+
Status s(Status::create(result.status()));
if (s.is()) {
_set_receiver_eof(id);
@@ -381,10 +381,10 @@ Status ExchangeSinkBuffer::_send_rpc(InstanceLoId
id) {
}
// attach task for memory tracker and query id when core
SCOPED_ATTACH_TASK(_state);
-if (_state->enable_verbose_profile()) {
-auto end_rpc_time = GetCurrentTimeNanos();
-update_rpc_time(id, start_rpc_time, end_rpc_time);
-}
+
+auto end_rpc_time = GetCurrentTimeNanos();
+update_rpc_time(id, start_rpc_time, end_rpc_time);
+
Status s(Status::create(result.status()));
if (s.is()) {
_set_receiver_eof(id);
@@ -529,10 +529,10 @@ void
ExchangeSinkBuffer::update_rpc_time(InstanceLoId id, int64_t start_
template
void ExchangeSinkBuffer::update_profile(RuntimeProfile* profile) {
-auto* _max_rpc_timer = ADD_TIMER(profile, "RpcMaxTime");
+auto* _max_rpc_timer = ADD_TIMER_WITH_LEVEL(profile, "RpcMaxTime", 1);
auto* _min_rpc_timer = ADD_TIMER(profile, "RpcMinTime");
auto* _sum_rpc_timer = ADD_TIMER(profile, "RpcSumTime");
-auto* _count_rpc = ADD_COUNTER(profile, "RpcCount", TUnit::UNIT);
+auto* _count_rpc = ADD_COUNTER_WITH_LEVEL(profile, "RpcCount",
TUnit::UNIT, 1);
auto* _avg_rpc_timer = ADD_TIMER(profile, "RpcAvgTime");
int64_t max_rpc_time = 0, min_rpc_time = 0;
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](profile) update_rpc_time when enable_verbose_profile = false [doris]
yiguolei merged PR #43096: URL: https://github.com/apache/doris/pull/43096 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-2.1 updated: [branch-2.1](timezone) Preload time offset in datetime (#42395) (#42607)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new d9339564496 [branch-2.1](timezone) Preload time offset in datetime
(#42395) (#42607)
d9339564496 is described below
commit d9339564496e984cba11352ef6724f3f7b38bd9a
Author: zclllhhjj
AuthorDate: Sun Nov 10 00:30:28 2024 +0800
[branch-2.1](timezone) Preload time offset in datetime (#42395) (#42607)
pick https://github.com/apache/doris/pull/42395
---
be/src/util/timezone_utils.cpp | 50 +---
be/src/util/timezone_utils.h | 3 +
be/test/util/timezone_utils_test.cpp | 144 +++
3 files changed, 188 insertions(+), 9 deletions(-)
diff --git a/be/src/util/timezone_utils.cpp b/be/src/util/timezone_utils.cpp
index 5aef6f8702b..6bb71ac4647 100644
--- a/be/src/util/timezone_utils.cpp
+++ b/be/src/util/timezone_utils.cpp
@@ -30,6 +30,7 @@
#include
#include
+#include
#include
#include
#include
@@ -58,6 +59,9 @@ static const char* tzdir = "/usr/share/zoneinfo"; // default
value, may change b
void TimezoneUtils::clear_timezone_caches() {
lower_zone_cache_->clear();
}
+int TimezoneUtils::cache_size() {
+return lower_zone_cache_->size();
+}
static bool parse_save_name_tz(const std::string& tz_name) {
cctz::time_zone tz;
@@ -106,24 +110,54 @@ void TimezoneUtils::load_timezones_to_cache() {
}
lower_zone_cache_->erase("lmt"); // local mean time for every timezone
-LOG(INFO) << "Read " << lower_zone_cache_->size() << " timezones.";
+
+load_offsets_to_cache();
+LOG(INFO) << "Preloaded" << lower_zone_cache_->size() << " timezones.";
+}
+
+static std::string to_hour_string(int arg) {
+if (arg < 0 && arg > -10) { // -9 to -1
+return std::string {"-0"} + std::to_string(std::abs(arg));
+} else if (arg >= 0 && arg < 10) { //0 to 9
+return std::string {"0"} + std::to_string(arg);
+}
+return std::to_string(arg);
+}
+
+void TimezoneUtils::load_offsets_to_cache() {
+for (int hour = -12; hour <= +14; hour++) {
+for (int minute = 0; minute <= 30; minute += 30) {
+std::string offset_str = (hour >= 0 ? "+" : "") +
to_hour_string(hour) + ':' +
+ (minute == 0 ? "00" : "30");
+cctz::time_zone result;
+parse_tz_offset_string(offset_str, result);
+lower_zone_cache_->emplace(offset_str, result);
+}
+}
+// -00 for hour is also valid
+std::string offset_str = "-00:00";
+cctz::time_zone result;
+parse_tz_offset_string(offset_str, result);
+lower_zone_cache_->emplace(offset_str, result);
+offset_str = "-00:30";
+parse_tz_offset_string(offset_str, result);
+lower_zone_cache_->emplace(offset_str, result);
}
bool TimezoneUtils::find_cctz_time_zone(const std::string& timezone,
cctz::time_zone& ctz) {
-if (auto it = lower_zone_cache_->find(to_lower_copy(timezone));
-it != lower_zone_cache_->end()) {
+if (auto it = lower_zone_cache_->find(to_lower_copy(timezone)); it !=
lower_zone_cache_->end())
+[[likely]] {
ctz = it->second;
return true;
}
-// offset format or just illegal
-return parse_tz_offset_string(timezone, ctz);
+return false;
}
bool TimezoneUtils::parse_tz_offset_string(const std::string& timezone,
cctz::time_zone& ctz) {
// like +08:00, which not in timezone_names_map_
re2::StringPiece value;
-if (time_zone_offset_format_reg.Match(timezone, 0, timezone.size(),
RE2::UNANCHORED, &value,
- 1)) {
+if (time_zone_offset_format_reg.Match(timezone, 0, timezone.size(),
RE2::UNANCHORED, &value, 1))
+[[likely]] {
bool positive = value[0] != '-';
//Regular expression guarantees hour and minute must be int
@@ -139,8 +173,6 @@ bool TimezoneUtils::parse_tz_offset_string(const
std::string& timezone, cctz::ti
int offset = hour * 60 * 60 + minute * 60;
offset *= positive ? 1 : -1;
ctz = cctz::fixed_time_zone(cctz::seconds(offset));
-// try to push the result time offset of "+08:00" need lock. now it's
harmful for performance.
-// maybe we can use rcu of hazard-pointer to opt it.
return true;
}
return false;
diff --git a/be/src/util/timezone_utils.h b/be/src/util/timezone_utils.h
index c8bce44b5ab..3cdb17fc6fd 100644
--- a/be/src/util/timezone_utils.h
+++ b/be/src/util/timezone_utils.h
@@ -41,6 +41,9 @@ public:
private:
// for ut only
static void clear_timezone_caches();
+static int cache_size();
+
+static void load_offsets_to_cache();
static bool parse_tz_offset_string(const std::string& timezone,
cctz::time_zone& ctz);
};
diff --git a/be/test/util/timezone_
Re: [PR] branch-2.1: [fix](regression) fix flaky partial update cases [doris]
yiguolei closed pull request #43143: branch-2.1: [fix](regression) fix flaky partial update cases URL: https://github.com/apache/doris/pull/43143 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [branch-2.1](timezone) Preload time offset in datetime (#42395) [doris]
yiguolei merged PR #42607: URL: https://github.com/apache/doris/pull/42607 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[PR] branch-2.1: [fix](regression) fix flaky partial update cases [doris]
github-actions[bot] opened a new pull request, #43143: URL: https://github.com/apache/doris/pull/43143 Cherry-picked from #42908 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-2.1 updated: branch-2.1: [fix](regression) fix flaky partial update cases (#43143)
This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new e8d4c4cb7a8 branch-2.1: [fix](regression) fix flaky partial update cases (#43143) e8d4c4cb7a8 is described below commit e8d4c4cb7a8784ead3df4ef89c1199623e384717 Author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> AuthorDate: Sun Nov 10 00:37:33 2024 +0800 branch-2.1: [fix](regression) fix flaky partial update cases (#43143) Cherry-picked from #42908 Co-authored-by: zhannngchen <48427519+zhannngc...@users.noreply.github.com> - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-2.1: [fix](auth)Fix concurrency issue during role manager upgrade [doris]
yiguolei closed pull request #43194: branch-2.1: [fix](auth)Fix concurrency issue during role manager upgrade URL: https://github.com/apache/doris/pull/43194 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-2.1: [fix](regression) fix flaky partial update cases [doris]
yiguolei merged PR #43143: URL: https://github.com/apache/doris/pull/43143 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[PR] branch-2.1: [fix](auth)Fix concurrency issue during role manager upgrade [doris]
github-actions[bot] opened a new pull request, #43194: URL: https://github.com/apache/doris/pull/43194 Cherry-picked from #42419 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Opt](orc)Optimize the merge io when orc reader read multiple tiny stripes. (#42004) [doris]
doris-robot commented on PR #43467: URL: https://github.com/apache/doris/pull/43467#issuecomment-246621 TPC-DS: Total hot run time: 193440 ms ``` machine: 'aliyun_ecs.c7a.8xlarge_32C64G' scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools TPC-DS sf100 test result on commit 8689affd465f32332fea42b7a4ce8dc3bc732649, data reload: false query1 2596225822422242 query2 6218205719901990 query3 14451 9436244 244 query4 32903 23406 23387 23387 query5 3770447 441 441 query6 283 192 189 189 query7 3998330 312 312 query8 296 245 252 245 query9 9469269026922690 query10 482 272 270 270 query11 17703 15120 15156 15120 query12 161 101 100 100 query13 1544433 399 399 query14 8307752073107310 query15 227 186 187 186 query16 7440532 471 471 query17 1584603 586 586 query18 1608635 637 635 query19 232 188 195 188 query20 124 114 120 114 query21 212 108 106 106 query22 4956461843514351 query23 34563 33806 33786 33786 query24 12381 337933623362 query25 712 416 425 416 query26 1705185 190 185 query27 2514290 294 290 query28 7806249124752475 query29 1045443 450 443 query30 397 302 299 299 query31 1014792 830 792 query32 97 83 54 54 query33 762 289 267 267 query34 1008507 501 501 query35 903 733 724 724 query36 1106907 920 907 query37 257 75 72 72 query38 3927393338373837 query39 1480141314111411 query40 254 96 97 96 query41 53 48 51 48 query42 117 96 98 96 query43 526 492 478 478 query44 1240776 775 775 query45 178 165 162 162 query46 1124718 741 718 query47 1855179117651765 query48 451 368 362 362 query49 1057361 375 361 query50 806 398 419 398 query51 7342712870687068 query52 105 89 89 89 query53 265 182 181 181 query54 1162446 443 443 query55 77 76 75 75 query56 253 236 246 236 query57 1202108910951089 query58 226 200 203 200 query59 3114285227632763 query60 287 247 265 247 query61 109 133 139 133 query62 859 668 654 654 query63 220 184 185 184 query64 5104652 619 619 query65 3263320832043204 query66 1253299 304 299 query67 15723 15455 15307 15307 query68 4651565 569 565 query69 432 252 255 252 query70 1167108611241086 query71 334 269 260 260 query72 6157396140303961 query73 749 346 346 346 query74 10346 896789068906 query75 263126262626 query76 2740928 1043928 query77 363 269 278 269 query78 10600 977094099409 query79 8673579 594 579 query80 2586435 422 422 query81 583 245 255 245 query82 1394111 114 111 query83 274 156 154 154 query84 286 80 77 77 query85 2178297 285 285 query86 495 266 289 266 query87 4382421742384217 query88 5619237723802377 query89 577 292 287 287 query90 2184186 187 186 query91 180 144 144 144 query92 63 50 49 49 query93 7366545 542 542 query94 929 289 305 289 query95 354 254 252 252 query96 633 276 298 276 query97 318231433143 query98 213 204 199 199 query99 1628132812851285 Total cold run time: 323268 ms Total hot run time: 193440 ms ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to
[PR] [opt](log) change lzo decompress log to debug level [doris]
morningman opened a new pull request, #43540: URL: https://github.com/apache/doris/pull/43540 ### What problem does this PR solve? Problem Summary: the log level in lzo decompressor is INFO, which causing too much log and impact the performance. Change the log level to VLOG. ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](log) change lzo decompress log to debug level [doris]
doris-robot commented on PR #43540: URL: https://github.com/apache/doris/pull/43540#issuecomment-2466244724 Thank you for your contribution to Apache Doris. Don't know what should be done next? See [How to process your PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR). Please clearly describe your PR: 1. What problem was fixed (it's best to include specific error reporting information). How it was fixed. 2. Which behaviors were modified. What was the previous behavior, what is it now, why was it modified, and what possible impacts might there be. 3. What features were added. Why was this function added? 4. Which code was refactored and why was this part of the code refactored? 5. Which functions were optimized and what is the difference before and after the optimization? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Enhancement] replace table options to keep original dropped table in recycle bin [doris]
dataroaring commented on PR #43538: URL: https://github.com/apache/doris/pull/43538#issuecomment-2466242505 Please add document in doris-website. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](log) change lzo decompress log to debug level [doris]
github-actions[bot] commented on PR #43540: URL: https://github.com/apache/doris/pull/43540#issuecomment-2466246385 PR approved by anyone and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](log) change lzo decompress log to debug level [doris]
github-actions[bot] commented on PR #43540: URL: https://github.com/apache/doris/pull/43540#issuecomment-2466246051 clang-tidy review says "All clean, LGTM! :+1:" -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](log) change lzo decompress log to debug level [doris]
morningman commented on PR #43540: URL: https://github.com/apache/doris/pull/43540#issuecomment-2466245601 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-2.1 updated: branch-2.1: [fix](mtmv)Fix the problem where the job does not exist, which prevents the deletion of MTMV (#43325)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new fe1b8d44fdc branch-2.1: [fix](mtmv)Fix the problem where the job does
not exist, which prevents the deletion of MTMV (#43325)
fe1b8d44fdc is described below
commit fe1b8d44fdc58bbfcd40647202f47b3e90568c03
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Nov 10 01:11:26 2024 +0800
branch-2.1: [fix](mtmv)Fix the problem where the job does not exist, which
prevents the deletion of MTMV (#43325)
Cherry-picked from #43080
Co-authored-by: zhangdong <493738...@qq.com>
---
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java
index 8ffcea423d7..2c03ad16176 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java
@@ -130,7 +130,7 @@ public class MTMVJobManager implements MTMVHookService {
public void dropMTMV(MTMV mtmv) throws DdlException {
try {
Env.getCurrentEnv().getJobManager()
-.unregisterJob(mtmv.getJobInfo().getJobName(), false);
+.unregisterJob(mtmv.getJobInfo().getJobName(), true);
} catch (JobException e) {
LOG.warn("drop mtmv job failed, mtmvName: {}", mtmv.getName(), e);
throw new DdlException(e.getMessage());
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-2.1: [enhance](mtmv)MTMV interface optimization [doris]
yiguolei closed pull request #43329: branch-2.1: [enhance](mtmv)MTMV interface optimization URL: https://github.com/apache/doris/pull/43329 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-2.1: [fix](mtmv)Fix the problem where the job does not exist, which prevents the deletion of MTMV [doris]
yiguolei merged PR #43325: URL: https://github.com/apache/doris/pull/43325 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [Opt](orc)Optimize the merge io when orc reader read multiple tiny stripes. (#42004) [doris]
github-actions[bot] commented on PR #43467: URL: https://github.com/apache/doris/pull/43467#issuecomment-2466174964 clang-tidy review says "All clean, LGTM! :+1:" -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris-website) branch asf-site updated (eadab973d0b -> 084a4a2ea31)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/doris-website.git discard eadab973d0b Automated deployment with doris branch @ 8392376723a884f6649d19ad0749b3191a66a71c new 084a4a2ea31 Automated deployment with doris branch @ 8392376723a884f6649d19ad0749b3191a66a71c This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (eadab973d0b) \ N -- N -- N refs/heads/asf-site (084a4a2ea31) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: docs/1.2/search-index.json | 2 +- docs/2.0/search-index.json | 2 +- docs/3.0/search-index.json | 2 +- docs/dev/search-index.json | 2 +- search-index.json| 2 +- zh-CN/docs/1.2/search-index.json | 2 +- zh-CN/docs/2.0/search-index.json | 2 +- zh-CN/docs/3.0/search-index.json | 2 +- zh-CN/docs/dev/search-index.json | 2 +- zh-CN/search-index.json | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[PR] [fix](mtmv) Fix generate hyper graph wrongly when has filter which can not push down [doris]
seawinde opened a new pull request, #43539: URL: https://github.com/apache/doris/pull/43539 ### What problem does this PR solve? When query is as following: ```sql select o_orderdate, o_shippriority, o_comment, l_orderkey, l_partkey from orders left join lineitem on l_orderkey = o_orderkey left join partsupp on ps_partkey = l_partkey and l_suppkey = ps_suppkey; ``` hyper grapch is as following: this` leftExtendedNodes` Shoule be contained both orders and lineitem, which should be 3, but now is 2 https://github.com/user-attachments/assets/9f70aa15-895e-4ef2-9d49-a4b8a5cbb01e";> Issue Number: close #xxx Related PR: https://github.com/apache/doris/pull/28006 Problem Summary: This may cause associate wrongly when use hyper graph and materialized view which depends on it rewrite wrongly ### Release note Fix generate hyper graph wrongly when has filter which can not push down ### Check List (For Author) - Test - [x] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](mtmv) Fix generate hyper graph wrongly when has filter which can not push down [doris]
doris-robot commented on PR #43539: URL: https://github.com/apache/doris/pull/43539#issuecomment-2466211183 Thank you for your contribution to Apache Doris. Don't know what should be done next? See [How to process your PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR). Please clearly describe your PR: 1. What problem was fixed (it's best to include specific error reporting information). How it was fixed. 2. Which behaviors were modified. What was the previous behavior, what is it now, why was it modified, and what possible impacts might there be. 3. What features were added. Why was this function added? 4. Which code was refactored and why was this part of the code refactored? 5. Which functions were optimized and what is the difference before and after the optimization? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](mtmv) Fix generate hyper graph wrongly when has filter which can not push down [doris]
seawinde commented on PR #43539: URL: https://github.com/apache/doris/pull/43539#issuecomment-2466213470 run buildall -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-2.1: [Optimize](Expr) Opt getting value of VLitreal [doris]
yiguolei closed pull request #43249: branch-2.1: [Optimize](Expr) Opt getting value of VLitreal URL: https://github.com/apache/doris/pull/43249 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[PR] branch-2.1: [Optimize](Expr) Opt getting value of VLitreal [doris]
github-actions[bot] opened a new pull request, #43249: URL: https://github.com/apache/doris/pull/43249 Cherry-picked from #43204 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-2.1 updated: [cherry-pick](branch-2.1)add SessionVariable for enableCooldownReplicaAffinity (#42675)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new fba06b33b9f [cherry-pick](branch-2.1)add SessionVariable for
enableCooldownReplicaAffinity (#42675)
fba06b33b9f is described below
commit fba06b33b9fc8b9766bec1afcee287cd8fb5198c
Author: kkop <2449402...@qq.com>
AuthorDate: Sun Nov 10 00:46:26 2024 +0800
[cherry-pick](branch-2.1)add SessionVariable for
enableCooldownReplicaAffinity (#42675)
pick from master：https://github.com/apache/doris/pull/41741
---
.../main/java/org/apache/doris/common/Config.java | 3 --
.../org/apache/doris/planner/OlapScanNode.java | 10 +-
.../java/org/apache/doris/qe/SessionVariable.java | 9 +
.../suites/show_p0/test_show_variables.groovy | 38 ++
4 files changed, 56 insertions(+), 4 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 72ce2871038..0fe3d0cbf14 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2844,9 +2844,6 @@ public class Config extends ConfigBase {
"Stream_Load When importing, the maximum length of label is
limited"})
public static int label_regex_length = 128;
-@ConfField(mutable = true)
-public static boolean enable_cooldown_replica_affinity = true;
-
//==
// end of cloud config
//==
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index 4ffa12e8f05..5c6b47fec71 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -830,7 +830,7 @@ public class OlapScanNode extends ScanNode {
}
}
-if (Config.enable_cooldown_replica_affinity) {
+if (isEnableCooldownReplicaAffinity()) {
final long coolDownReplicaId = tablet.getCooldownReplicaId();
// we prefer to query using cooldown replica to make sure the
cache is fully utilized
// for example: consider there are 3BEs(A,B,C) and each has
one replica for tablet X. and X
@@ -932,6 +932,14 @@ public class OlapScanNode extends ScanNode {
}
}
+private boolean isEnableCooldownReplicaAffinity() {
+ConnectContext connectContext = ConnectContext.get();
+if (connectContext != null) {
+return
connectContext.getSessionVariable().isEnableCooldownReplicaAffinity();
+}
+return true;
+}
+
private void computePartitionInfo() throws AnalysisException {
long start = System.currentTimeMillis();
// Step1: compute partition ids
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 6b0c0791683..aa4d06c93dc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -644,6 +644,8 @@ public class SessionVariable implements Serializable,
Writable {
public static final String SKIP_CHECKING_ACID_VERSION_FILE =
"skip_checking_acid_version_file";
+public static final String ENABLE_COOLDOWN_REPLICA_AFFINITY =
+"enable_cooldown_replica_affinity";
/**
* If set false, user couldn't submit analyze SQL and FE won't allocate
any related resources.
*/
@@ -2139,6 +2141,9 @@ public class SessionVariable implements Serializable,
Writable {
})
public boolean skipCheckingAcidVersionFile = false;
+@VariableMgr.VarAttr(name = ENABLE_COOLDOWN_REPLICA_AFFINITY, needForward
= true)
+public boolean enableCooldownReplicaAffinity = true;
+
public void setEnableEsParallelScroll(boolean enableESParallelScroll) {
this.enableESParallelScroll = enableESParallelScroll;
}
@@ -4297,4 +4302,8 @@ public class SessionVariable implements Serializable,
Writable {
throw new IllegalArgumentException("Unknown serde dialect: " +
serdeDialect);
}
}
+
+public boolean isEnableCooldownReplicaAffinity() {
+return enableCooldownReplicaAffinity;
+}
}
diff --git a/regression-test/suites/show_p0/test_show_variables.groovy
b/regression-test/suites/show_p0/test_show_variables.groovy
new file mode 100644
index 000..6de8075b630
--- /dev/null
+++ b/regression-test/suites/show_p0/test_show_variable
Re: [PR] branch-2.1: [fix](auth)Fix concurrency issue during role manager upgrade [doris]
yiguolei merged PR #43194: URL: https://github.com/apache/doris/pull/43194 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-2.1 updated: branch-2.1: [fix](auth)Fix concurrency issue during role manager upgrade (#43194)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 486dfe9f420 branch-2.1: [fix](auth)Fix concurrency issue during role
manager upgrade (#43194)
486dfe9f420 is described below
commit 486dfe9f420a0cc60b88313cdf67833651accb30
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Nov 10 00:45:07 2024 +0800
branch-2.1: [fix](auth)Fix concurrency issue during role manager upgrade
(#43194)
Cherry-picked from #42419
Co-authored-by: zhangdong <493738...@qq.com>
---
.../src/main/java/org/apache/doris/mysql/privilege/RoleManager.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RoleManager.java
b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RoleManager.java
index 654db93f21d..5ce213e2957 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RoleManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RoleManager.java
@@ -53,6 +53,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -63,7 +64,7 @@ public class RoleManager implements Writable,
GsonPostProcessable {
// Concurrency control is delegated by Auth, so not concurrentMap
@SerializedName(value = "roles")
-private Map roles = Maps.newHashMap();
+private ConcurrentMap roles = Maps.newConcurrentMap();
public RoleManager() {
roles.put(Role.OPERATOR.getRoleName(), Role.OPERATOR);
@@ -295,7 +296,7 @@ public class RoleManager implements Writable,
GsonPostProcessable {
// should be removed after version 3.0
private void removeClusterPrefix() {
-Map newRoles = Maps.newHashMap();
+ConcurrentMap newRoles = Maps.newConcurrentMap();
for (Map.Entry entry : roles.entrySet()) {
String roleName =
ClusterNamespace.getNameFromFullName(entry.getKey());
newRoles.put(roleName, entry.getValue());
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [cherry-pick](branch-2.1)add SessionVariable for enableCooldownReplicaAffinity [doris]
yiguolei merged PR #42675: URL: https://github.com/apache/doris/pull/42675 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-2.1: [Optimize](Expr) Opt getting value of VLitreal [doris]
github-actions[bot] commented on PR #43249: URL: https://github.com/apache/doris/pull/43249#issuecomment-2466287958 clang-tidy review says "All clean, LGTM! :+1:" -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
svn commit: r72925 - /dev/doris/2.1.7-rc03/ /release/doris/2.1/2.1.7/
Author: yiguolei Date: Sat Nov 9 16:56:43 2024 New Revision: 72925 Log: move doris 2.1.7-rc03 to release Added: release/doris/2.1/2.1.7/ - copied from r72924, dev/doris/2.1.7-rc03/ Removed: dev/doris/2.1.7-rc03/ - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-2.1: [Optimize](Expr) Opt getting value of VLitreal [doris]
yiguolei merged PR #43249: URL: https://github.com/apache/doris/pull/43249 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-3.0 updated: branch-3.0: [Optimize](Expr) Opt getting value of VLitreal (#43248)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 282d31a0446 branch-3.0: [Optimize](Expr) Opt getting value of VLitreal
(#43248)
282d31a0446 is described below
commit 282d31a044661cdcdcde4238654cf4e184f6fe3b
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Nov 10 01:04:24 2024 +0800
branch-3.0: [Optimize](Expr) Opt getting value of VLitreal (#43248)
Cherry-picked from #43204
Co-authored-by: zclllhhjj
---
be/src/vec/exprs/vliteral.cpp | 11 ++-
1 file changed, 2 insertions(+), 9 deletions(-)
diff --git a/be/src/vec/exprs/vliteral.cpp b/be/src/vec/exprs/vliteral.cpp
index d4e8c883106..bb95788bb33 100644
--- a/be/src/vec/exprs/vliteral.cpp
+++ b/be/src/vec/exprs/vliteral.cpp
@@ -70,15 +70,8 @@ Status VLiteral::execute(VExprContext* context,
vectorized::Block* block, int* r
}
std::string VLiteral::value() const {
-//TODO: dcheck the equality of size with 1. then use string with size to
replace the ss.
-std::stringstream out;
-for (size_t i = 0; i < _column_ptr->size(); i++) {
-if (i != 0) {
-out << ", ";
-}
-out << _data_type->to_string(*_column_ptr, i);
-}
-return out.str();
+DCHECK(_column_ptr->size() == 1);
+return _data_type->to_string(*_column_ptr, 0);
}
std::string VLiteral::debug_string() const {
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-3.0: [Optimize](Expr) Opt getting value of VLitreal [doris]
yiguolei merged PR #43248: URL: https://github.com/apache/doris/pull/43248 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-2.1 updated: branch-2.1: [Optimize](Expr) Opt getting value of VLitreal (#43249)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 625a1ea6ad0 branch-2.1: [Optimize](Expr) Opt getting value of VLitreal
(#43249)
625a1ea6ad0 is described below
commit 625a1ea6ad08ba02dc812fcb0b38ebafac0121af
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Nov 10 01:03:56 2024 +0800
branch-2.1: [Optimize](Expr) Opt getting value of VLitreal (#43249)
Cherry-picked from #43204
Co-authored-by: zclllhhjj
---
be/src/vec/exprs/vliteral.cpp | 11 ++-
1 file changed, 2 insertions(+), 9 deletions(-)
diff --git a/be/src/vec/exprs/vliteral.cpp b/be/src/vec/exprs/vliteral.cpp
index 1d02a63321b..8dcc5e3cb5a 100644
--- a/be/src/vec/exprs/vliteral.cpp
+++ b/be/src/vec/exprs/vliteral.cpp
@@ -76,15 +76,8 @@ Status VLiteral::execute(VExprContext* context,
vectorized::Block* block, int* r
}
std::string VLiteral::value() const {
-//TODO: dcheck the equality of size with 1. then use string with size to
replace the ss.
-std::stringstream out;
-for (size_t i = 0; i < _column_ptr->size(); i++) {
-if (i != 0) {
-out << ", ";
-}
-out << _data_type->to_string(*_column_ptr, i);
-}
-return out.str();
+DCHECK(_column_ptr->size() == 1);
+return _data_type->to_string(*_column_ptr, 0);
}
std::string VLiteral::debug_string() const {
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] branch-2.1: [fix](mtmv)Fix the problem where the job does not exist, which prevents the deletion of MTMV [doris]
yiguolei closed pull request #43325: branch-2.1: [fix](mtmv)Fix the problem where the job does not exist, which prevents the deletion of MTMV URL: https://github.com/apache/doris/pull/43325 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [opt](log) change lzo decompress log to debug level [doris]
doris-robot commented on PR #43540: URL: https://github.com/apache/doris/pull/43540#issuecomment-2466309820 TeamCity be ut coverage result: Function Coverage: 37.91% (9868/26027) Line Coverage: 29.09% (82263/282773) Region Coverage: 28.25% (42362/149949) Branch Coverage: 24.80% (21445/86488) Coverage Report: http://coverage.selectdb-in.cc/coverage/d7e83f69eed49ea2a35581ee73b8da4cef5cd93a_d7e83f69eed49ea2a35581ee73b8da4cef5cd93a/report/index.html -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris-website) branch asf-site updated (084a4a2ea31 -> 410988085bc)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/doris-website.git discard 084a4a2ea31 Automated deployment with doris branch @ 8392376723a884f6649d19ad0749b3191a66a71c new 410988085bc Automated deployment with doris branch @ 8392376723a884f6649d19ad0749b3191a66a71c This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (084a4a2ea31) \ N -- N -- N refs/heads/asf-site (410988085bc) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: docs/1.2/search-index.json | 2 +- docs/2.0/search-index.json | 2 +- docs/3.0/search-index.json | 2 +- docs/dev/search-index.json | 2 +- search-index.json| 2 +- zh-CN/docs/1.2/search-index.json | 2 +- zh-CN/docs/2.0/search-index.json | 2 +- zh-CN/docs/3.0/search-index.json | 2 +- zh-CN/docs/dev/search-index.json | 2 +- zh-CN/search-index.json | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [fix](column_complex) wrong type of Field returned by ColumnComplex [doris]
doris-robot commented on PR #43515: URL: https://github.com/apache/doris/pull/43515#issuecomment-2466182461 TeamCity be ut coverage result: Function Coverage: 37.92% (9869/26027) Line Coverage: 29.09% (82271/282774) Region Coverage: 28.25% (42354/149928) Branch Coverage: 24.79% (21441/86488) Coverage Report: http://coverage.selectdb-in.cc/coverage/4c8e095ddaa8c9e88010235d1f32c23290914c74_4c8e095ddaa8c9e88010235d1f32c23290914c74/report/index.html -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [feature](array-func)support array_match_all/any (#40605) [doris]
github-actions[bot] commented on PR #43514: URL: https://github.com/apache/doris/pull/43514#issuecomment-2466193830 PR approved by at least one committer and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
Re: [PR] [feature](array-func)support array_match_all/any (#40605) [doris]
github-actions[bot] commented on PR #43514: URL: https://github.com/apache/doris/pull/43514#issuecomment-2466193833 PR approved by anyone and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-3.0 updated: [feature](array-func)support array_match_all/any (#40605) (#43514)
This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new c8b2a2ecf3f [feature](array-func)support array_match_all/any (#40605)
(#43514)
c8b2a2ecf3f is described below
commit c8b2a2ecf3f61b3f4b9cdf60943b29ba96f9
Author: amory
AuthorDate: Sat Nov 9 20:20:33 2024 +0800
[feature](array-func)support array_match_all/any (#40605) (#43514)
array_match_all
means that every element in array column should all match filter
according to lambda expr
array_match_any
means that any element in array column match filter according to lambda
expr
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test
- [x] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason
- Behavior changed:
- [x] No.
- [ ] Yes.
- Does this need documentation?
- [ ] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [x] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../functions/array/function_array_register.cpp|2 +
.../vec/functions/array/varray_match_function.cpp | 145 +++
.../doris/analysis/LambdaFunctionCallExpr.java |4 +-
.../doris/catalog/BuiltinScalarFunctions.java |4 +
.../glue/translator/ExpressionTranslator.java |7 +-
.../functions/scalar/ArrayMatchAll.java| 68 +
.../functions/scalar/ArrayMatchAny.java| 68 +
.../expressions/visitor/ScalarFunctionVisitor.java | 10 +
.../data/nereids_function_p0/fn_test_am.csv| 102 ++
.../nereids_function_p0/scalar_function/Array.out | 1320
.../suites/nereids_function_p0/load.groovy | 25 +
.../scalar_function/Array.groovy | 42 +
12 files changed, 1789 insertions(+), 8 deletions(-)
diff --git a/be/src/vec/functions/array/function_array_register.cpp
b/be/src/vec/functions/array/function_array_register.cpp
index 6ddf6b6a5e2..071c3198189 100644
--- a/be/src/vec/functions/array/function_array_register.cpp
+++ b/be/src/vec/functions/array/function_array_register.cpp
@@ -56,6 +56,7 @@ void register_function_array_cum_sum(SimpleFunctionFactory&
factory);
void register_function_array_count(SimpleFunctionFactory&);
void register_function_array_filter_function(SimpleFunctionFactory&);
void register_function_array_splits(SimpleFunctionFactory&);
+void register_function_array_match(SimpleFunctionFactory&);
void register_function_array(SimpleFunctionFactory& factory) {
register_function_array_shuffle(factory);
@@ -92,6 +93,7 @@ void register_function_array(SimpleFunctionFactory& factory) {
register_function_array_count(factory);
register_function_array_filter_function(factory);
register_function_array_splits(factory);
+register_function_array_match(factory);
}
} // namespace doris::vectorized
diff --git a/be/src/vec/functions/array/varray_match_function.cpp
b/be/src/vec/functions/array/varray_match_function.cpp
new file mode 100644
index 000..c322fea7566
--- /dev/null
+++ b/be/src/vec/functions/array/varray_match_function.cpp
@@ -0,0 +1,145 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include
+
+#include
+#include
+#include
+#include
+
+#include "common/status.h"
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_vector.h"
+#include "vec/columns/columns
