[doris] branch master updated: [Log](query) Add log of fragment instance num for query (#20597)
This is an automated email from the ASF dual-hosted git repository.
weizuo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 71e8cb061c [Log](query) Add log of fragment instance num for query
(#20597)
71e8cb061c is described below
commit 71e8cb061cadbe7cfbcb653d6def72695769f3d5
Author: weizuo93
AuthorDate: Thu Jun 15 09:52:13 2023 +0800
[Log](query) Add log of fragment instance num for query (#20597)
Co-authored-by: weizuo
---
.../main/java/org/apache/doris/qe/Coordinator.java | 6 ++
.../java/org/apache/doris/qe/StmtExecutor.java | 22 ++
2 files changed, 28 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 1affdee118..75489d8246 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -187,6 +187,7 @@ public class Coordinator {
private final Map
fragmentExecParamsMap = Maps.newHashMap();
private final List fragments;
+private int instanceTotalNum;
private Map beToExecStates = Maps.newHashMap();
private Map beToPipelineExecCtxs =
Maps.newHashMap();
@@ -482,6 +483,10 @@ public class Coordinator {
return result;
}
+public int getInstanceTotalNum() {
+return instanceTotalNum;
+}
+
// Initialize
private void prepare() {
for (PlanFragment fragment : fragments) {
@@ -664,6 +669,7 @@ public class Coordinator {
// 1. set up exec states
int instanceNum = params.instanceExecParams.size();
Preconditions.checkState(instanceNum > 0);
+instanceTotalNum += instanceNum;
List tParams =
params.toThrift(backendIdx);
// 2. update memory limit for colocate join
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 9b0b8708aa..729e4fdbee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -1371,6 +1371,18 @@ public class StmtExecutor {
}
profile.getSummaryProfile().setQueryScheduleFinishTime();
updateProfile(false);
+if (coord.getInstanceTotalNum() > 1 && LOG.isDebugEnabled()) {
+try {
+LOG.debug("Start to execute fragment. user: {}, db: {}, sql:
{}, fragment instance num: {}",
+context.getQualifiedUser(), context.getDatabase(),
+parsedStmt.getOrigStmt().originStmt.replace("\n", " "),
+coord.getInstanceTotalNum());
+} catch (Exception e) {
+LOG.warn("Fail to print fragment concurrency for Query.", e);
+}
+}
+
+
Span fetchResultSpan = context.getTracer().spanBuilder("fetch
result").setParent(Context.current()).startSpan();
try (Scope scope = fetchResultSpan.makeCurrent()) {
while (true) {
@@ -1449,6 +1461,16 @@ public class StmtExecutor {
throw e;
} finally {
fetchResultSpan.end();
+if (coord.getInstanceTotalNum() > 1 && LOG.isDebugEnabled()) {
+try {
+LOG.debug("Finish to execute fragment. user: {}, db: {},
sql: {}, fragment instance num: {}",
+context.getQualifiedUser(), context.getDatabase(),
+parsedStmt.getOrigStmt().originStmt.replace("\n",
" "),
+coord.getInstanceTotalNum());
+} catch (Exception e) {
+LOG.warn("Fail to print fragment concurrency for Query.",
e);
+}
+}
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[doris] branch master updated (851886cc18 -> 303bee6fa3)
This is an automated email from the ASF dual-hosted git repository. weizuo pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/doris.git from 851886cc18 [minor](datev2) remove datev2 because datev2 is used by default (#19777) add 303bee6fa3 [Fix](single replica load) add inverted index copy for single replica load (#19663) No new revisions were added by this update. Summary of changes: be/src/olap/delta_writer.cpp| 25 be/src/service/internal_service.cpp | 115 ++-- gensrc/proto/internal_service.proto | 10 3 files changed, 119 insertions(+), 31 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[doris] branch master updated: [Bug](tablet) Fix bug that segment files are removed as trash but tablet meta is normal (#10721)
This is an automated email from the ASF dual-hosted git repository.
weizuo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 39325bd666 [Bug](tablet) Fix bug that segment files are removed as
trash but tablet meta is normal (#10721)
39325bd666 is described below
commit 39325bd66618adcfb826707d3d82aec4926cb330
Author: weizuo93
AuthorDate: Fri Jul 15 11:07:16 2022 +0800
[Bug](tablet) Fix bug that segment files are removed as trash but tablet
meta is normal (#10721)
---
be/src/olap/tablet_manager.cpp | 10 ++
1 file changed, 10 insertions(+)
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 5c4cd04a17..f61b0cc06a 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -161,6 +161,16 @@ Status TabletManager::_add_tablet_unlocked(TTabletId
tablet_id, const TabletShar
res = _add_tablet_to_map_unlocked(tablet_id, tablet, update_meta,
keep_files,
true /*drop_old*/);
} else {
+tablet->set_tablet_state(TABLET_SHUTDOWN);
+tablet->save_meta();
+{
+std::lock_guard
shutdown_tablets_wrlock(_shutdown_tablets_lock);
+_shutdown_tablets.push_back(tablet);
+}
+LOG(INFO) << "set tablet to shutdown state."
+ << "tablet_id=" << tablet->tablet_id()
+ << ", tablet_path=" << tablet->tablet_path();
+
res = Status::OLAPInternalError(OLAP_ERR_ENGINE_INSERT_OLD_TABLET);
}
LOG(WARNING) << "add duplicated tablet. force=" << force << ", res=" << res
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[doris] branch master updated: [Feature] Add interface to check tablet segment lost (#10711)
This is an automated email from the ASF dual-hosted git repository.
weizuo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5c1cd058f2 [Feature] Add interface to check tablet segment lost
(#10711)
5c1cd058f2 is described below
commit 5c1cd058f29e0840a8cbea3adbf8319c520f98f8
Author: weizuo93
AuthorDate: Tue Aug 2 09:40:04 2022 +0800
[Feature] Add interface to check tablet segment lost (#10711)
Co-authored-by: weizuo
---
be/src/http/CMakeLists.txt | 1 +
be/src/http/action/check_tablet_segment_action.cpp | 70 ++
be/src/http/action/check_tablet_segment_action.h | 36 +++
be/src/olap/rowset/beta_rowset.cpp | 17 ++
be/src/olap/rowset/beta_rowset.h | 2 +
be/src/olap/rowset/rowset.cpp | 5 ++
be/src/olap/rowset/rowset.h| 5 ++
be/src/olap/tablet.cpp | 15 +
be/src/olap/tablet.h | 2 +
be/src/olap/tablet_manager.cpp | 26
be/src/olap/tablet_manager.h | 2 +
be/src/service/http_service.cpp| 7 +++
be/test/testutil/mock_rowset.h | 2 +
.../http-actions/check-tablet-segment-action.md| 53
.../http-actions/check-tablet-segment-action.md| 53
15 files changed, 296 insertions(+)
diff --git a/be/src/http/CMakeLists.txt b/be/src/http/CMakeLists.txt
index b956d0982b..22fe2665b2 100644
--- a/be/src/http/CMakeLists.txt
+++ b/be/src/http/CMakeLists.txt
@@ -52,4 +52,5 @@ add_library(Webserver STATIC
action/config_action.cpp
action/check_rpc_channel_action.cpp
action/reset_rpc_channel_action.cpp
+ action/check_tablet_segment_action.cpp
)
diff --git a/be/src/http/action/check_tablet_segment_action.cpp
b/be/src/http/action/check_tablet_segment_action.cpp
new file mode 100644
index 00..6e822ace7d
--- /dev/null
+++ b/be/src/http/action/check_tablet_segment_action.cpp
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "http/action/check_tablet_segment_action.h"
+
+#include
+
+#include "http/http_channel.h"
+#include "http/http_headers.h"
+#include "http/http_request.h"
+#include "http/http_status.h"
+#include "olap/storage_engine.h"
+#include "service/backend_options.h"
+
+namespace doris {
+
+const static std::string HEADER_JSON = "application/json";
+
+CheckTabletSegmentAction::CheckTabletSegmentAction() {
+_host = BackendOptions::get_localhost();
+}
+
+void CheckTabletSegmentAction::handle(HttpRequest* req) {
+bool repair = false;
+std::string is_repair = req->param("repair");
+if (is_repair == "true") {
+repair = true;
+} else if (is_repair != "" && is_repair != "false") {
+EasyJson result_ej;
+result_ej["status"] = "Fail";
+result_ej["msg"] = "Parameter 'repair' must be set to 'true' or
'false'";
+req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.c_str());
+HttpChannel::send_reply(req, HttpStatus::OK, result_ej.ToString());
+return;
+}
+
+LOG(INFO) << "start to check tablet segment.";
+std::set bad_tablets =
+
StorageEngine::instance()->tablet_manager()->check_all_tablet_segment(repair);
+LOG(INFO) << "finish to check tablet segment.";
+
+EasyJson result_ej;
+result_ej["status"] = "Success";
+result_ej["msg"] = "Succeed to check all tablet segment";
+result_ej["num"] = bad_tablets.size();
+EasyJson tablets = result_ej.Set("bad_tablets", EasyJson::kArray);
+for (int64_t tablet_id : bad_tablets) {
+tablets.PushBack(tablet_id);
+}
+result_ej["set_
[doris] branch branch-1.1-lts updated: Revert "[Performance] Optimize performance of like expr (#15168)" (#16619)
This is an automated email from the ASF dual-hosted git repository.
weizuo pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
new ea0700e454 Revert "[Performance] Optimize performance of like expr
(#15168)" (#16619)
ea0700e454 is described below
commit ea0700e45498a3fe12df92a71697e60fd8a6b441
Author: Henry2SS <45096548+henry...@users.noreply.github.com>
AuthorDate: Thu Feb 16 11:39:47 2023 +0800
Revert "[Performance] Optimize performance of like expr (#15168)" (#16619)
This reverts commit 8f6c8ec5cd5d2452ce3d1dd7a678028e9bc91c40.
---
be/src/runtime/string_search.hpp | 8 +---
be/src/vec/functions/like.cpp| 2 +-
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/be/src/runtime/string_search.hpp b/be/src/runtime/string_search.hpp
index 14800f2903..3f657b6fa4 100644
--- a/be/src/runtime/string_search.hpp
+++ b/be/src/runtime/string_search.hpp
@@ -41,11 +41,13 @@ public:
return -1;
}
-char* occurence = std::strstr(str->ptr, _pattern->ptr);
-if (occurence == nullptr) {
+auto it = std::search(str->ptr, str->ptr + str->len,
+ std::default_searcher(_pattern->ptr,
_pattern->ptr + _pattern->len));
+if (it == str->ptr + str->len) {
return -1;
+} else {
+return it - str->ptr;
}
-return occurence - str->ptr;
}
private:
diff --git a/be/src/vec/functions/like.cpp b/be/src/vec/functions/like.cpp
index f0c37787ad..c6bf479582 100644
--- a/be/src/vec/functions/like.cpp
+++ b/be/src/vec/functions/like.cpp
@@ -76,7 +76,7 @@ Status
FunctionLikeBase::constant_substring_fn(LikeSearchState* state, const Str
*result = true;
return Status::OK();
}
-StringValue pattern_value(val.ptr, val.len);
+StringValue pattern_value = StringValue::from_string_val(val.ptr);
*result = state->substring_pattern.search(&pattern_value) != -1;
return Status::OK();
}
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[doris] branch dev-1.1.2 updated: [Bug](tablet) Fix bug that segment files are removed as trash but tablet meta is normal (#10721)
This is an automated email from the ASF dual-hosted git repository.
weizuo pushed a commit to branch dev-1.1.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/dev-1.1.2 by this push:
new 07b7048faa [Bug](tablet) Fix bug that segment files are removed as
trash but tablet meta is normal (#10721)
07b7048faa is described below
commit 07b7048faacc6e5bc6af1b73597b643b767d0f4d
Author: weizuo93
AuthorDate: Fri Jul 15 11:07:16 2022 +0800
[Bug](tablet) Fix bug that segment files are removed as trash but tablet
meta is normal (#10721)
---
be/src/olap/tablet_manager.cpp | 10 ++
1 file changed, 10 insertions(+)
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 66d4031fe9..041124ea61 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -160,6 +160,16 @@ OLAPStatus TabletManager::_add_tablet_unlocked(TTabletId
tablet_id, SchemaHash s
res = _add_tablet_to_map_unlocked(tablet_id, schema_hash, tablet,
update_meta, keep_files,
true /*drop_old*/);
} else {
+tablet->set_tablet_state(TABLET_SHUTDOWN);
+tablet->save_meta();
+{
+std::lock_guard
shutdown_tablets_wrlock(_shutdown_tablets_lock);
+_shutdown_tablets.push_back(tablet);
+}
+LOG(INFO) << "set tablet to shutdown state."
+ << "tablet_id=" << tablet->tablet_id()
+ << ", tablet_path=" << tablet->tablet_path();
+
res = OLAP_ERR_ENGINE_INSERT_OLD_TABLET;
}
LOG(WARNING) << "add duplicated tablet. force=" << force << ", res=" << res
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
