This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit a20439a920e9a77f0c8b9db853ca2c20638224b3
Author: morningman <chenmin...@baidu.com>
AuthorDate: Thu Apr 28 10:22:07 2022 +0800
[feature] add `SHOW TABLET STORAGE FORMAT` stmt (#9037)
use this stmt to show tablets storage format in be, if verbose is set,
will show detail message of tablet storage format.
e.g.
```
MySQL [(none)]> admin show tablet storage format;
+-----------+---------+---------+
| BackendId | V1Count | V2Count |
+-----------+---------+---------+
| 10002 | 0 | 2867 |
+-----------+---------+---------+
1 row in set (0.003 sec)
MySQL [test_query_qa]> admin show tablet storage format verbose;
+-----------+----------+---------------+
| BackendId | TabletId | StorageFormat |
+-----------+----------+---------------+
| 10002 | 39227 | V2 |
| 10002 | 39221 | V2 |
| 10002 | 39215 | V2 |
| 10002 | 39199 | V2 |
+-----------+----------+---------------+
4 rows in set (0.034 sec)
```
add storage format infomation to show full table statment.
```
MySQL [test_query_qa]> show full tables;
+-------------------------+------------+---------------+
| Tables_in_test_query_qa | Table_type | StorageFormat |
+-------------------------+------------+---------------+
| bigtable | BASE TABLE | V2 |
| test_dup | BASE TABLE | V2 |
| test | BASE TABLE | V2 |
| baseall | BASE TABLE | V2 |
| test_string | BASE TABLE | V2 |
+-------------------------+------------+---------------+
5 rows in set (0.002 sec)
```
---
be/src/olap/tablet_manager.cpp | 20 ++++++++
be/src/olap/tablet_manager.h | 2 +
be/src/service/backend_service.cpp | 4 ++
be/src/service/backend_service.h | 2 +
docs/.vuepress/sidebar/en.js | 1 +
docs/.vuepress/sidebar/zh-CN.js | 1 +
fe/fe-core/src/main/cup/sql_parser.cup | 8 +++
.../org/apache/doris/analysis/ShowTableStmt.java | 2 +
.../java/org/apache/doris/qe/ShowExecutor.java | 57 ++++++++++++++++++----
.../java/org/apache/doris/task/AgentClient.java | 17 +++++++
.../apache/doris/analysis/ShowTableStmtTest.java | 6 +--
.../org/apache/doris/common/GenericPoolTest.java | 6 +++
.../apache/doris/utframe/MockedBackendFactory.java | 6 +++
gensrc/thrift/BackendService.thrift | 8 +++
14 files changed, 128 insertions(+), 12 deletions(-)
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 31c74c650f..cf75b8209c 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -1449,4 +1449,24 @@ void
TabletManager::get_tablets_distribution_on_different_disks(
}
}
+void TabletManager::get_all_tablets_storage_format(TCheckStorageFormatResult*
result) {
+ DCHECK(result != nullptr);
+ for (const auto& tablets_shard : _tablets_shards) {
+ std::shared_lock rdlock(tablets_shard.lock);
+ for (const auto& item : tablets_shard.tablet_map) {
+ uint64_t tablet_id = item.first;
+ for (auto& tablet : item.second.table_arr) {
+ if (tablet->all_beta()) {
+ result->v2_tablets.push_back(tablet_id);
+ } else {
+ result->v1_tablets.push_back(tablet_id);
+ }
+ break;
+ }
+ }
+ }
+ result->__isset.v1_tablets = true;
+ result->__isset.v2_tablets = true;
+}
+
} // end namespace doris
diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h
index 9643aefbf5..90f03fa680 100644
--- a/be/src/olap/tablet_manager.h
+++ b/be/src/olap/tablet_manager.h
@@ -144,6 +144,8 @@ public:
std::map<int64_t, std::map<DataDir*, int64_t>>&
tablets_num_on_disk,
std::map<int64_t, std::map<DataDir*, std::vector<TabletSize>>>&
tablets_info_on_disk);
+ void get_all_tablets_storage_format(TCheckStorageFormatResult* result);
+
private:
// Add a tablet pointer to StorageEngine
// If force, drop the existing tablet add this new one
diff --git a/be/src/service/backend_service.cpp
b/be/src/service/backend_service.cpp
index 05eac43034..97e35e1597 100644
--- a/be/src/service/backend_service.cpp
+++ b/be/src/service/backend_service.cpp
@@ -370,4 +370,8 @@ void
BackendService::get_stream_load_record(TStreamLoadRecordResult& result,
void BackendService::clean_trash() {
StorageEngine::instance()->start_trash_sweep(nullptr, true);
}
+
+void BackendService::check_storage_format(TCheckStorageFormatResult& result) {
+
StorageEngine::instance()->tablet_manager()->get_all_tablets_storage_format(&result);
+}
} // namespace doris
diff --git a/be/src/service/backend_service.h b/be/src/service/backend_service.h
index 3c9b3bd13c..7991f59c9f 100644
--- a/be/src/service/backend_service.h
+++ b/be/src/service/backend_service.h
@@ -156,6 +156,8 @@ public:
virtual void clean_trash() override;
+ virtual void check_storage_format(TCheckStorageFormatResult& result)
override;
+
private:
Status start_plan_fragment_execution(const TExecPlanFragmentParams&
exec_params);
ExecEnv* _exec_env;
diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js
index 2361963495..593b1519b6 100644
--- a/docs/.vuepress/sidebar/en.js
+++ b/docs/.vuepress/sidebar/en.js
@@ -589,6 +589,7 @@ module.exports = [
"ADMIN SHOW REPLICA DISTRIBUTION",
"ADMIN SHOW REPLICA STATUS",
"ADMIN-DIAGNOSE-TABLET",
+ "ADMIN SHOW TABLET STORAGE FORMAT",
"ALTER CLUSTER",
"ALTER SYSTEM",
"CANCEL DECOMMISSION",
diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js
index dcb4a87597..8a804841cb 100644
--- a/docs/.vuepress/sidebar/zh-CN.js
+++ b/docs/.vuepress/sidebar/zh-CN.js
@@ -591,6 +591,7 @@ module.exports = [
"ADMIN SHOW REPLICA DISTRIBUTION",
"ADMIN SHOW REPLICA STATUS",
"ADMIN-DIAGNOSE-TABLET",
+ "ADMIN SHOW TABLET STORAGE FORMAT",
"ALTER CLUSTER",
"ALTER SYSTEM",
"CANCEL DECOMMISSION",
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup
b/fe/fe-core/src/main/cup/sql_parser.cup
index 3078d8f068..e60e78c0b4 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -5287,6 +5287,14 @@ admin_stmt ::=
{:
RESULT = new AdminDiagnoseTabletStmt(tabletId);
:}
+ | KW_ADMIN KW_TABLET KW_STORAGE KW_FORMAT
+ {:
+ RESULT = new AdminShowTabletStorageFormatStmt(false);
+ :}
+ | KW_ADMIN KW_TABLET KW_STORAGE KW_FORMAT KW_VERBOSE
+ {:
+ RESULT = new AdminShowTabletStorageFormatStmt(true);
+ :}
;
truncate_stmt ::=
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java
index f81dc9d0d6..41dd44adfd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java
@@ -37,6 +37,7 @@ public class ShowTableStmt extends ShowStmt {
private static final Logger LOG =
LogManager.getLogger(ShowTableStmt.class);
private static final String NAME_COL_PREFIX = "Tables_in_";
private static final String TYPE_COL = "Table_type";
+ private static final String STORAGE_FORMAT_COL = "StorageFormat";
private static final TableName TABLE_NAME = new
TableName(InfoSchemaDb.DATABASE_NAME, "tables");
private String db;
private boolean isVerbose;
@@ -146,6 +147,7 @@ public class ShowTableStmt extends ShowStmt {
new Column(NAME_COL_PREFIX +
ClusterNamespace.getNameFromFullName(db), ScalarType.createVarchar(20)));
if (isVerbose) {
builder.addColumn(new Column(TYPE_COL,
ScalarType.createVarchar(20)));
+ builder.addColumn(new Column(STORAGE_FORMAT_COL,
ScalarType.createVarchar(20)));
}
return builder.build();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 5fb6ade642..4a9d0b7dbb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -21,6 +21,7 @@ import org.apache.doris.analysis.AdminDiagnoseTabletStmt;
import org.apache.doris.analysis.AdminShowConfigStmt;
import org.apache.doris.analysis.AdminShowReplicaDistributionStmt;
import org.apache.doris.analysis.AdminShowReplicaStatusStmt;
+import org.apache.doris.analysis.AdminShowTabletStorageFormatStmt;
import org.apache.doris.analysis.DescribeStmt;
import org.apache.doris.analysis.HelpStmt;
import org.apache.doris.analysis.PartitionNames;
@@ -158,13 +159,14 @@ import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.system.Backend;
import org.apache.doris.system.Diagnoser;
import org.apache.doris.system.SystemInfoService;
+import org.apache.doris.task.AgentClient;
+import org.apache.doris.thrift.TCheckStorageFormatResult;
import org.apache.doris.thrift.TUnit;
import org.apache.doris.transaction.GlobalTransactionMgr;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.tuple.Triple;
@@ -183,7 +185,6 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@@ -335,6 +336,8 @@ public class ShowExecutor {
handleShowTableCreation();
} else if (stmt instanceof ShowLastInsertStmt) {
handleShowLastInsert();
+ } else if (stmt instanceof AdminShowTabletStorageFormatStmt) {
+ handleAdminShowTabletStorageFormat();
} else if (stmt instanceof AdminDiagnoseTabletStmt) {
handleAdminDiagnoseTablet();
} else {
@@ -659,7 +662,6 @@ public class ShowExecutor {
ShowTableStmt showTableStmt = (ShowTableStmt) stmt;
List<List<String>> rows = Lists.newArrayList();
Database db =
ctx.getCatalog().getDbOrAnalysisException(showTableStmt.getDb());
- Map<String, String> tableMap = Maps.newTreeMap();
PatternMatcher matcher = null;
if (showTableStmt.getPattern() != null) {
matcher =
PatternMatcher.createMysqlPattern(showTableStmt.getPattern(),
@@ -675,14 +677,14 @@ public class ShowExecutor {
PrivPredicate.SHOW)) {
continue;
}
- tableMap.put(tbl.getName(), tbl.getMysqlType());
- }
-
- for (Map.Entry<String, String> entry : tableMap.entrySet()) {
if (showTableStmt.isVerbose()) {
- rows.add(Lists.newArrayList(entry.getKey(), entry.getValue()));
+ String storageFormat = "NONE";
+ if (tbl instanceof OlapTable) {
+ storageFormat = ((OlapTable)
tbl).getStorageFormat().toString();
+ }
+ rows.add(Lists.newArrayList(tbl.getName(), tbl.getMysqlType(),
storageFormat));
} else {
- rows.add(Lists.newArrayList(entry.getKey()));
+ rows.add(Lists.newArrayList(tbl.getName()));
}
}
resultSet = new ShowResultSet(showTableStmt.getMetaData(), rows);
@@ -2132,6 +2134,43 @@ public class ShowExecutor {
resultSet = new ShowResultSet(showMetaData, resultRowSet);
}
+ private void handleAdminShowTabletStorageFormat() throws AnalysisException
{
+ List<List<String>> resultRowSet = Lists.newArrayList();
+ for (Backend be :
Catalog.getCurrentSystemInfo().getIdToBackend().values()) {
+ if (be.isQueryAvailable() && be.isLoadAvailable()) {
+ AgentClient client = new AgentClient(be.getHost(),
be.getBePort());
+ TCheckStorageFormatResult result = client.checkStorageFormat();
+ if (result == null) {
+ throw new AnalysisException("get tablet data from backend:
" + be.getId() + "error.");
+ }
+ if (stmt.isVerbose()) {
+ for (long tabletId : result.getV1Tablets()) {
+ List<String> row = new ArrayList<>();
+ row.add(String.valueOf(be.getId()));
+ row.add(String.valueOf(tabletId));
+ row.add("V1");
+ resultRowSet.add(row);
+ }
+ for (long tabletId : result.getV2Tablets()) {
+ List<String> row = new ArrayList<>();
+ row.add(String.valueOf(be.getId()));
+ row.add(String.valueOf(tabletId));
+ row.add("V2");
+ resultRowSet.add(row);
+ }
+ } else {
+ List<String> row = new ArrayList<>();
+ row.add(String.valueOf(be.getId()));
+ row.add(String.valueOf(result.getV1Tablets().size()));
+ row.add(String.valueOf(result.getV2Tablets().size()));
+ resultRowSet.add(row);
+ }
+ }
+ }
+ ShowResultSetMetaData showMetaData = stmt.getMetaData();
+ resultSet = new ShowResultSet(showMetaData, resultRowSet);
+ }
+
private void handleAdminDiagnoseTablet() {
AdminDiagnoseTabletStmt showStmt = (AdminDiagnoseTabletStmt) stmt;
List<List<String>> resultRowSet =
Diagnoser.diagnoseTablet(showStmt.getTabletId());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/AgentClient.java
b/fe/fe-core/src/main/java/org/apache/doris/task/AgentClient.java
index a62cb0f2f0..22dce44c3c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/AgentClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/AgentClient.java
@@ -22,6 +22,7 @@ import org.apache.doris.common.Status;
import org.apache.doris.thrift.BackendService;
import org.apache.doris.thrift.TAgentResult;
import org.apache.doris.thrift.TAgentServiceVersion;
+import org.apache.doris.thrift.TCheckStorageFormatResult;
import org.apache.doris.thrift.TMiniLoadEtlStatusRequest;
import org.apache.doris.thrift.TMiniLoadEtlStatusResult;
import org.apache.doris.thrift.TMiniLoadEtlTaskRequest;
@@ -167,6 +168,21 @@ public class AgentClient {
return result;
}
+ public TCheckStorageFormatResult checkStorageFormat() {
+ TCheckStorageFormatResult result = null;
+ LOG.debug("submit make snapshot task.");
+ try {
+ borrowClient();
+ result = client.checkStorageFormat();
+ ok = true;
+ } catch (Exception e) {
+ LOG.warn("checkStorageFormat error", e);
+ } finally {
+ returnClient();
+ }
+ return result;
+ }
+
public void deleteEtlFiles(long dbId, long jobId, String dbName, String
label) {
TDeleteEtlFilesRequest request = new
TDeleteEtlFilesRequest(TAgentServiceVersion.V1,
new TUniqueId(dbId, jobId), dbName, label);
@@ -197,4 +213,5 @@ public class AgentClient {
ClientPool.backendPool.invalidateObject(address, client);
}
}
+
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowTableStmtTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowTableStmtTest.java
index 1e7a97312f..89c7336610 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowTableStmtTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowTableStmtTest.java
@@ -56,7 +56,7 @@ public class ShowTableStmtTest {
stmt = new ShowTableStmt("abc", true, null);
stmt.analyze(analyzer);
Assert.assertEquals("SHOW FULL TABLES FROM testCluster:abc",
stmt.toString());
- Assert.assertEquals(2, stmt.getMetaData().getColumnCount());
+ Assert.assertEquals(3, stmt.getMetaData().getColumnCount());
Assert.assertEquals("Tables_in_abc",
stmt.getMetaData().getColumn(0).getName());
Assert.assertEquals("Table_type",
stmt.getMetaData().getColumn(1).getName());
@@ -64,7 +64,7 @@ public class ShowTableStmtTest {
stmt.analyze(analyzer);
Assert.assertEquals("bcd", stmt.getPattern());
Assert.assertEquals("SHOW FULL TABLES FROM testCluster:abc LIKE
'bcd'", stmt.toString());
- Assert.assertEquals(2, stmt.getMetaData().getColumnCount());
+ Assert.assertEquals(3, stmt.getMetaData().getColumnCount());
Assert.assertEquals("Tables_in_abc",
stmt.getMetaData().getColumn(0).getName());
Assert.assertEquals("Table_type",
stmt.getMetaData().getColumn(1).getName());
}
@@ -75,4 +75,4 @@ public class ShowTableStmtTest {
stmt.analyze(AccessTestUtil.fetchEmptyDbAnalyzer());
Assert.fail("No exception throws");
}
-}
\ No newline at end of file
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java
index 3247e5e09c..feab0c256e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java
@@ -24,6 +24,7 @@ import org.apache.doris.thrift.TAgentResult;
import org.apache.doris.thrift.TAgentTaskRequest;
import org.apache.doris.thrift.TCancelPlanFragmentParams;
import org.apache.doris.thrift.TCancelPlanFragmentResult;
+import org.apache.doris.thrift.TCheckStorageFormatResult;
import org.apache.doris.thrift.TDeleteEtlFilesRequest;
import org.apache.doris.thrift.TDiskTrashInfo;
import org.apache.doris.thrift.TExecPlanFragmentParams;
@@ -242,6 +243,11 @@ public class GenericPoolTest {
public void cleanTrash() throws TException {
// TODO Auto-generated method stub
}
+
+ @Override
+ public TCheckStorageFormatResult checkStorageFormat() throws
TException {
+ return new TCheckStorageFormatResult();
+ }
}
@Test
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java
b/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java
index 42dab10411..bc3e7928fd 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java
@@ -32,6 +32,7 @@ import org.apache.doris.thrift.TBackend;
import org.apache.doris.thrift.TBackendInfo;
import org.apache.doris.thrift.TCancelPlanFragmentParams;
import org.apache.doris.thrift.TCancelPlanFragmentResult;
+import org.apache.doris.thrift.TCheckStorageFormatResult;
import org.apache.doris.thrift.TCloneReq;
import org.apache.doris.thrift.TDeleteEtlFilesRequest;
import org.apache.doris.thrift.TDiskTrashInfo;
@@ -319,6 +320,11 @@ public class MockedBackendFactory {
public void cleanTrash() throws TException {
return;
}
+
+ @Override
+ public TCheckStorageFormatResult checkStorageFormat() throws
TException {
+ return new TCheckStorageFormatResult();
+ }
}
// The default Brpc service.
diff --git a/gensrc/thrift/BackendService.thrift
b/gensrc/thrift/BackendService.thrift
index 4517076f00..8e534f4677 100644
--- a/gensrc/thrift/BackendService.thrift
+++ b/gensrc/thrift/BackendService.thrift
@@ -113,6 +113,11 @@ struct TDiskTrashInfo {
3: required i64 trash_used_capacity
}
+struct TCheckStorageFormatResult {
+ 1: optional list<i64> v1_tablets;
+ 2: optional list<i64> v2_tablets;
+}
+
service BackendService {
// Called by coord to start asynchronous execution of plan fragment in
backend.
// Returns as soon as all incoming data streams have been set up.
@@ -174,4 +179,7 @@ service BackendService {
TStreamLoadRecordResult get_stream_load_record(1: i64
last_stream_record_time);
oneway void clean_trash();
+
+ // check tablet rowset type
+ TCheckStorageFormatResult check_storage_format();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
