This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e8da5d2e8de [fix](cloud) Support cloud multi cluster run in docker
(#37925)
e8da5d2e8de is described below
commit e8da5d2e8de05d6301795c1134218229083f67de
Author: deardeng <565620...@qq.com>
AuthorDate: Wed Jul 17 10:06:52 2024 +0800
[fix](cloud) Support cloud multi cluster run in docker (#37925)
---
docker/runtime/doris-compose/command.py | 2 +
.../org/apache/doris/regression/suite/Suite.groovy | 14 +++-
.../doris/regression/suite/SuiteCluster.groovy | 87 ++++++++++++++++++----
.../apache/doris/regression/util/DebugPoint.groovy | 3 +
4 files changed, 90 insertions(+), 16 deletions(-)
diff --git a/docker/runtime/doris-compose/command.py
b/docker/runtime/doris-compose/command.py
index 87ae862236a..b0863eae153 100644
--- a/docker/runtime/doris-compose/command.py
+++ b/docker/runtime/doris-compose/command.py
@@ -707,6 +707,8 @@ class ListNode(object):
http_port = CLUSTER.FE_HTTP_PORT
elif self.node_type == CLUSTER.Node.TYPE_BE:
http_port = CLUSTER.BE_WEBSVR_PORT
+ elif self.node_type == CLUSTER.Node.TYPE_MS or self.node_type ==
CLUSTER.Node.TYPE_RECYCLE:
+ http_port = CLUSTER.MS_PORT
else:
pass
result += [
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index fec337f59ed..50d430b6c6d 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -1570,7 +1570,14 @@ class Suite implements GroovyInterceptable {
json.result.cluster
}
- def drop_cluster = { cluster_name, cluster_id ->
+ // cloud
+ String getCloudBeTagByName(String clusterName) {
+ def bes = sql_return_maparray "show backends"
+ def be = bes.stream().filter(be ->
be.Tag.contains(clusterName)).findFirst().orElse(null)
+ return be.Tag
+ }
+
+ def drop_cluster = { cluster_name, cluster_id, MetaService ms=null ->
def jsonOutput = new JsonOutput()
def reqBody = [
type: "COMPUTE",
@@ -1585,6 +1592,11 @@ class Suite implements GroovyInterceptable {
def drop_cluster_api = { request_body, check_func ->
httpTest {
+ if (ms) {
+ endpoint ms.host+':'+ms.httpPort
+ } else {
+ endpoint context.config.metaServiceHttpAddress
+ }
endpoint context.config.metaServiceHttpAddress
uri "/MetaService/http/drop_cluster?token=${token}"
body request_body
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
index 49bfbc18792..68a57ad27f3 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
@@ -170,6 +170,34 @@ class Backend extends ServerNode {
}
+class MetaService extends ServerNode {
+
+ static MetaService fromCompose(ListHeader header, int index, List<Object>
fields) {
+ MetaService ms = new MetaService()
+ ServerNode.fromCompose(ms, header, index, fields)
+ return ms
+ }
+
+ NodeType getNodeType() {
+ return NodeType.MS
+ }
+
+}
+
+class Recycler extends ServerNode {
+
+ static Recycler fromCompose(ListHeader header, int index, List<Object>
fields) {
+ Recycler rs = new Recycler()
+ ServerNode.fromCompose(rs, header, index, fields)
+ return rs
+ }
+
+ NodeType getNodeType() {
+ return NodeType.RECYCLER
+ }
+
+}
+
@Slf4j
@CompileStatic
class SuiteCluster {
@@ -287,21 +315,39 @@ class SuiteCluster {
return getBackends().stream().filter(be -> be.alive ||
!needAlive).collect(Collectors.toList());
}
+ List<MetaService> getAllMetaservices(boolean needAlive = false) {
+ return getMetaservices().stream().filter(ms -> ms.alive ||
!needAlive).collect(Collectors.toList());
+ }
+
+ List<MetaService> getAllRecyclers(boolean needAlive = false) {
+ return getRecyclers().stream().filter(rc -> rc.alive ||
!needAlive).collect(Collectors.toList());
+ }
+
private List<Frontend> getFrontends() {
- List<Frontend> frontends = []
- List<Backend> backends = []
- getAllNodes(frontends, backends)
- return frontends
+ def ret = getAllNodes()
+ return ret.getV1()
}
private List<Backend> getBackends() {
- List<Frontend> frontends = []
- List<Backend> backends = []
- getAllNodes(frontends, backends)
- return backends
+ def ret = getAllNodes()
+ return ret.getV2()
}
- private void getAllNodes(List<Frontend> frontends, List<Backend> backends)
{
+ private List<MetaService> getMetaservices() {
+ def ret = getAllNodes()
+ return ret.getV3()
+ }
+
+ private List<Recycler> getRecyclers() {
+ def ret = getAllNodes()
+ return ret.getV4()
+ }
+
+ private Tuple4<List<Frontend>, List<Backend>, List<MetaService>,
List<Recycler>> getAllNodes() {
+ List<Frontend> frontends = []
+ List<Backend> backends = []
+ List<MetaService> metaservices = []
+ List<Recycler> recyclers = []
def cmd = 'ls ' + name + ' --detail'
def data = runCmd(cmd)
assert data instanceof List
@@ -316,25 +362,33 @@ class SuiteCluster {
} else if (name.startsWith('fe-')) {
int index = name.substring('fe-'.length()) as int
frontends.add(Frontend.fromCompose(header, index, row))
- } else if (name.startsWith('ms-') || name.startsWith('recycle-')
|| name.startsWith('fdb-')) {
- // TODO: handle these nodes
+ } else if (name.startsWith('ms-')){
+ int index = name.substring('ms-'.length()) as int
+ metaservices.add(MetaService.fromCompose(header, index, row))
+ } else if (name.startsWith('recycle-')){
+ int index = name.substring('recycle-'.length()) as int
+ recyclers.add(Recycler.fromCompose(header, index, row))
+ } else if (name.startsWith('fdb-')) {
+ // current not used
} else {
assert false : 'Unknown node type with name: ' + name
}
}
+ return new Tuple4(frontends, backends, metaservices, recyclers)
}
List<Integer> addFrontend(int num) throws Exception {
- def result = add(num, 0)
+ def result = add(num, 0, null)
return result.first
}
- List<Integer> addBackend(int num) throws Exception {
- def result = add(0, num)
+ List<Integer> addBackend(int num, String ClusterName="") throws Exception {
+ def result = add(0, num, ClusterName)
return result.second
}
- Tuple2<List<Integer>, List<Integer>> add(int feNum, int beNum) throws
Exception {
+ // APPR: clusterName just used for cloud mode, 1 cluster has n bes
+ Tuple2<List<Integer>, List<Integer>> add(int feNum, int beNum, String
clusterName) throws Exception {
assert feNum > 0 || beNum > 0
def sb = new StringBuilder()
@@ -344,6 +398,9 @@ class SuiteCluster {
}
if (beNum > 0) {
sb.append('--add-be-num ' + beNum + ' ')
+ if (clusterName != null && !clusterName.isEmpty()) {
+ sb.append(' --be-cluster ' + clusterName + ' ')
+ }
}
sb.append('--wait-timeout 60')
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/DebugPoint.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/DebugPoint.groovy
index a28683f0ee9..8c764eb453d 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/DebugPoint.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/DebugPoint.groovy
@@ -24,6 +24,9 @@ import org.slf4j.LoggerFactory
enum NodeType {
FE,
BE,
+ // cloud
+ MS,
+ RECYCLER,
}
class DebugPoint {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
