caishunfeng commented on code in PR #16287:
URL:
https://github.com/apache/dolphinscheduler/pull/16287#discussion_r1673552342
##########
dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql:
##########
@@ -2185,3 +2185,64 @@ CREATE TABLE t_ds_listener_event
KEY idx_status (post_status),
KEY idx_event_sign (sign)
);
+
+
+-- ----------------------------
+-- Table structure for jdbc registry
+-- ----------------------------
+
+DROP TABLE IF EXISTS `t_ds_jdbc_registry_data`;
+CREATE TABLE `t_ds_jdbc_registry_data`
+(
+ `id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'primary
key',
+ `data_key` varchar(256) NOT NULL COMMENT 'key, like zookeeper node
path',
+ `data_value` text NOT NULL COMMENT 'data, like zookeeper
node value',
+ `data_type` varchar(64) NOT NULL COMMENT 'EPHEMERAL, PERSISTENT',
+ `client_id` bigint NOT NULL COMMENT 'client id',
+ `create_time` timestamp NOT NULL default current_timestamp COMMENT
'create time',
+ `last_update_time` timestamp NOT NULL default current_timestamp COMMENT
'last update time',
+ PRIMARY KEY (`id`),
+ unique KEY `uk_t_ds_jdbc_registry_dataKey`(`data_key`)
+) ENGINE = InnoDB
+ DEFAULT CHARSET = utf8;
+
+
+DROP TABLE IF EXISTS `t_ds_jdbc_registry_lock`;
+CREATE TABLE `t_ds_jdbc_registry_lock`
+(
+ `id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `lock_key` varchar(256) NOT NULL COMMENT 'lock path',
+ `lock_owner` varchar(256) NOT NULL COMMENT 'the lock owner, ip_processId',
+ `client_id` bigint NOT NULL COMMENT 'client id',
+ `create_time` timestamp NOT NULL default current_timestamp COMMENT
'create time',
+ PRIMARY KEY (`id`),
+ unique KEY `uk_t_ds_jdbc_registry_lockKey`(`lock_key`)
+) ENGINE = InnoDB
+ DEFAULT CHARSET = utf8;
+
+DROP TABLE IF EXISTS `t_ds_jdbc_registry_client_heartbeat`;
+CREATE TABLE `t_ds_jdbc_registry_client_heartbeat`
Review Comment:
Heartbeat is just an attribute of client, so maybe registry_client is better.
```suggestion
CREATE TABLE `t_ds_jdbc_registry_client`
```
##########
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh:
##########
@@ -45,8 +45,12 @@ export
DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py}
export
PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH
-export MASTER_RESERVED_MEMORY=0.01
-export WORKER_RESERVED_MEMORY=0.01
+export
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS=0.7
Review Comment:
The k8s config also needs to be adjusted too.
##########
dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql:
##########
@@ -2185,3 +2185,64 @@ CREATE TABLE t_ds_listener_event
KEY idx_status (post_status),
KEY idx_event_sign (sign)
);
+
+
+-- ----------------------------
+-- Table structure for jdbc registry
+-- ----------------------------
+
+DROP TABLE IF EXISTS `t_ds_jdbc_registry_data`;
+CREATE TABLE `t_ds_jdbc_registry_data`
+(
+ `id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'primary
key',
+ `data_key` varchar(256) NOT NULL COMMENT 'key, like zookeeper node
path',
+ `data_value` text NOT NULL COMMENT 'data, like zookeeper
node value',
+ `data_type` varchar(64) NOT NULL COMMENT 'EPHEMERAL, PERSISTENT',
+ `client_id` bigint NOT NULL COMMENT 'client id',
+ `create_time` timestamp NOT NULL default current_timestamp COMMENT
'create time',
+ `last_update_time` timestamp NOT NULL default current_timestamp COMMENT
'last update time',
+ PRIMARY KEY (`id`),
+ unique KEY `uk_t_ds_jdbc_registry_dataKey`(`data_key`)
+) ENGINE = InnoDB
+ DEFAULT CHARSET = utf8;
+
+
+DROP TABLE IF EXISTS `t_ds_jdbc_registry_lock`;
+CREATE TABLE `t_ds_jdbc_registry_lock`
+(
+ `id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `lock_key` varchar(256) NOT NULL COMMENT 'lock path',
+ `lock_owner` varchar(256) NOT NULL COMMENT 'the lock owner, ip_processId',
+ `client_id` bigint NOT NULL COMMENT 'client id',
+ `create_time` timestamp NOT NULL default current_timestamp COMMENT
'create time',
+ PRIMARY KEY (`id`),
+ unique KEY `uk_t_ds_jdbc_registry_lockKey`(`lock_key`)
+) ENGINE = InnoDB
+ DEFAULT CHARSET = utf8;
+
+DROP TABLE IF EXISTS `t_ds_jdbc_registry_client_heartbeat`;
+CREATE TABLE `t_ds_jdbc_registry_client_heartbeat`
+(
+ `id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'primary
key',
+ `client_name` varchar(256) NOT NULL COMMENT 'client name,
ip_processId',
+ `last_heartbeat_time` bigint NOT NULL COMMENT 'last heartbeat
timestamp',
+ `connection_config` text NOT NULL COMMENT 'connection config',
+ `create_time` timestamp NOT NULL default current_timestamp
COMMENT 'create time',
+ PRIMARY KEY (`id`)
+) ENGINE = InnoDB
+ DEFAULT CHARSET = utf8;
+
+DROP TABLE IF EXISTS `t_ds_jdbc_registry_data_change_event`;
+CREATE TABLE `t_ds_jdbc_registry_data_change_event`
+(
+ `id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'primary
key',
+ `event_type` varchar(64) NOT NULL COMMENT 'ADD, UPDATE, DELETE',
+ `jdbc_registry_data` text NOT NULL COMMENT 'jdbc registry data',
Review Comment:
```suggestion
`registry_data` text NOT NULL COMMENT 'registry data',
```
##########
dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/PostgresqlJdbcRegistryTestCase.java:
##########
@@ -63,30 +63,50 @@ public static void setUpTestingServer() {
statement.execute(
Review Comment:
Can execute the init.sql directly?
##########
dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/server/IRegistryRowChangeNotifier.java:
##########
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.registry.jdbc.server;
+
+public interface IRegistryRowChangeNotifier<T> {
+
+ int DEFAULT_DETECT_INTERVAL = 1000;
+
+ void start();
+
+ void subscribeRegistryRowChange(RegistryRowChangeListener<T>
registryRowChangeListener);
+
+ interface RegistryRowChangeListener<T> {
Review Comment:
`RegistryRow` means `RegistryData`?
##########
dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/server/IRegistryRowChangeNotifier.java:
##########
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.registry.jdbc.server;
+
+public interface IRegistryRowChangeNotifier<T> {
+
+ int DEFAULT_DETECT_INTERVAL = 1000;
+
+ void start();
+
+ void subscribeRegistryRowChange(RegistryRowChangeListener<T>
registryRowChangeListener);
+
+ interface RegistryRowChangeListener<T> {
Review Comment:
row is a database concept, and it is best to use logical concepts.
##########
dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/LockUtilsTest.java:
##########
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.plugin.registry.jdbc;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import org.junit.jupiter.api.Test;
+
+class LockUtilsTest {
+
+ @Test
+ void getLockOwner() {
+ assertThat(LockUtils.getLockOwner()).isNotNull();
+
assertThat(LockUtils.getLockOwner()).isEqualTo(LockUtils.getLockOwner());
Review Comment:
What's test target in these cases? Please add some comments.
##########
dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/mapper/JdbcRegistryDataChanceEventMapper.java:
##########
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.registry.jdbc.mapper;
+
+import
org.apache.dolphinscheduler.plugin.registry.jdbc.model.DO.JdbcRegistryDataChanceEvent;
+
+import org.apache.ibatis.annotations.Delete;
+import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.Select;
+
+import java.util.Date;
+import java.util.List;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface JdbcRegistryDataChanceEventMapper extends
BaseMapper<JdbcRegistryDataChanceEvent> {
+
+ @Select("select max(id) from t_ds_jdbc_registry_data_change_event")
+ Long getMaxId();
+
+ @Select("select * from t_ds_jdbc_registry_data_change_event where id >
#{id} order by id asc limit 1000")
Review Comment:
```suggestion
@Select("select * from t_ds_jdbc_registry_data_change_event where id >
#{id} limit 1000")
```
##########
dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql:
##########
@@ -2185,3 +2185,64 @@ CREATE TABLE t_ds_listener_event
KEY idx_status (post_status),
KEY idx_event_sign (sign)
);
+
+
+-- ----------------------------
+-- Table structure for jdbc registry
+-- ----------------------------
+
+DROP TABLE IF EXISTS `t_ds_jdbc_registry_data`;
+CREATE TABLE `t_ds_jdbc_registry_data`
+(
+ `id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'primary
key',
+ `data_key` varchar(256) NOT NULL COMMENT 'key, like zookeeper node
path',
+ `data_value` text NOT NULL COMMENT 'data, like zookeeper
node value',
+ `data_type` varchar(64) NOT NULL COMMENT 'EPHEMERAL, PERSISTENT',
+ `client_id` bigint NOT NULL COMMENT 'client id',
+ `create_time` timestamp NOT NULL default current_timestamp COMMENT
'create time',
+ `last_update_time` timestamp NOT NULL default current_timestamp COMMENT
'last update time',
+ PRIMARY KEY (`id`),
+ unique KEY `uk_t_ds_jdbc_registry_dataKey`(`data_key`)
+) ENGINE = InnoDB
+ DEFAULT CHARSET = utf8;
+
+
+DROP TABLE IF EXISTS `t_ds_jdbc_registry_lock`;
+CREATE TABLE `t_ds_jdbc_registry_lock`
+(
+ `id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `lock_key` varchar(256) NOT NULL COMMENT 'lock path',
+ `lock_owner` varchar(256) NOT NULL COMMENT 'the lock owner, ip_processId',
+ `client_id` bigint NOT NULL COMMENT 'client id',
+ `create_time` timestamp NOT NULL default current_timestamp COMMENT
'create time',
+ PRIMARY KEY (`id`),
+ unique KEY `uk_t_ds_jdbc_registry_lockKey`(`lock_key`)
+) ENGINE = InnoDB
+ DEFAULT CHARSET = utf8;
+
+DROP TABLE IF EXISTS `t_ds_jdbc_registry_client_heartbeat`;
+CREATE TABLE `t_ds_jdbc_registry_client_heartbeat`
+(
+ `id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'primary
key',
+ `client_name` varchar(256) NOT NULL COMMENT 'client name,
ip_processId',
+ `last_heartbeat_time` bigint NOT NULL COMMENT 'last heartbeat
timestamp',
+ `connection_config` text NOT NULL COMMENT 'connection config',
+ `create_time` timestamp NOT NULL default current_timestamp
COMMENT 'create time',
+ PRIMARY KEY (`id`)
+) ENGINE = InnoDB
+ DEFAULT CHARSET = utf8;
+
+DROP TABLE IF EXISTS `t_ds_jdbc_registry_data_change_event`;
+CREATE TABLE `t_ds_jdbc_registry_data_change_event`
Review Comment:
is not data change like client online or offline.
```suggestion
CREATE TABLE `t_ds_jdbc_registry_client_change_event`
```
##########
dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/server/JdbcRegistryServerState.java:
##########
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.registry.jdbc.server;
+
+public enum JdbcRegistryServerState {
+ INIT,
+ STARTED,
+ SUSPENDED,
+ DISCONNECTED,
+ STOPPED,
+ ;
+
+}
Review Comment:
Should add some comments for every state.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
