This is an automated email from the ASF dual-hosted git repository. airborne pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push: new 0c0b1095c76 [fix] (regression test) Fix unstable regression test (#36563) (#39055) 0c0b1095c76 is described below commit 0c0b1095c767de0aece60770959529bdc9c16338 Author: Sun Chenyang <csun5...@gmail.com> AuthorDate: Fri Aug 9 15:04:19 2024 +0800 [fix] (regression test) Fix unstable regression test (#36563) (#39055) ## Proposed changes pick from master #36563 --- .../test_compaction_with_visible_version.groovy | 7 +- .../test_single_compaction_fault_injection.groovy | 419 +++++++++----------- ...e_compaction_with_variant_inverted_index.groovy | 381 +++++++++--------- .../test_single_replica_compaction.groovy | 434 +++++++++------------ .../test_time_series_compaction_policy.groovy | 11 +- .../test_segcompaction_fault_injection.groovy | 18 - .../test_too_many_segments_fault_injection.groovy | 19 - .../schema_change/test_number_overflow.groovy | 20 - .../test_agg_keys_schema_change.groovy | 17 - .../test_agg_mv_schema_change.groovy | 17 - .../test_agg_rollup_schema_change.groovy | 17 - .../test_agg_vals_schema_change.groovy | 279 +++++++------ .../test_dup_keys_schema_change.groovy | 17 - .../test_dup_mv_schema_change.groovy | 16 - .../test_dup_rollup_schema_change.groovy | 15 - .../test_dup_vals_schema_change.groovy | 15 - .../test_uniq_keys_schema_change.groovy | 250 ++++++------ .../test_uniq_mv_schema_change.groovy | 15 - .../test_uniq_rollup_schema_change.groovy | 15 - .../test_uniq_vals_schema_change.groovy | 15 - .../test_varchar_schema_change.groovy | 16 - .../test_segcompaction_agg_keys.groovy | 19 - .../test_segcompaction_agg_keys_index.groovy | 18 - .../test_segcompaction_dup_keys.groovy | 20 - .../test_segcompaction_dup_keys_index.groovy | 19 - .../test_segcompaction_unique_keys.groovy | 20 - .../test_segcompaction_unique_keys_index.groovy | 19 - .../test_segcompaction_unique_keys_mow.groovy | 20 - ...test_segcompaction_unique_keys_mow_index.groovy | 20 - 29 files changed, 825 insertions(+), 1343 deletions(-) diff --git a/regression-test/suites/compaction/test_compaction_with_visible_version.groovy b/regression-test/suites/compaction/test_compaction_with_visible_version.groovy index 194a1b67566..4a6ee4c847a 100644 --- a/regression-test/suites/compaction/test_compaction_with_visible_version.groovy +++ b/regression-test/suites/compaction/test_compaction_with_visible_version.groovy @@ -26,7 +26,6 @@ suite('test_compaction_with_visible_version') { 'partition_info_update_interval_secs=5', ] options.beConfigs += [ - 'disable_auto_compaction=true', 'report_tablet_interval_seconds=1', 'tablet_rowset_stale_sweep_by_size=true', 'tablet_rowset_stale_sweep_threshold_size=0', @@ -166,7 +165,11 @@ suite('test_compaction_with_visible_version') { } } - sql " CREATE TABLE ${tableName} (k1 int, k2 int) DISTRIBUTED BY HASH(k1) BUCKETS 1 " + sql """ + CREATE TABLE ${tableName} (k1 int, k2 int) DISTRIBUTED BY HASH(k1) BUCKETS 1 PROPERTIES ( + "disable_auto_compaction" = "true" + ) + """ // normal def rowNum = 0L diff --git a/regression-test/suites/compaction/test_single_compaction_fault_injection.groovy b/regression-test/suites/compaction/test_single_compaction_fault_injection.groovy index ebc74257032..839bfaa10dd 100644 --- a/regression-test/suites/compaction/test_single_compaction_fault_injection.groovy +++ b/regression-test/suites/compaction/test_single_compaction_fault_injection.groovy @@ -17,19 +17,12 @@ import org.codehaus.groovy.runtime.IOGroovyMethods -suite("test_single_compaction_fault_injection", "p2") { +suite("test_single_compaction_fault_injection", "p2, nonConcurrent") { def tableName = "test_single_compaction" - - def set_be_config = { key, value -> - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - for (String backend_id: backendId_to_backendIP.keySet()) { - def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) - logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) - } - } + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); def triggerCompaction = { be_host, be_http_port, compact_type, tablet_id -> if (compact_type == "cumulative") { @@ -123,253 +116,209 @@ suite("test_single_compaction_fault_injection", "p2") { return tabletStatus } - boolean disableAutoCompaction = true + // find the master be for single compaction + Boolean found = false + String master_backend_id + List<String> follower_backend_id = new ArrayList<>() + String tablet_id + def tablets try { - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } - set_be_config.call("disable_auto_compaction", "true") - set_be_config.call("update_replica_infos_interval_seconds", "5") - - - // find the master be for single compaction - Boolean found = false - String master_backend_id - List<String> follower_backend_id = new ArrayList<>() - String tablet_id - def tablets - try { - GetDebugPoint().enableDebugPointForAllFEs('getTabletReplicaInfos.returnEmpty') - sql """ DROP TABLE IF EXISTS ${tableName}; """ - sql """ - CREATE TABLE ${tableName} ( - `id` int(11) NULL, - `name` varchar(255) NULL, - `score` int(11) NULL - ) ENGINE=OLAP - UNIQUE KEY(`id`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( "replication_num" = "2", "enable_single_replica_compaction" = "true", "enable_unique_key_merge_on_write" = "false" ); - """ + GetDebugPoint().enableDebugPointForAllFEs('getTabletReplicaInfos.returnEmpty') + sql """ DROP TABLE IF EXISTS ${tableName}; """ + sql """ + CREATE TABLE ${tableName} ( + `id` int(11) NULL, + `name` varchar(255) NULL, + `score` int(11) NULL + ) ENGINE=OLAP + UNIQUE KEY(`id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( "replication_num" = "2", "enable_single_replica_compaction" = "true", "enable_unique_key_merge_on_write" = "false", "compaction_policy" = "time_series"); + """ - tablets = sql_return_maparray """ show tablets from ${tableName}; """ - // wait for update replica infos - Thread.sleep(20000) - // The test table only has one bucket with 2 replicas, - // and `show tablets` will return 2 different replicas with the same tablet. - // So we can use the same tablet_id to get tablet/trigger compaction with different backends. - tablet_id = tablets[0].TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - logger.info("tablet: " + tablet_info) - for (def tablet in tablets) { - String trigger_backend_id = tablet.BackendId - def tablet_status = getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) - if (!tablet_status.containsKey("single replica compaction status")) { - if (found) { - found = false - logger.warn("multipe master"); - break; - } - found = true - master_backend_id = trigger_backend_id - } else { - follower_backend_id.add(trigger_backend_id) - } - } - assertFalse(found) - assertFalse(master_backend_id.isEmpty()) - assertTrue(follower_backend_id.isEmpty()) - master_backend_id = "" - } finally { - GetDebugPoint().disableDebugPointForAllFEs('getTabletReplicaInfos.returnEmpty') - // wait for update replica infos - // be.conf: update_replica_infos_interval_seconds + 2s - Thread.sleep(20000) - // The test table only has one bucket with 2 replicas, - // and `show tablets` will return 2 different replicas with the same tablet. - // So we can use the same tablet_id to get tablet/trigger compaction with different backends. - tablet_id = tablets[0].TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - for (def tablet in tablets) { - String trigger_backend_id = tablet.BackendId - def tablet_status = getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); - if (!tablet_status.containsKey("single replica compaction status")) { - if (found) { - logger.warn("multipe master") - assertTrue(false) - } - found = true - master_backend_id = trigger_backend_id - } else { - follower_backend_id.add(trigger_backend_id) + tablets = sql_return_maparray """ show tablets from ${tableName}; """ + // wait for update replica infos + Thread.sleep(70000) + // The test table only has one bucket with 2 replicas, + // and `show tablets` will return 2 different replicas with the same tablet. + // So we can use the same tablet_id to get tablet/trigger compaction with different backends. + tablet_id = tablets[0].TabletId + def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ + logger.info("tablet: " + tablet_info) + for (def tablet in tablets) { + String trigger_backend_id = tablet.BackendId + def tablet_status = getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) + if (!tablet_status.containsKey("single replica compaction status")) { + if (found) { + found = false + logger.warn("multipe master"); + break; } - } - assertTrue(found) - assertFalse(master_backend_id.isEmpty()) - assertFalse(follower_backend_id.isEmpty()) - } - - - def checkSucceedCompactionResult = { - def master_tablet_status = getTabletStatus(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id); - def master_rowsets = master_tablet_status."rowsets" - assert master_rowsets instanceof List - logger.info("rowset size: " + master_rowsets.size()) - - for (String backend: follower_backend_id) { - def tablet_status = getTabletStatus(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id); - def rowsets = tablet_status."rowsets" - assert rowsets instanceof List - assertEquals(master_rowsets.size(), rowsets.size()) + found = true + master_backend_id = trigger_backend_id + } else { + follower_backend_id.add(trigger_backend_id) } } - - def checkFailedCompactionResult = { - def master_tablet_status = getTabletStatus(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id); - def master_rowsets = master_tablet_status."rowsets" - assert master_rowsets instanceof List - logger.info("rowset size: " + master_rowsets.size()) - - for (String backend: follower_backend_id) { - def tablet_status = getTabletStatus(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id); - def rowsets = tablet_status."rowsets" - assert rowsets instanceof List - assertFalse(master_rowsets.size() == rowsets.size()) - } - } - - // return ok - try { - GetDebugPoint().enableDebugPointForAllBEs("do_single_compaction_return_ok"); - for (String id in follower_backend_id) { - assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id); + assertFalse(found) + assertFalse(master_backend_id.isEmpty()) + assertTrue(follower_backend_id.isEmpty()) + master_backend_id = "" + } finally { + GetDebugPoint().disableDebugPointForAllFEs('getTabletReplicaInfos.returnEmpty') + // wait for update replica infos + Thread.sleep(70000) + // The test table only has one bucket with 2 replicas, + // and `show tablets` will return 2 different replicas with the same tablet. + // So we can use the same tablet_id to get tablet/trigger compaction with different backends. + tablet_id = tablets[0].TabletId + def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ + for (def tablet in tablets) { + String trigger_backend_id = tablet.BackendId + def tablet_status = getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); + if (!tablet_status.containsKey("single replica compaction status")) { + if (found) { + logger.warn("multipe master") + assertTrue(false) + } + found = true + master_backend_id = trigger_backend_id + } else { + follower_backend_id.add(trigger_backend_id) } - } finally { - GetDebugPoint().disableDebugPointForAllBEs("do_single_compaction_return_ok"); } - sql """ INSERT INTO ${tableName} VALUES (1, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (1, "b", 100); """ - sql """ INSERT INTO ${tableName} VALUES (2, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (2, "b", 100); """ - sql """ INSERT INTO ${tableName} VALUES (3, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (3, "b", 100); """ - - // trigger master be to do cumu compaction - assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], - "cumulative", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) - - try { - GetDebugPoint().enableDebugPointForAllBEs("single_compaction_failed_get_peer"); - for (String id in follower_backend_id) { - out = triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) - assertTrue(out.contains("compaction task is successfully triggered") || out.contains("tablet don't have peer replica")); - } - checkFailedCompactionResult.call() - } finally { - GetDebugPoint().disableDebugPointForAllBEs("single_compaction_failed_get_peer") + assertTrue(found) + assertFalse(master_backend_id.isEmpty()) + assertFalse(follower_backend_id.isEmpty()) + } + + + def checkSucceedCompactionResult = { + def master_tablet_status = getTabletStatus(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id); + def master_rowsets = master_tablet_status."rowsets" + assert master_rowsets instanceof List + logger.info("rowset size: " + master_rowsets.size()) + + for (String backend: follower_backend_id) { + def tablet_status = getTabletStatus(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id); + def rowsets = tablet_status."rowsets" + assert rowsets instanceof List + assertEquals(master_rowsets.size(), rowsets.size()) } + } - try { - GetDebugPoint().enableDebugPointForAllBEs("single_compaction_failed_get_peer_versions"); - for (String id in follower_backend_id) { - out = triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) - assertTrue(out.contains("compaction task is successfully triggered") || out.contains("tablet failed get peer versions")); - } - checkFailedCompactionResult.call() - } finally { - GetDebugPoint().disableDebugPointForAllBEs("single_compaction_failed_get_peer_versions") + def checkFailedCompactionResult = { + def master_tablet_status = getTabletStatus(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id); + def master_rowsets = master_tablet_status."rowsets" + assert master_rowsets instanceof List + logger.info("rowset size: " + master_rowsets.size()) + + for (String backend: follower_backend_id) { + def tablet_status = getTabletStatus(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id); + def rowsets = tablet_status."rowsets" + assert rowsets instanceof List + assertFalse(master_rowsets.size() == rowsets.size()) } + } - try { - GetDebugPoint().enableDebugPointForAllBEs("single_compaction_failed_make_snapshot"); - for (String id in follower_backend_id) { - out = triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) - assertTrue(out.contains("compaction task is successfully triggered") || out.contains("failed snapshot")); - } - checkFailedCompactionResult.call() - } finally { - GetDebugPoint().disableDebugPointForAllBEs("single_compaction_failed_make_snapshot") + // return ok + try { + GetDebugPoint().enableDebugPointForAllBEs("do_single_compaction_return_ok"); + for (String id in follower_backend_id) { + assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); + waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id); } + } finally { + GetDebugPoint().disableDebugPointForAllBEs("do_single_compaction_return_ok"); + } + sql """ INSERT INTO ${tableName} VALUES (1, "a", 100); """ + sql """ INSERT INTO ${tableName} VALUES (1, "b", 100); """ + sql """ INSERT INTO ${tableName} VALUES (2, "a", 100); """ + sql """ INSERT INTO ${tableName} VALUES (2, "b", 100); """ + sql """ INSERT INTO ${tableName} VALUES (3, "a", 100); """ + sql """ INSERT INTO ${tableName} VALUES (3, "b", 100); """ + + // trigger master be to do compaction + assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], + "full", tablet_id).contains("Success")); + waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) - try { - GetDebugPoint().enableDebugPointForAllBEs("single_compaction_failed_download_file"); - for (String id in follower_backend_id) { - out = triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) - assertTrue(out.contains("compaction task is successfully triggered") || out.contains("failed to download file")); - } - checkFailedCompactionResult.call() - } finally { - GetDebugPoint().disableDebugPointForAllBEs("single_compaction_failed_download_file") + try { + GetDebugPoint().enableDebugPointForAllBEs("single_compaction_failed_get_peer"); + for (String id in follower_backend_id) { + out = triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) + assertTrue(out.contains("compaction task is successfully triggered") || out.contains("tablet don't have peer replica")); } + checkFailedCompactionResult.call() + } finally { + GetDebugPoint().disableDebugPointForAllBEs("single_compaction_failed_get_peer") + } - // trigger follower be to fetch compaction result + try { + GetDebugPoint().enableDebugPointForAllBEs("single_compaction_failed_get_peer_versions"); for (String id in follower_backend_id) { - assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) + out = triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) + assertTrue(out.contains("compaction task is successfully triggered") || out.contains("tablet failed get peer versions")); } + checkFailedCompactionResult.call() + } finally { + GetDebugPoint().disableDebugPointForAllBEs("single_compaction_failed_get_peer_versions") + } - // check rowsets - checkSucceedCompactionResult.call() - - sql """ INSERT INTO ${tableName} VALUES (4, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (5, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (6, "a", 100); """ - sql """ DELETE FROM ${tableName} WHERE id = 4; """ - sql """ INSERT INTO ${tableName} VALUES (7, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (8, "a", 100); """ - - // trigger master be to do cumu compaction with delete - assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], - "cumulative", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) - - // trigger follower be to fetch compaction result + try { + GetDebugPoint().enableDebugPointForAllBEs("single_compaction_failed_make_snapshot"); for (String id in follower_backend_id) { - assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) + out = triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) + assertTrue(out.contains("compaction task is successfully triggered") || out.contains("failed snapshot")); } + checkFailedCompactionResult.call() + } finally { + GetDebugPoint().disableDebugPointForAllBEs("single_compaction_failed_make_snapshot") + } - // check rowsets - checkSucceedCompactionResult.call() - - // trigger master be to do base compaction - assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], - "full", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) - - // trigger follower be to fetch compaction result + try { + GetDebugPoint().enableDebugPointForAllBEs("single_compaction_failed_download_file"); for (String id in follower_backend_id) { - assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) + out = triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) + assertTrue(out.contains("compaction task is successfully triggered") || out.contains("failed to download file")); } + checkFailedCompactionResult.call() + } finally { + GetDebugPoint().disableDebugPointForAllBEs("single_compaction_failed_download_file") + } - // check rowsets - checkSucceedCompactionResult.call() + // trigger follower be to fetch compaction result + for (String id in follower_backend_id) { + assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); + waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) + } - qt_sql """ - select * from ${tableName} order by id - """ - - } finally { - set_be_config.call("disable_auto_compaction", disableAutoCompaction.toString()) + // check rowsets + checkSucceedCompactionResult.call() + + sql """ INSERT INTO ${tableName} VALUES (4, "a", 100); """ + sql """ INSERT INTO ${tableName} VALUES (5, "a", 100); """ + sql """ INSERT INTO ${tableName} VALUES (6, "a", 100); """ + sql """ DELETE FROM ${tableName} WHERE id = 4; """ + sql """ INSERT INTO ${tableName} VALUES (7, "a", 100); """ + sql """ INSERT INTO ${tableName} VALUES (8, "a", 100); """ + + // trigger master be to do compaction with delete + assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], + "full", tablet_id).contains("Success")); + waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) + + // trigger follower be to fetch compaction result + for (String id in follower_backend_id) { + assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); + waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) } -} + + // check rowsets + checkSucceedCompactionResult.call() + + qt_sql """ + select * from ${tableName} order by id + """ +} \ No newline at end of file diff --git a/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy b/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy index c31d7ab7c28..0e13662f3b4 100644 --- a/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy +++ b/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy @@ -19,116 +19,80 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_single_compaction_with_variant_inverted", "p2") { def tableName = "test_single_compaction_with_variant_inverted" - - def set_be_config = { key, value -> - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - for (String backend_id: backendId_to_backendIP.keySet()) { - def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) - logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) - } - } def calc_file_crc_on_tablet = { ip, port, tablet -> return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s";, ip, port, tablet)) } boolean disableAutoCompaction = true - boolean has_update_be_config = false - try { - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + backend_id = backendId_to_backendIP.keySet()[0] + def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) + + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + for (Object ele in (List) configList) { + assert ele instanceof List<String> + if (((List<String>) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) } - set_be_config.call("disable_auto_compaction", "true") - has_update_be_config = true - - def triggerCompaction = { be_host, be_http_port, compact_type, tablet_id -> - StringBuilder sb = new StringBuilder(); - sb.append("curl -X POST http://${be_host}:${be_http_port}";) - sb.append("/api/compaction/run?tablet_id=") - sb.append(tablet_id) - sb.append("&compact_type=${compact_type}") - - String command = sb.toString() - logger.info(command) - process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Run compaction: code=" + code + ", out=" + out + ", disableAutoCompaction " + disableAutoCompaction + ", err=" + err) - if (!disableAutoCompaction) { - return "Success, " + out - } - assertEquals(code, 0) - return out - } - - def triggerSingleCompaction = { be_host, be_http_port, tablet_id -> - StringBuilder sb = new StringBuilder(); - sb.append("curl -X POST http://${be_host}:${be_http_port}";) - sb.append("/api/compaction/run?tablet_id=") - sb.append(tablet_id) - sb.append("&compact_type=cumulative&remote=true") + } - String command = sb.toString() - logger.info(command) - process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Run compaction: code=" + code + ", out=" + out + ", disableAutoCompaction " + disableAutoCompaction + ", err=" + err) - if (!disableAutoCompaction) { - return "Success, " + out - } - assertEquals(code, 0) - return out + def triggerCompaction = { be_host, be_http_port, compact_type, tablet_id -> + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://${be_host}:${be_http_port}";) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=${compact_type}") + + String command = sb.toString() + logger.info(command) + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Run compaction: code=" + code + ", out=" + out + ", disableAutoCompaction " + disableAutoCompaction + ", err=" + err) + if (!disableAutoCompaction) { + return "Success, " + out } - def waitForCompaction = { be_host, be_http_port, tablet_id -> - boolean running = true - do { - Thread.sleep(1000) - StringBuilder sb = new StringBuilder(); - sb.append("curl -X GET http://${be_host}:${be_http_port}";) - sb.append("/api/compaction/run_status?tablet_id=") - sb.append(tablet_id) - - String command = sb.toString() - logger.info(command) - process = command.execute() - code = process.waitFor() - out = process.getText() - logger.info("Get compaction status: code=" + code + ", out=" + out) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) + assertEquals(code, 0) + return out + } + + def triggerSingleCompaction = { be_host, be_http_port, tablet_id -> + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://${be_host}:${be_http_port}";) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=cumulative&remote=true") + + String command = sb.toString() + logger.info(command) + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Run compaction: code=" + code + ", out=" + out + ", disableAutoCompaction " + disableAutoCompaction + ", err=" + err) + if (!disableAutoCompaction) { + return "Success, " + out } - - def getTabletStatus = { be_host, be_http_port, tablet_id -> - boolean running = true + assertEquals(code, 0) + return out + } + def waitForCompaction = { be_host, be_http_port, tablet_id -> + boolean running = true + do { Thread.sleep(1000) StringBuilder sb = new StringBuilder(); sb.append("curl -X GET http://${be_host}:${be_http_port}";) - sb.append("/api/compaction/show?tablet_id=") + sb.append("/api/compaction/run_status?tablet_id=") sb.append(tablet_id) String command = sb.toString() @@ -136,119 +100,138 @@ suite("test_single_compaction_with_variant_inverted", "p2") { process = command.execute() code = process.waitFor() out = process.getText() - logger.info("Get tablet status: code=" + code + ", out=" + out) + logger.info("Get compaction status: code=" + code + ", out=" + out) assertEquals(code, 0) - def tabletStatus = parseJson(out.trim()) - return tabletStatus - } + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + def getTabletStatus = { be_host, be_http_port, tablet_id -> + boolean running = true + Thread.sleep(1000) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}";) + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + process = command.execute() + code = process.waitFor() + out = process.getText() + logger.info("Get tablet status: code=" + code + ", out=" + out) + assertEquals(code, 0) + def tabletStatus = parseJson(out.trim()) + return tabletStatus + } - sql """ DROP TABLE IF EXISTS ${tableName}; """ - sql """ - CREATE TABLE ${tableName} ( - `id` int(11) NULL, - `name` varchar(255) NULL, - `score` int(11) NULL, - `properties` variant, - INDEX idx_props (`properties`) USING INVERTED PROPERTIES("parser" = "none") COMMENT '' - ) ENGINE=OLAP - DUPLICATE KEY(`id`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( "replication_num" = "2", "enable_single_replica_compaction" = "true", "inverted_index_storage_format" = "V1"); - """ - - def tablets = sql_return_maparray """ show tablets from ${tableName}; """ - - // wait for update replica infos - // be.conf: update_replica_infos_interval_seconds + 2s - Thread.sleep(62000) - - // find the master be for single replica compaction - Boolean found = false - String master_backend_id; - List<String> follower_backend_id = new ArrayList<>() - // The test table only has one bucket with 2 replicas, - // and `show tablets` will return 2 different replicas with the same tablet. - // So we can use the same tablet_id to get tablet/trigger compaction with different backends. - String tablet_id = tablets[0].TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - logger.info("tablet: " + tablet_info) - for (def tablet in tablets) { - String trigger_backend_id = tablet.BackendId - def tablet_status = getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); - if (!tablet_status.containsKey("single replica compaction status")) { - if (found) { - logger.warn("multipe master"); - assertTrue(false) - } - found = true - master_backend_id = trigger_backend_id - } else { - follower_backend_id.add(trigger_backend_id) + sql """ DROP TABLE IF EXISTS ${tableName}; """ + sql """ + CREATE TABLE ${tableName} ( + `id` int(11) NULL, + `name` varchar(255) NULL, + `score` int(11) NULL, + `properties` variant, + INDEX idx_props (`properties`) USING INVERTED PROPERTIES("parser" = "none") COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "2", + "enable_single_replica_compaction" = "true", + "inverted_index_storage_format" = "V1", + "compaction_policy" = "time_series" + ); + """ + + def tablets = sql_return_maparray """ show tablets from ${tableName}; """ + + // wait for update replica infos + Thread.sleep(72000) + + // find the master be for single replica compaction + Boolean found = false + String master_backend_id; + List<String> follower_backend_id = new ArrayList<>() + // The test table only has one bucket with 2 replicas, + // and `show tablets` will return 2 different replicas with the same tablet. + // So we can use the same tablet_id to get tablet/trigger compaction with different backends. + String tablet_id = tablets[0].TabletId + def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ + logger.info("tablet: " + tablet_info) + for (def tablet in tablets) { + String trigger_backend_id = tablet.BackendId + def tablet_status = getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); + if (!tablet_status.containsKey("single replica compaction status")) { + if (found) { + logger.warn("multipe master"); + assertTrue(false) } + found = true + master_backend_id = trigger_backend_id + } else { + follower_backend_id.add(trigger_backend_id) } + } - def checkCompactionResult = { - def master_tablet_status = getTabletStatus(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id); - def master_rowsets = master_tablet_status."rowsets" - assert master_rowsets instanceof List - logger.info("rowset size: " + master_rowsets.size()) - - for (String backend: follower_backend_id) { - def tablet_status = getTabletStatus(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id); - def rowsets = tablet_status."rowsets" - assert rowsets instanceof List - assertEquals(master_rowsets.size(), rowsets.size()) - } + def checkCompactionResult = { + def master_tablet_status = getTabletStatus(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id); + def master_rowsets = master_tablet_status."rowsets" + assert master_rowsets instanceof List + logger.info("rowset size: " + master_rowsets.size()) + + for (String backend: follower_backend_id) { + def tablet_status = getTabletStatus(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id); + def rowsets = tablet_status."rowsets" + assert rowsets instanceof List + assertEquals(master_rowsets.size(), rowsets.size()) } + } - def checkTabletFileCrc = { - def (master_code, master_out, master_err) = calc_file_crc_on_tablet(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) - logger.info("Run calc_file_crc_on_tablet: ip=" + backendId_to_backendIP[master_backend_id] + " code=" + master_code + ", out=" + master_out + ", err=" + master_err) - - for (String backend: follower_backend_id) { - def (follower_code, follower_out, follower_err) = calc_file_crc_on_tablet(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id) - logger.info("Run calc_file_crc_on_tablet: ip=" + backendId_to_backendIP[backend] + " code=" + follower_code + ", out=" + follower_out + ", err=" + follower_err) - assertTrue(parseJson(follower_out.trim()).crc_value == parseJson(master_out.trim()).crc_value) - assertTrue(parseJson(follower_out.trim()).start_version == parseJson(master_out.trim()).start_version) - assertTrue(parseJson(follower_out.trim()).end_version == parseJson(master_out.trim()).end_version) - assertTrue(parseJson(follower_out.trim()).file_count == parseJson(master_out.trim()).file_count) - assertTrue(parseJson(follower_out.trim()).rowset_count == parseJson(master_out.trim()).rowset_count) - } + def checkTabletFileCrc = { + def (master_code, master_out, master_err) = calc_file_crc_on_tablet(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) + logger.info("Run calc_file_crc_on_tablet: ip=" + backendId_to_backendIP[master_backend_id] + " code=" + master_code + ", out=" + master_out + ", err=" + master_err) + + for (String backend: follower_backend_id) { + def (follower_code, follower_out, follower_err) = calc_file_crc_on_tablet(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id) + logger.info("Run calc_file_crc_on_tablet: ip=" + backendId_to_backendIP[backend] + " code=" + follower_code + ", out=" + follower_out + ", err=" + follower_err) + assertTrue(parseJson(follower_out.trim()).crc_value == parseJson(master_out.trim()).crc_value) + assertTrue(parseJson(follower_out.trim()).start_version == parseJson(master_out.trim()).start_version) + assertTrue(parseJson(follower_out.trim()).end_version == parseJson(master_out.trim()).end_version) + assertTrue(parseJson(follower_out.trim()).file_count == parseJson(master_out.trim()).file_count) + assertTrue(parseJson(follower_out.trim()).rowset_count == parseJson(master_out.trim()).rowset_count) } + } - sql """ INSERT INTO ${tableName} VALUES (1, "a", 100, '{"a" : 1234, "point" : 1, "xxxx" : "ddddd"}'); """ - sql """ INSERT INTO ${tableName} VALUES (1, "b", 100, '{"%a" : 1234, "@point" : 1, "[xxxx" : "ddddd"}'); """ - sql """ INSERT INTO ${tableName} VALUES (2, "a", 100, '{"@a" : 1234, "%point" : 1, "]xxxx" : "ddddd"}'); """ - sql """ INSERT INTO ${tableName} VALUES (2, "b", 100, '{"%a" : 1234, "%point" : 1, "{xxxx" : "ddddd"}'); """ - sql """ INSERT INTO ${tableName} VALUES (3, "a", 100, '{"@a" : 1234, "@point" : 1, "}xxxx" : "ddddd"}'); """ - sql """ INSERT INTO ${tableName} VALUES (3, "b", 100, '{"a" : 1234, "point" : 1, "|xxxx" : "ddddd"}'); """ - - // trigger master be to do full compaction - assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], - "full", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) - - // trigger follower be to fetch compaction result - for (String id in follower_backend_id) { - assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) - } + sql """ INSERT INTO ${tableName} VALUES (1, "a", 100, '{"a" : 1234, "point" : 1, "xxxx" : "ddddd"}'); """ + sql """ INSERT INTO ${tableName} VALUES (1, "b", 100, '{"%a" : 1234, "@point" : 1, "[xxxx" : "ddddd"}'); """ + sql """ INSERT INTO ${tableName} VALUES (2, "a", 100, '{"@a" : 1234, "%point" : 1, "]xxxx" : "ddddd"}'); """ + sql """ INSERT INTO ${tableName} VALUES (2, "b", 100, '{"%a" : 1234, "%point" : 1, "{xxxx" : "ddddd"}'); """ + sql """ INSERT INTO ${tableName} VALUES (3, "a", 100, '{"@a" : 1234, "@point" : 1, "}xxxx" : "ddddd"}'); """ + sql """ INSERT INTO ${tableName} VALUES (3, "b", 100, '{"a" : 1234, "point" : 1, "|xxxx" : "ddddd"}'); """ + + // trigger master be to do full compaction + assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], + "full", tablet_id).contains("Success")); + waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) + + // trigger follower be to fetch compaction result + for (String id in follower_backend_id) { + assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); + waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) + } - // check rowsets - checkCompactionResult.call() - checkTabletFileCrc.call() + // check rowsets + checkCompactionResult.call() + checkTabletFileCrc.call() - qt_sql """ - select count() from ${tableName} where properties MATCH_ANY 'point xxxx'; - """ + qt_sql """ + select count() from ${tableName} where properties MATCH_ANY 'point xxxx'; + """ - sql """ DROP TABLE IF EXISTS ${tableName}; """ - - } finally { - if (has_update_be_config) { - set_be_config.call("disable_auto_compaction", disableAutoCompaction.toString()) - } - } + sql """ DROP TABLE IF EXISTS ${tableName}; """ } diff --git a/regression-test/suites/compaction/test_single_replica_compaction.groovy b/regression-test/suites/compaction/test_single_replica_compaction.groovy index 0b83ddd51e6..d8ff209b93c 100644 --- a/regression-test/suites/compaction/test_single_replica_compaction.groovy +++ b/regression-test/suites/compaction/test_single_replica_compaction.groovy @@ -17,128 +17,78 @@ import org.codehaus.groovy.runtime.IOGroovyMethods -suite("test_single_replica_compaction", "p2") { +suite("test_single_compaction_p2", "p2") { + if (isCloudMode()) { + return; + } def tableName = "test_single_replica_compaction" - def set_be_config = { key, value -> - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - for (String backend_id: backendId_to_backendIP.keySet()) { - def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) - logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) - } - } - def calc_file_crc_on_tablet = { ip, port, tablet -> return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s";, ip, port, tablet)) } - boolean disableAutoCompaction = true - boolean has_update_be_config = false - try { - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } - set_be_config.call("disable_auto_compaction", "true") - has_update_be_config = true - - def triggerCompaction = { be_host, be_http_port, compact_type, tablet_id -> - if (compact_type == "cumulative") { - def (code_1, out_1, err_1) = be_run_cumulative_compaction(be_host, be_http_port, tablet_id) - logger.info("Run compaction: code=" + code_1 + ", out=" + out_1 + ", err=" + err_1) - assertEquals(code_1, 0) - return out_1 - } else if (compact_type == "full") { - def (code_2, out_2, err_2) = be_run_full_compaction(be_host, be_http_port, tablet_id) - logger.info("Run compaction: code=" + code_2 + ", out=" + out_2 + ", err=" + err_2) - assertEquals(code_2, 0) - return out_2 - } else { - assertFalse(True) - } + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + def triggerCompaction = { be_host, be_http_port, compact_type, tablet_id -> + if (compact_type == "cumulative") { + def (code_1, out_1, err_1) = be_run_cumulative_compaction(be_host, be_http_port, tablet_id) + logger.info("Run compaction: code=" + code_1 + ", out=" + out_1 + ", err=" + err_1) + assertEquals(code_1, 0) + return out_1 + } else if (compact_type == "full") { + def (code_2, out_2, err_2) = be_run_full_compaction(be_host, be_http_port, tablet_id) + logger.info("Run compaction: code=" + code_2 + ", out=" + out_2 + ", err=" + err_2) + assertEquals(code_2, 0) + return out_2 + } else { + assertFalse(True) } + } - def triggerSingleCompaction = { be_host, be_http_port, tablet_id -> - StringBuilder sb = new StringBuilder(); - sb.append("curl -X POST http://${be_host}:${be_http_port}";) - sb.append("/api/compaction/run?tablet_id=") - sb.append(tablet_id) - sb.append("&compact_type=cumulative&remote=true") - - Integer maxRetries = 10; // Maximum number of retries - Integer retryCount = 0; // Current retry count - Integer sleepTime = 5000; // Sleep time in milliseconds - String cmd = sb.toString() - def process - int code_3 - String err_3 - String out_3 - - while (retryCount < maxRetries) { - process = cmd.execute() - code_3 = process.waitFor() - err_3 = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))) - out_3 = process.getText() - - // If the command was successful, break the loop - if (code_3 == 0) { - break - } - - // If the command was not successful, increment the retry count, sleep for a while and try again - retryCount++ - sleep(sleepTime) + def triggerSingleCompaction = { be_host, be_http_port, tablet_id -> + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://${be_host}:${be_http_port}";) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=cumulative&remote=true") + + Integer maxRetries = 10; // Maximum number of retries + Integer retryCount = 0; // Current retry count + Integer sleepTime = 5000; // Sleep time in milliseconds + String cmd = sb.toString() + def process + int code_3 + String err_3 + String out_3 + + while (retryCount < maxRetries) { + process = cmd.execute() + code_3 = process.waitFor() + err_3 = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))) + out_3 = process.getText() + + // If the command was successful, break the loop + if (code_3 == 0) { + break } - assertEquals(code_3, 0) - logger.info("Get compaction status: code=" + code_3 + ", out=" + out_3) - return out_3 - } - def waitForCompaction = { be_host, be_http_port, tablet_id -> - boolean running = true - do { - Thread.sleep(1000) - StringBuilder sb = new StringBuilder(); - sb.append("curl -X GET http://${be_host}:${be_http_port}";) - sb.append("/api/compaction/run_status?tablet_id=") - sb.append(tablet_id) - String command = sb.toString() - logger.info(command) - process = command.execute() - code = process.waitFor() - out = process.getText() - logger.info("Get compaction status: code=" + code + ", out=" + out) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) + // If the command was not successful, increment the retry count, sleep for a while and try again + retryCount++ + sleep(sleepTime) } - - def getTabletStatus = { be_host, be_http_port, tablet_id -> - boolean running = true + assertEquals(code_3, 0) + logger.info("Get compaction status: code=" + code_3 + ", out=" + out_3) + return out_3 + } + def waitForCompaction = { be_host, be_http_port, tablet_id -> + boolean running = true + do { Thread.sleep(1000) StringBuilder sb = new StringBuilder(); sb.append("curl -X GET http://${be_host}:${be_http_port}";) - sb.append("/api/compaction/show?tablet_id=") + sb.append("/api/compaction/run_status?tablet_id=") sb.append(tablet_id) String command = sb.toString() @@ -146,150 +96,156 @@ suite("test_single_replica_compaction", "p2") { process = command.execute() code = process.waitFor() out = process.getText() - logger.info("Get tablet status: code=" + code + ", out=" + out) + logger.info("Get compaction status: code=" + code + ", out=" + out) assertEquals(code, 0) - def tabletStatus = parseJson(out.trim()) - return tabletStatus - } - - - sql """ DROP TABLE IF EXISTS ${tableName}; """ - sql """ - CREATE TABLE ${tableName} ( - `id` int(11) NULL, - `name` varchar(255) NULL, - `score` int(11) NULL - ) ENGINE=OLAP - UNIQUE KEY(`id`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( "replication_num" = "2", "enable_single_replica_compaction" = "true", "enable_unique_key_merge_on_write" = "false" ); - """ - - def tablets = sql_return_maparray """ show tablets from ${tableName}; """ - - // wait for update replica infos - // be.conf: update_replica_infos_interval_seconds + 2s - Thread.sleep(62000) - - // find the master be for single replica compaction - Boolean found = false - String master_backend_id; - List<String> follower_backend_id = new ArrayList<>() - // The test table only has one bucket with 2 replicas, - // and `show tablets` will return 2 different replicas with the same tablet. - // So we can use the same tablet_id to get tablet/trigger compaction with different backends. - String tablet_id = tablets[0].TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - logger.info("tablet: " + tablet_info) - for (def tablet in tablets) { - String trigger_backend_id = tablet.BackendId - def tablet_status = getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); - if (!tablet_status.containsKey("single replica compaction status")) { - if (found) { - logger.warn("multipe master"); - assertTrue(false) - } - found = true - master_backend_id = trigger_backend_id - } else { - follower_backend_id.add(trigger_backend_id) - } - } - - def checkCompactionResult = { - def master_tablet_status = getTabletStatus(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id); - def master_rowsets = master_tablet_status."rowsets" - assert master_rowsets instanceof List - logger.info("rowset size: " + master_rowsets.size()) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } - for (String backend: follower_backend_id) { - def tablet_status = getTabletStatus(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id); - def rowsets = tablet_status."rowsets" - assert rowsets instanceof List - assertEquals(master_rowsets.size(), rowsets.size()) - } - } + def getTabletStatus = { be_host, be_http_port, tablet_id -> + boolean running = true + Thread.sleep(1000) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}";) + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + process = command.execute() + code = process.waitFor() + out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out) + assertEquals(code, 0) + def tabletStatus = parseJson(out.trim()) + return tabletStatus + } - def checkTabletFileCrc = { - def (master_code, master_out, master_err) = calc_file_crc_on_tablet(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) - logger.info("Run calc_file_crc_on_tablet: ip=" + backendId_to_backendIP[master_backend_id] + " code=" + master_code + ", out=" + master_out + ", err=" + master_err) - for (String backend: follower_backend_id) { - def (follower_code, follower_out, follower_err) = calc_file_crc_on_tablet(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id) - logger.info("Run calc_file_crc_on_tablet: ip=" + backendId_to_backendIP[backend] + " code=" + follower_code + ", out=" + follower_out + ", err=" + follower_err) - assertTrue(parseJson(follower_out.trim()).crc_value == parseJson(master_out.trim()).crc_value) - assertTrue(parseJson(follower_out.trim()).start_version == parseJson(master_out.trim()).start_version) - assertTrue(parseJson(follower_out.trim()).end_version == parseJson(master_out.trim()).end_version) - assertTrue(parseJson(follower_out.trim()).file_count == parseJson(master_out.trim()).file_count) - assertTrue(parseJson(follower_out.trim()).rowset_count == parseJson(master_out.trim()).rowset_count) + sql """ DROP TABLE IF EXISTS ${tableName}; """ + sql """ + CREATE TABLE ${tableName} ( + `id` int(11) NULL, + `name` varchar(255) NULL, + `score` int(11) NULL + ) ENGINE=OLAP + UNIQUE KEY(`id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "2", + "enable_single_replica_compaction" = "true", + "enable_unique_key_merge_on_write" = "false", + "compaction_policy" = "time_series" + ); + """ + + def tablets = sql_return_maparray """ show tablets from ${tableName}; """ + + // wait for update replica infos + Thread.sleep(70000) + + // find the master be for single replica compaction + Boolean found = false + String master_backend_id; + List<String> follower_backend_id = new ArrayList<>() + // The test table only has one bucket with 2 replicas, + // and `show tablets` will return 2 different replicas with the same tablet. + // So we can use the same tablet_id to get tablet/trigger compaction with different backends. + String tablet_id = tablets[0].TabletId + def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ + logger.info("tablet: " + tablet_info) + for (def tablet in tablets) { + String trigger_backend_id = tablet.BackendId + def tablet_status = getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); + if (!tablet_status.containsKey("single replica compaction status")) { + if (found) { + logger.warn("multipe master"); + assertTrue(false) } + found = true + master_backend_id = trigger_backend_id + } else { + follower_backend_id.add(trigger_backend_id) } + } - sql """ INSERT INTO ${tableName} VALUES (1, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (1, "b", 100); """ - sql """ INSERT INTO ${tableName} VALUES (2, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (2, "b", 100); """ - sql """ INSERT INTO ${tableName} VALUES (3, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (3, "b", 100); """ - - // trigger master be to do cumu compaction - assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], - "cumulative", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) - - // trigger follower be to fetch compaction result - for (String id in follower_backend_id) { - assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) + def checkCompactionResult = { + def master_tablet_status = getTabletStatus(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id); + def master_rowsets = master_tablet_status."rowsets" + assert master_rowsets instanceof List + logger.info("rowset size: " + master_rowsets.size()) + + for (String backend: follower_backend_id) { + def tablet_status = getTabletStatus(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id); + def rowsets = tablet_status."rowsets" + assert rowsets instanceof List + assertEquals(master_rowsets.size(), rowsets.size()) } + } - // check rowsets - checkCompactionResult.call() - - sql """ INSERT INTO ${tableName} VALUES (4, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (5, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (6, "a", 100); """ - sql """ DELETE FROM ${tableName} WHERE id = 4; """ - sql """ INSERT INTO ${tableName} VALUES (7, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (8, "a", 100); """ - - // trigger master be to do cumu compaction with delete - assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], - "cumulative", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) - - // trigger follower be to fetch compaction result - for (String id in follower_backend_id) { - assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) + def checkTabletFileCrc = { + def (master_code, master_out, master_err) = calc_file_crc_on_tablet(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) + logger.info("Run calc_file_crc_on_tablet: ip=" + backendId_to_backendIP[master_backend_id] + " code=" + master_code + ", out=" + master_out + ", err=" + master_err) + + for (String backend: follower_backend_id) { + def (follower_code, follower_out, follower_err) = calc_file_crc_on_tablet(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id) + logger.info("Run calc_file_crc_on_tablet: ip=" + backendId_to_backendIP[backend] + " code=" + follower_code + ", out=" + follower_out + ", err=" + follower_err) + assertTrue(parseJson(follower_out.trim()).crc_value == parseJson(master_out.trim()).crc_value) + assertTrue(parseJson(follower_out.trim()).start_version == parseJson(master_out.trim()).start_version) + assertTrue(parseJson(follower_out.trim()).end_version == parseJson(master_out.trim()).end_version) + assertTrue(parseJson(follower_out.trim()).file_count == parseJson(master_out.trim()).file_count) + assertTrue(parseJson(follower_out.trim()).rowset_count == parseJson(master_out.trim()).rowset_count) } + } - // check rowsets - checkCompactionResult.call() + sql """ INSERT INTO ${tableName} VALUES (1, "a", 100); """ + sql """ INSERT INTO ${tableName} VALUES (1, "b", 100); """ + sql """ INSERT INTO ${tableName} VALUES (2, "a", 100); """ + sql """ INSERT INTO ${tableName} VALUES (2, "b", 100); """ + sql """ INSERT INTO ${tableName} VALUES (3, "a", 100); """ + sql """ INSERT INTO ${tableName} VALUES (3, "b", 100); """ + + // trigger master be to do compaction + assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], + "full", tablet_id).contains("Success")); + waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) + + // trigger follower be to fetch compaction result + for (String id in follower_backend_id) { + assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); + waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) + } - // trigger master be to do full compaction - assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], - "full", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) + // check rowsets + checkCompactionResult.call() + + sql """ INSERT INTO ${tableName} VALUES (4, "a", 100); """ + sql """ INSERT INTO ${tableName} VALUES (5, "a", 100); """ + sql """ INSERT INTO ${tableName} VALUES (6, "a", 100); """ + sql """ DELETE FROM ${tableName} WHERE id = 4; """ + sql """ INSERT INTO ${tableName} VALUES (7, "a", 100); """ + sql """ INSERT INTO ${tableName} VALUES (8, "a", 100); """ + + // trigger master be to do compaction with delete + assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], + "full", tablet_id).contains("Success")); + waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) + + // trigger follower be to fetch compaction result + for (String id in follower_backend_id) { + assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); + waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) + } - // trigger follower be to fetch compaction result - for (String id in follower_backend_id) { - assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) - } + // check rowsets + checkCompactionResult.call() + checkTabletFileCrc.call() - // check rowsets - checkCompactionResult.call() - checkTabletFileCrc.call() + qt_sql """ + select * from ${tableName} order by id + """ - qt_sql """ - select * from ${tableName} order by id - """ - - } finally { - if (has_update_be_config) { - set_be_config.call("disable_auto_compaction", disableAutoCompaction.toString()) - } - } } diff --git a/regression-test/suites/compaction/test_time_series_compaction_policy.groovy b/regression-test/suites/compaction/test_time_series_compaction_policy.groovy index 5c37ff45cb1..2e8018f94a6 100644 --- a/regression-test/suites/compaction/test_time_series_compaction_policy.groovy +++ b/regression-test/suites/compaction/test_time_series_compaction_policy.groovy @@ -22,7 +22,7 @@ suite("test_time_series_compaction_polciy", "p0") { def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - + def trigger_cumulative_compaction_on_tablets = { tablets -> for (def tablet : tablets) { String tablet_id = tablet.TabletId @@ -86,7 +86,11 @@ suite("test_time_series_compaction_polciy", "p0") { DUPLICATE KEY(`id`) COMMENT 'OLAP' DISTRIBUTED BY HASH(`id`) BUCKETS 2 - PROPERTIES ( "replication_num" = "1", "disable_auto_compaction" = "true", "compaction_policy" = "time_series"); + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "compaction_policy" = "time_series" + ); """ // insert 16 lines, BUCKETS = 2 sql """ INSERT INTO ${tableName} VALUES (1, "andy", "andy love apple", 100); """ @@ -149,6 +153,9 @@ suite("test_time_series_compaction_polciy", "p0") { assert (rowsetCount == 22 * replicaNum) qt_sql_2 """ select count() from ${tableName}""" + if (isCloudMode()) { + return; + } sql """ alter table ${tableName} set ("time_series_compaction_file_count_threshold"="10")""" sql """sync""" // trigger cumulative compactions for all tablets in ${tableName} diff --git a/regression-test/suites/fault_injection_p0/test_segcompaction_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_segcompaction_fault_injection.groovy index ceac616a8f1..b37ac4797df 100644 --- a/regression-test/suites/fault_injection_p0/test_segcompaction_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_segcompaction_fault_injection.groovy @@ -43,26 +43,8 @@ suite("test_segcompaction_correctness", "nonConcurrent,p2") { String endpoint = getS3Endpoint() String region = getS3Region() String bucket = getS3BucketName() - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - String backend_id; try { - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } sql """ DROP TABLE IF EXISTS ${tableName} """ sql "${create_table_sql}" diff --git a/regression-test/suites/fault_injection_p0/test_too_many_segments_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_too_many_segments_fault_injection.groovy index 9c55d2938ba..6556c792a67 100644 --- a/regression-test/suites/fault_injection_p0/test_too_many_segments_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_too_many_segments_fault_injection.groovy @@ -44,26 +44,7 @@ suite("test_too_many_segments", "nonConcurrent,p2") { // the epic -238 case String endpoint = getS3Endpoint() String region = getS3Region() String bucket = getS3BucketName() - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - String backend_id; try { - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } sql """ DROP TABLE IF EXISTS ${tableName} """ sql "${create_table_sql}" diff --git a/regression-test/suites/schema_change/test_number_overflow.groovy b/regression-test/suites/schema_change/test_number_overflow.groovy index e69bb4e6ce8..d13e797c1e3 100644 --- a/regression-test/suites/schema_change/test_number_overflow.groovy +++ b/regression-test/suites/schema_change/test_number_overflow.groovy @@ -25,26 +25,6 @@ suite ("test_number_overflow") { try { - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } - sql """ DROP TABLE IF EXISTS test_number_overflow """ sql """ CREATE TABLE IF NOT EXISTS test_number_overflow ( k1 INT NOT NULL, k2 VARCHAR(4096) NOT NULL, k3 VARCHAR(4096) NOT NULL, k4 VARCHAR(4096) NOT NULL, k5 VARCHAR(4096) NOT NULL, k6 VARCHAR(4096) NOT NULL, k7 VARCHAR(4096) NOT NULL, k8 VARCHAR(4096) NOT NULL, k9 VARCHAR(4096) NOT NULL, v1 FLOAT SUM NOT NULL, v2 DECIMAL(20,7) SUM NOT NULL ) AGGREGATE KEY(k1,k2,k3,k4,k5,k6,k7,k8,k9) PARTITION BY RANGE(k1) ( PARTITION partition_a VALUES LESS THAN ("5"), PARTITION partition_b VAL [...] diff --git a/regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy b/regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy index afc5b784fbe..93e9443accd 100644 --- a/regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy @@ -27,27 +27,10 @@ suite ("test_agg_keys_schema_change") { try { - String backend_id; def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } - sql """ DROP TABLE IF EXISTS schema_change_agg_keys_regression_test """ sql """ CREATE TABLE IF NOT EXISTS schema_change_agg_keys_regression_test ( diff --git a/regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy index 59ebfdd35cd..f7af7226982 100644 --- a/regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy @@ -44,27 +44,10 @@ suite ("test_agg_mv_schema_change") { def tableName = "schema_change_agg_mv_regression_test" try { - String backend_id; def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } - sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ CREATE TABLE IF NOT EXISTS ${tableName} ( diff --git a/regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy b/regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy index b567c8bc03c..78578c7b522 100644 --- a/regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy @@ -44,27 +44,10 @@ suite ("test_agg_rollup_schema_change") { } try { - String backend_id; def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } - sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ CREATE TABLE IF NOT EXISTS ${tableName} ( diff --git a/regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy b/regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy index ebe882a543d..165fba94b9a 100644 --- a/regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy @@ -22,163 +22,148 @@ suite ("test_agg_vals_schema_change") { try { - String backend_id; def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } - - sql """ DROP TABLE IF EXISTS ${tableName} """ - - sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( - `user_id` LARGEINT NOT NULL COMMENT "用户id", - `date` DATE NOT NULL COMMENT "数据灌入日期时间", - `city` VARCHAR(20) COMMENT "用户所在城市", - `age` SMALLINT COMMENT "用户年龄", - `sex` TINYINT COMMENT "用户性别", - `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", - `last_update_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", - `last_visit_date_not_null` DATETIME REPLACE NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", - `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", - `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", - `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间", - `hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列", - `bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列") - AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - BUCKETS 8 - PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" ); - """ - - sql """ INSERT INTO ${tableName} VALUES - (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20, hll_hash(1), to_bitmap(1)) - """ - - sql """ INSERT INTO ${tableName} VALUES - (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19, hll_hash(2), to_bitmap(2)) - """ - - qt_sc """ - select * from ${tableName} order by user_id - """ - - // alter and test light schema change - sql """ALTER TABLE ${tableName} SET ("light_schema_change" = "true");""" - - sql """ INSERT INTO ${tableName} VALUES - (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21, hll_hash(2), to_bitmap(2)) - """ - - sql """ INSERT INTO ${tableName} VALUES - (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(3), to_bitmap(3)) - """ - qt_sc """ - select * from ${tableName} order by user_id + sql """ DROP TABLE IF EXISTS ${tableName} """ + + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME REPLACE NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间", + `hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列", + `bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列") + AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + BUCKETS 8 + PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20, hll_hash(1), to_bitmap(1)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19, hll_hash(2), to_bitmap(2)) + """ + + qt_sc """ + select * from ${tableName} order by user_id """ - // add column - sql """ - ALTER table ${tableName} ADD COLUMN new_column INT MAX default "1" - """ - - qt_sc """ SELECT * FROM ${tableName} WHERE user_id=2 """ - - sql """ INSERT INTO ${tableName} VALUES - (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(4), to_bitmap(4), 2) - """ - qt_sc """ SELECT * FROM ${tableName} WHERE user_id=2 """ - - - sql """ INSERT INTO ${tableName} (`user_id`,`date`,`city`,`age`,`sex`,`last_visit_date`,`last_update_date`, - `last_visit_date_not_null`,`cost`,`max_dwell_time`,`min_dwell_time`, `hll_col`, `bitmap_col`) - VALUES - (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(4), to_bitmap(4)) - """ - - qt_sc """ SELECT * FROM ${tableName} WHERE user_id=3 """ - - sql """ INSERT INTO ${tableName} VALUES - (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(4), to_bitmap(4), 2) - """ - qt_sc """ SELECT * FROM ${tableName} WHERE user_id = 3 """ - - qt_sc """ select count(*) from ${tableName} """ - - // drop column - sql """ - ALTER TABLE ${tableName} DROP COLUMN last_visit_date - """ - qt_sc """ select * from ${tableName} where user_id = 3 """ - - sql """ INSERT INTO ${tableName} VALUES - (4, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(4), to_bitmap(4), 2) - """ - - qt_sc """ select * from ${tableName} where user_id = 4 """ - - sql """ INSERT INTO ${tableName} VALUES - (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) - """ - sql """ INSERT INTO ${tableName} VALUES - (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) - """ - sql """ INSERT INTO ${tableName} VALUES - (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) - """ - sql """ INSERT INTO ${tableName} VALUES - (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) - """ - sql """ INSERT INTO ${tableName} VALUES - (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) - """ - sql """ INSERT INTO ${tableName} VALUES - (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) - """ - - // compaction - String[][] tablets = sql """ show tablets from ${tableName}; """ - for (String[] tablet in tablets) { - String tablet_id = tablet[0] - backend_id = tablet[2] - logger.info("run compaction:" + tablet_id) - (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - //assertEquals(code, 0) - } + // alter and test light schema change + if (!isCloudMode()) { + sql """ALTER TABLE ${tableName} SET ("light_schema_change" = "true");""" + } - // wait for all compactions done - for (String[] tablet in tablets) { - boolean running = true - do { - Thread.sleep(100) + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21, hll_hash(2), to_bitmap(2)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(3), to_bitmap(3)) + """ + qt_sc """ + select * from ${tableName} order by user_id + """ + + // add column + sql """ + ALTER table ${tableName} ADD COLUMN new_column INT MAX default "1" + """ + + qt_sc """ SELECT * FROM ${tableName} WHERE user_id=2 """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(4), to_bitmap(4), 2) + """ + qt_sc """ SELECT * FROM ${tableName} WHERE user_id=2 """ + + + sql """ INSERT INTO ${tableName} (`user_id`,`date`,`city`,`age`,`sex`,`last_visit_date`,`last_update_date`, + `last_visit_date_not_null`,`cost`,`max_dwell_time`,`min_dwell_time`, `hll_col`, `bitmap_col`) + VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(4), to_bitmap(4)) + """ + + qt_sc """ SELECT * FROM ${tableName} WHERE user_id=3 """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(4), to_bitmap(4), 2) + """ + qt_sc """ SELECT * FROM ${tableName} WHERE user_id = 3 """ + + qt_sc """ select count(*) from ${tableName} """ + + // drop column + sql """ + ALTER TABLE ${tableName} DROP COLUMN last_visit_date + """ + qt_sc """ select * from ${tableName} where user_id = 3 """ + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(4), to_bitmap(4), 2) + """ + + qt_sc """ select * from ${tableName} where user_id = 4 """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) + """ + + // compaction + String[][] tablets = sql """ show tablets from ${tableName}; """ + for (String[] tablet in tablets) { String tablet_id = tablet[0] backend_id = tablet[2] - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - qt_sc """ select count(*) from ${tableName} """ + logger.info("run compaction:" + tablet_id) + (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + //assertEquals(code, 0) + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(100) + String tablet_id = tablet[0] + backend_id = tablet[2] + (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + qt_sc """ select count(*) from ${tableName} """ - qt_sc """ SELECT * FROM ${tableName} WHERE user_id=2 """ + qt_sc """ SELECT * FROM ${tableName} WHERE user_id=2 """ } finally { //try_sql("DROP TABLE IF EXISTS ${tableName}") diff --git a/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy index dd2fa43f095..52814b312f2 100644 --- a/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy @@ -25,27 +25,10 @@ suite ("test_dup_keys_schema_change") { } try { - String backend_id; def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } - sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ diff --git a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy index 168c450ebe1..6ce9812e130 100644 --- a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy @@ -44,22 +44,6 @@ suite ("test_dup_mv_schema_change") { def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } - sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ diff --git a/regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy index 1f1e0c5b61a..ad29f7d2277 100644 --- a/regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy @@ -49,21 +49,6 @@ suite ("test_dup_rollup_schema_change") { def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ diff --git a/regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy index 6a3d9f61c9e..ca7c0f598d8 100644 --- a/regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy @@ -26,21 +26,6 @@ suite ("test_dup_vals_schema_change") { def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ diff --git a/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy index d3eaec8cfa3..e06c27b8abf 100644 --- a/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy @@ -25,143 +25,129 @@ suite ("test_uniq_keys_schema_change") { def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } + sql """ DROP TABLE IF EXISTS ${tableName} """ + + sql """ + CREATE TABLE IF NOT EXISTS schema_change_uniq_keys_regression_test ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + BUCKETS 8 + PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false"); + """ + + sql """ INSERT INTO schema_change_uniq_keys_regression_test VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20) + """ + + sql """ INSERT INTO schema_change_uniq_keys_regression_test VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19) + """ + + qt_sc """ select count(*) from schema_change_uniq_keys_regression_test """ + + // alter and test light schema change + if (!isCloudMode()) { + sql """ALTER TABLE ${tableName} SET ("light_schema_change" = "true");""" } - sql """ DROP TABLE IF EXISTS ${tableName} """ - - sql """ - CREATE TABLE IF NOT EXISTS schema_change_uniq_keys_regression_test ( - `user_id` LARGEINT NOT NULL COMMENT "用户id", - `date` DATE NOT NULL COMMENT "数据灌入日期时间", - `city` VARCHAR(20) COMMENT "用户所在城市", - `age` SMALLINT COMMENT "用户年龄", - `sex` TINYINT COMMENT "用户性别", - `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", - `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", - `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", - `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", - `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", - `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") - UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - BUCKETS 8 - PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false"); - """ - - sql """ INSERT INTO schema_change_uniq_keys_regression_test VALUES - (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20) - """ - - sql """ INSERT INTO schema_change_uniq_keys_regression_test VALUES - (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19) - """ - - qt_sc """ select count(*) from schema_change_uniq_keys_regression_test """ - - // alter and test light schema change - sql """ALTER TABLE ${tableName} SET ("light_schema_change" = "true");""" - - sql """ INSERT INTO schema_change_uniq_keys_regression_test VALUES - (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21) - """ - - sql """ INSERT INTO schema_change_uniq_keys_regression_test VALUES - (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) - """ - qt_sc """ - select count(*) from schema_change_uniq_keys_regression_test - """ - - // add column - sql """ - ALTER table ${tableName} ADD COLUMN new_column INT default "1" - """ - - sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ - - sql """ INSERT INTO ${tableName} (`user_id`,`date`,`city`,`age`,`sex`,`last_visit_date`,`last_update_date`, - `last_visit_date_not_null`,`cost`,`max_dwell_time`,`min_dwell_time`) - VALUES - (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) - """ - - qt_sc """ SELECT * FROM ${tableName} WHERE user_id=3 """ - - - sql """ INSERT INTO ${tableName} VALUES - (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) - """ - qt_sc """ SELECT * FROM ${tableName} WHERE user_id = 3 """ - - qt_sc """ select count(*) from ${tableName} """ - - sql """ INSERT INTO ${tableName} VALUES - (4, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) - """ - - qt_sc """ select * from ${tableName} where user_id = 4 """ - - sql """ INSERT INTO ${tableName} VALUES - (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) - """ - sql """ INSERT INTO ${tableName} VALUES - (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) - """ - sql """ INSERT INTO ${tableName} VALUES - (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) - """ - sql """ INSERT INTO ${tableName} VALUES - (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) - """ - sql """ INSERT INTO ${tableName} VALUES - (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) - """ - sql """ INSERT INTO ${tableName} VALUES - (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) - """ - - // compaction - String[][] tablets = sql """ show tablets from ${tableName}; """ - for (String[] tablet in tablets) { - String tablet_id = tablet[0] - backend_id = tablet[2] - logger.info("run compaction:" + tablet_id) - (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - //assertEquals(code, 0) - } - // wait for all compactions done - for (String[] tablet in tablets) { - boolean running = true - do { - Thread.sleep(100) + sql """ INSERT INTO schema_change_uniq_keys_regression_test VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21) + """ + + sql """ INSERT INTO schema_change_uniq_keys_regression_test VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + qt_sc """ + select count(*) from schema_change_uniq_keys_regression_test + """ + + // add column + sql """ + ALTER table ${tableName} ADD COLUMN new_column INT default "1" + """ + + sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + + sql """ INSERT INTO ${tableName} (`user_id`,`date`,`city`,`age`,`sex`,`last_visit_date`,`last_update_date`, + `last_visit_date_not_null`,`cost`,`max_dwell_time`,`min_dwell_time`) + VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + + qt_sc """ SELECT * FROM ${tableName} WHERE user_id=3 """ + + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + qt_sc """ SELECT * FROM ${tableName} WHERE user_id = 3 """ + + qt_sc """ select count(*) from ${tableName} """ + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + + qt_sc """ select * from ${tableName} where user_id = 4 """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + + // compaction + String[][] tablets = sql """ show tablets from ${tableName}; """ + for (String[] tablet in tablets) { String tablet_id = tablet[0] backend_id = tablet[2] - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - qt_sc """ select count(*) from ${tableName} """ + logger.info("run compaction:" + tablet_id) + (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + //assertEquals(code, 0) + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(100) + String tablet_id = tablet[0] + backend_id = tablet[2] + (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + qt_sc """ select count(*) from ${tableName} """ - qt_sc """ SELECT * FROM ${tableName} WHERE user_id=2 """ + qt_sc """ SELECT * FROM ${tableName} WHERE user_id=2 """ } finally { //try_sql("DROP TABLE IF EXISTS ${tableName}") diff --git a/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy index f56c49a2d45..6167b757d09 100644 --- a/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy @@ -44,21 +44,6 @@ suite ("test_uniq_mv_schema_change") { def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ diff --git a/regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy index c2d8e366d48..515d1ee9151 100644 --- a/regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy @@ -48,21 +48,6 @@ suite ("test_uniq_rollup_schema_change") { def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ diff --git a/regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy index 43c08dc7300..78d97b539c3 100644 --- a/regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy @@ -28,21 +28,6 @@ suite ("test_uniq_vals_schema_change") { def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ diff --git a/regression-test/suites/schema_change_p0/test_varchar_schema_change.groovy b/regression-test/suites/schema_change_p0/test_varchar_schema_change.groovy index 8f3f0d3fcec..38bd996e89d 100644 --- a/regression-test/suites/schema_change_p0/test_varchar_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_varchar_schema_change.groovy @@ -32,22 +32,6 @@ suite ("test_varchar_schema_change") { def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } - sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ CREATE TABLE IF NOT EXISTS ${tableName} ( diff --git a/regression-test/suites/segcompaction_p2/test_segcompaction_agg_keys.groovy b/regression-test/suites/segcompaction_p2/test_segcompaction_agg_keys.groovy index 409bfd89c3e..480c5df7332 100644 --- a/regression-test/suites/segcompaction_p2/test_segcompaction_agg_keys.groovy +++ b/regression-test/suites/segcompaction_p2/test_segcompaction_agg_keys.groovy @@ -27,25 +27,6 @@ suite("test_segcompaction_agg_keys") { try { - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ diff --git a/regression-test/suites/segcompaction_p2/test_segcompaction_agg_keys_index.groovy b/regression-test/suites/segcompaction_p2/test_segcompaction_agg_keys_index.groovy index 596f70ca683..d7975508975 100644 --- a/regression-test/suites/segcompaction_p2/test_segcompaction_agg_keys_index.groovy +++ b/regression-test/suites/segcompaction_p2/test_segcompaction_agg_keys_index.groovy @@ -27,24 +27,6 @@ suite("test_segcompaction_agg_keys_index") { try { - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ diff --git a/regression-test/suites/segcompaction_p2/test_segcompaction_dup_keys.groovy b/regression-test/suites/segcompaction_p2/test_segcompaction_dup_keys.groovy index 6dcea441f01..857d5bba863 100644 --- a/regression-test/suites/segcompaction_p2/test_segcompaction_dup_keys.groovy +++ b/regression-test/suites/segcompaction_p2/test_segcompaction_dup_keys.groovy @@ -27,26 +27,6 @@ suite("test_segcompaction_dup_keys") { try { - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ diff --git a/regression-test/suites/segcompaction_p2/test_segcompaction_dup_keys_index.groovy b/regression-test/suites/segcompaction_p2/test_segcompaction_dup_keys_index.groovy index cc042c40f5b..50ae8f2ca48 100644 --- a/regression-test/suites/segcompaction_p2/test_segcompaction_dup_keys_index.groovy +++ b/regression-test/suites/segcompaction_p2/test_segcompaction_dup_keys_index.groovy @@ -27,25 +27,6 @@ suite("test_segcompaction_dup_keys_index") { try { - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ diff --git a/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys.groovy b/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys.groovy index 5e31abfd22b..6f98694a31e 100644 --- a/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys.groovy +++ b/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys.groovy @@ -27,26 +27,6 @@ suite("test_segcompaction_unique_keys") { try { - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ diff --git a/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy b/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy index eb39c48e0b7..1602e074a50 100644 --- a/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy +++ b/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy @@ -27,25 +27,6 @@ suite("test_segcompaction_unique_keys_index") { try { - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ diff --git a/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_mow.groovy b/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_mow.groovy index d1a7e774066..31111d3939e 100644 --- a/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_mow.groovy +++ b/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_mow.groovy @@ -27,26 +27,6 @@ suite("test_segcompaction_unique_keys_mow") { try { - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ diff --git a/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_mow_index.groovy b/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_mow_index.groovy index 628ac6dffd9..c9fa404cb96 100644 --- a/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_mow_index.groovy +++ b/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_mow_index.groovy @@ -27,26 +27,6 @@ suite("test_segcompaction_unique_keys_mow_index") { try { - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List<String> - if (((List<String>) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2]) - } - } sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org