w41ter commented on code in PR #27033: URL: https://github.com/apache/doris/pull/27033#discussion_r1395075436
########## regression-test/suites/backup_restore/test_backup_restore_reserve_dynamic_partition_true.groovy: ########## @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_backup_restore_dynamic_partition_reserve_true", "backup_restore") { + String repoName = "test_backup_restore_dynamic_partition_reserve_true_repo" + String dbName = "backup_restore_dynamic_partition_reserve_true_db" + String tableName = "dynamic_partition_reserve_true_table" + + def syncer = getSyncer() + syncer.createS3Repository(repoName) + + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + CREATE TABLE ${dbName}.${tableName} ( + `id` LARGEINT NOT NULL, + `sdate` DATE + ) + PARTITION BY RANGE(sdate)() + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES + ( + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "YEAR", + "dynamic_partition.start" = "-50", + "dynamic_partition.end" = "5", + "dynamic_partition.prefix" = "p", + "dynamic_partition.create_history_partition"="true", + "replication_num" = "1" + ) + """ + + sql """ + INSERT INTO ${dbName}.${tableName} VALUES + (1,"2023-11-01"), + (2,"2023-11-01"), + (3,"2023-11-01"), + (4,"2023-11-01"), + (5,"2023-11-01"), + (1,"2023-11-02"), + (2,"2023-11-02"), + (3,"2023-11-02"), + (4,"2023-11-02"), + (5,"2023-11-02"), + (1,"2024-11-01"), + (2,"2024-11-01"), + (3,"2024-11-01"), + (4,"2024-11-01"), + (5,"2024-11-01"), + (1,"2024-11-02"), + (2,"2024-11-02"), + (3,"2024-11-02"), + (4,"2024-11-02"), + (5,"2024-11-02") + """ + + def result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(),20) + + String snapshotName = "test_backup_restore_dynamic_partition_reserve_true_snapshot" + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + ON (${tableName}) + """ + + while (!syncer.checkSnapshotFinish(dbName)) { + Thread.sleep(3000) + } + def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + sql "TRUNCATE TABLE ${dbName}.${tableName}" Review Comment: ```suggestion sql "DROP TABLE ${dbName}.${tableName} FORCE" ``` ########## regression-test/suites/backup_restore/test_backup_restore_reserve_dynamic_partition_true.groovy: ########## @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_backup_restore_dynamic_partition_reserve_true", "backup_restore") { + String repoName = "test_backup_restore_dynamic_partition_reserve_true_repo" + String dbName = "backup_restore_dynamic_partition_reserve_true_db" + String tableName = "dynamic_partition_reserve_true_table" + + def syncer = getSyncer() + syncer.createS3Repository(repoName) + + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + CREATE TABLE ${dbName}.${tableName} ( + `id` LARGEINT NOT NULL, + `sdate` DATE + ) + PARTITION BY RANGE(sdate)() + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES + ( + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "YEAR", + "dynamic_partition.start" = "-50", + "dynamic_partition.end" = "5", + "dynamic_partition.prefix" = "p", + "dynamic_partition.create_history_partition"="true", + "replication_num" = "1" + ) + """ + + sql """ + INSERT INTO ${dbName}.${tableName} VALUES + (1,"2023-11-01"), + (2,"2023-11-01"), + (3,"2023-11-01"), + (4,"2023-11-01"), + (5,"2023-11-01"), + (1,"2023-11-02"), + (2,"2023-11-02"), + (3,"2023-11-02"), + (4,"2023-11-02"), + (5,"2023-11-02"), + (1,"2024-11-01"), + (2,"2024-11-01"), + (3,"2024-11-01"), + (4,"2024-11-01"), + (5,"2024-11-01"), + (1,"2024-11-02"), + (2,"2024-11-02"), + (3,"2024-11-02"), + (4,"2024-11-02"), + (5,"2024-11-02") + """ + + def result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(),20) + + String snapshotName = "test_backup_restore_dynamic_partition_reserve_true_snapshot" + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + ON (${tableName}) + """ + + while (!syncer.checkSnapshotFinish(dbName)) { + Thread.sleep(3000) + } + def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + sql "TRUNCATE TABLE ${dbName}.${tableName}" + + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON (`${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_dynamic_partition_enable" = "true", + "replication_num" = "1" + ) + """ + + // def restore_properties = sql "SHOW CREATE TABLE ${dbName}.${tableName}" + + // assertTrue(restore_properties.indexOf("\"dynamic_partition.enable\" = \"true\"") != -1) Review Comment: The result of `SHOW CREATE TABLE` contains a row with two columns, like: ``` | Table | Create Table | dynamic_partition_reserve_true_table | CREATE TABLE XXXXX ``` To access the first row and second column, use `restore_properties[0][1]` ########## regression-test/suites/backup_restore/test_backup_restore_reserve_dynamic_partition_false.groovy: ########## @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_backup_restore_reserve_dynamic_partition_false", "backup_restore") { + String repoName = "test_backup_restore_dynamic_partition_reserve_false_repo" + String dbName = "backup_restore_dynamic_partition_reserve_false_db" + String tableName = "dynamic_partition_reserve_false_table" + + def syncer = getSyncer() + syncer.createS3Repository(repoName) + + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + CREATE TABLE ${dbName}.${tableName} ( + `id` LARGEINT NOT NULL, + `sdate` DATE + ) + PARTITION BY RANGE(sdate)() + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES + ( + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "YEAR", + "dynamic_partition.start" = "-50", + "dynamic_partition.end" = "5", + "dynamic_partition.prefix" = "p", + "dynamic_partition.create_history_partition"="true", + "replication_num" = "1" + ) + """ + + sql """ + INSERT INTO ${dbName}.${tableName} VALUES + (1,"2023-11-01"), + (2,"2023-11-01"), + (3,"2023-11-01"), + (4,"2023-11-01"), + (5,"2023-11-01"), + (1,"2023-11-02"), + (2,"2023-11-02"), + (3,"2023-11-02"), + (4,"2023-11-02"), + (5,"2023-11-02"), + (1,"2024-11-01"), + (2,"2024-11-01"), + (3,"2024-11-01"), + (4,"2024-11-01"), + (5,"2024-11-01"), + (1,"2024-11-02"), + (2,"2024-11-02"), + (3,"2024-11-02"), + (4,"2024-11-02"), + (5,"2024-11-02") + """ + + def result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(),20) + + String snapshotName = "test_backup_restore_dynamic_partition_reserve_true_snapshot" + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + ON (${tableName}) + """ + + while (!syncer.checkSnapshotFinish(dbName)) { + Thread.sleep(3000) + } + def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + sql "TRUNCATE TABLE ${dbName}.${tableName}" Review Comment: DROP table to reset the properties. ########## regression-test/suites/backup_restore/test_backup_restore_table_auto_bucket.groovy: ########## @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_backup_restore_table_auto_bucket", "backup_restore") { + String repoName = "test_backup_restore_table_auto_bucket_repo" + String dbName = "backup_restore_table_with_auto_bucket_db" + String tableName = "auto_bucket_table" + + def syncer = getSyncer() + syncer.createS3Repository(repoName) + + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + CREATE TABLE ${dbName}.${tableName} ( + `id` LARGEINT NOT NULL, + `count` LARGEINT) + DISTRIBUTED BY HASH(`id`) BUCKETS AUTO + PROPERTIES + ( + "replication_num" = "1" + ) + """ + + List<String> values = [] + for(int i = 1;i <= 10; ++i){ + values.add("(${i}, ${i})") + } + sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}" + + def result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(),values.size()) + + String snapshotName = "test_backup_restore_table_auto_bucket_snapshot" + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + ON (${tableName}) + """ + + while (!syncer.checkSnapshotFinish(dbName)) { + Thread.sleep(3000) + } + def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + sql "TRUNCATE TABLE ${dbName}.${tableName}" Review Comment: Consider dropping the table and verifying that `DISTRIBUTED BY HASH(`id`) BUCKETS AUTO` exists in the result of `SHOW CREATE TABLE` after restoring finished. ########## regression-test/suites/backup_restore/test_backup_restore_reserve_dynamic_partition_false.groovy: ########## @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_backup_restore_reserve_dynamic_partition_false", "backup_restore") { + String repoName = "test_backup_restore_dynamic_partition_reserve_false_repo" + String dbName = "backup_restore_dynamic_partition_reserve_false_db" + String tableName = "dynamic_partition_reserve_false_table" + + def syncer = getSyncer() + syncer.createS3Repository(repoName) + + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + CREATE TABLE ${dbName}.${tableName} ( + `id` LARGEINT NOT NULL, + `sdate` DATE + ) + PARTITION BY RANGE(sdate)() + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES + ( + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "YEAR", + "dynamic_partition.start" = "-50", + "dynamic_partition.end" = "5", + "dynamic_partition.prefix" = "p", + "dynamic_partition.create_history_partition"="true", + "replication_num" = "1" + ) + """ + + sql """ + INSERT INTO ${dbName}.${tableName} VALUES + (1,"2023-11-01"), + (2,"2023-11-01"), + (3,"2023-11-01"), + (4,"2023-11-01"), + (5,"2023-11-01"), + (1,"2023-11-02"), + (2,"2023-11-02"), + (3,"2023-11-02"), + (4,"2023-11-02"), + (5,"2023-11-02"), + (1,"2024-11-01"), + (2,"2024-11-01"), + (3,"2024-11-01"), + (4,"2024-11-01"), + (5,"2024-11-01"), + (1,"2024-11-02"), + (2,"2024-11-02"), + (3,"2024-11-02"), + (4,"2024-11-02"), + (5,"2024-11-02") + """ + + def result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(),20) + + String snapshotName = "test_backup_restore_dynamic_partition_reserve_true_snapshot" + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + ON (${tableName}) + """ + + while (!syncer.checkSnapshotFinish(dbName)) { + Thread.sleep(3000) + } + def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + sql "TRUNCATE TABLE ${dbName}.${tableName}" + + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON (`${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_dynamic_partition_enable" = "false", + "replication_num" = "1" + ) + """ + while (!syncer.checkAllRestoreFinish(dbName)) { + Thread.sleep(3000) + } + + // def restore_properties = sql "SHOW CREATE TABLE ${dbName}.${tableName}" + + // assertTrue(restore_properties.indexOf("\"dynamic_partition.enable\" = \"false\"") != -1) Review Comment: ditto -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
