This is an automated email from the ASF dual-hosted git repository.
caiconghui pushed a commit to branch fix_export_bug
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 40c9aa67520e6d7d4354908be613c366935d4c17
Author: caiconghui1 <caicongh...@jd.com>
AuthorDate: Tue May 28 21:29:57 2024 +0800
[fix](export)fix that export table without index data when table has
materialized view index
---
.../main/java/org/apache/doris/load/ExportJob.java | 9 +-
.../test_export_table_with_materialized_view.out | 205 +++++++++++++++++++++
...test_export_table_with_materialized_view.groovy | 128 +++++++++++++
3 files changed, 337 insertions(+), 5 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
index 126b1f3f948..7b167df69fa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
@@ -541,11 +541,10 @@ public class ExportJob implements Writable {
if (!isPartitionConsistency()) {
partitionToVersion.put(partition.getName(),
partition.getVisibleVersion());
}
- for (MaterializedIndex index :
partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
- List<Long> tablets = index.getTabletIdsInOrder();
- tabletsAllNum += tablets.size();
- tabletIdList.add(tablets);
- }
+ MaterializedIndex index = partition.getBaseIndex();
+ List<Long> tablets = index.getTabletIdsInOrder();
+ tabletsAllNum += tablets.size();
+ tabletIdList.add(tablets);
}
} finally {
table.readUnlock();
diff --git
a/regression-test/data/export_p0/test_export_table_with_materialized_view.out
b/regression-test/data/export_p0/test_export_table_with_materialized_view.out
new file mode 100644
index 00000000000..7e0240d57de
--- /dev/null
+++
b/regression-test/data/export_p0/test_export_table_with_materialized_view.out
@@ -0,0 +1,205 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !order_qt_select_export1 --
+1 2017-10-01 2 1 1 1 1 1
+2 2017-10-01 3 2 2 2 2 2
+3 2017-10-01 4 3 3 3 3 3
+4 2017-10-01 5 4 4 4 4 4
+5 2017-10-01 6 5 5 5 5 5
+6 2017-10-01 7 6 6 6 6 6
+7 2017-10-01 8 7 7 7 7 7
+8 2017-10-01 9 8 8 8 8 8
+9 2017-10-01 10 9 9 9 9 9
+10 2017-10-01 11 10 10 10 10 10
+11 2017-10-01 12 11 11 11 11 11
+12 2017-10-01 13 12 12 12 12 12
+13 2017-10-01 14 13 13 13 13 13
+14 2017-10-01 15 14 14 14 14 14
+15 2017-10-01 16 15 15 15 15 15
+16 2017-10-01 17 16 16 16 16 16
+17 2017-10-01 18 17 17 17 17 17
+18 2017-10-01 19 18 18 18 18 18
+19 2017-10-01 20 19 19 19 19 19
+20 2017-10-01 21 20 20 20 20 20
+21 2017-10-01 22 21 21 21 21 21
+22 2017-10-01 23 22 22 22 22 22
+23 2017-10-01 24 23 23 23 23 23
+24 2017-10-01 25 24 24 24 24 24
+25 2017-10-01 26 25 25 25 25 25
+26 2017-10-01 27 26 26 26 26 26
+27 2017-10-01 28 27 27 27 27 27
+28 2017-10-01 29 28 28 28 28 28
+29 2017-10-01 30 29 29 29 29 29
+30 2017-10-01 31 30 30 30 30 30
+31 2017-10-01 32 31 31 31 31 31
+32 2017-10-01 33 32 32 32 32 32
+33 2017-10-01 34 33 33 33 33 33
+34 2017-10-01 35 34 34 34 34 34
+35 2017-10-01 36 35 35 35 35 35
+36 2017-10-01 37 36 36 36 36 36
+37 2017-10-01 38 37 37 37 37 37
+38 2017-10-01 39 38 38 38 38 38
+39 2017-10-01 40 39 39 39 39 39
+40 2017-10-01 41 40 40 40 40 40
+41 2017-10-01 42 41 41 41 41 41
+42 2017-10-01 43 42 42 42 42 42
+43 2017-10-01 44 43 43 43 43 43
+44 2017-10-01 45 44 44 44 44 44
+45 2017-10-01 46 45 45 45 45 45
+46 2017-10-01 47 46 46 46 46 46
+47 2017-10-01 48 47 47 47 47 47
+48 2017-10-01 49 48 48 48 48 48
+49 2017-10-01 50 49 49 49 49 49
+50 2017-10-01 51 50 50 50 50 50
+51 2017-10-01 52 51 51 51 51 51
+52 2017-10-01 53 52 52 52 52 52
+53 2017-10-01 54 53 53 53 53 53
+54 2017-10-01 55 54 54 54 54 54
+55 2017-10-01 56 55 55 55 55 55
+56 2017-10-01 57 56 56 56 56 56
+57 2017-10-01 58 57 57 57 57 57
+58 2017-10-01 59 58 58 58 58 58
+59 2017-10-01 60 59 59 59 59 59
+60 2017-10-01 61 60 60 60 60 60
+61 2017-10-01 62 61 61 61 61 61
+62 2017-10-01 63 62 62 62 62 62
+63 2017-10-01 64 63 63 63 63 63
+64 2017-10-01 65 64 64 64 64 64
+65 2017-10-01 66 65 65 65 65 65
+66 2017-10-01 67 66 66 66 66 66
+67 2017-10-01 68 67 67 67 67 67
+68 2017-10-01 69 68 68 68 68 68
+69 2017-10-01 70 69 69 69 69 69
+70 2017-10-01 71 70 70 70 70 70
+71 2017-10-01 72 71 71 71 71 71
+72 2017-10-01 73 72 72 72 72 72
+73 2017-10-01 74 73 73 73 73 73
+74 2017-10-01 75 74 74 74 74 74
+75 2017-10-01 76 75 75 75 75 75
+76 2017-10-01 77 76 76 76 76 76
+77 2017-10-01 78 77 77 77 77 77
+78 2017-10-01 79 78 78 78 78 78
+79 2017-10-01 80 79 79 79 79 79
+80 2017-10-01 81 80 80 80 80 80
+81 2017-10-01 82 81 81 81 81 81
+82 2017-10-01 83 82 82 82 82 82
+83 2017-10-01 84 83 83 83 83 83
+84 2017-10-01 85 84 84 84 84 84
+85 2017-10-01 86 85 85 85 85 85
+86 2017-10-01 87 86 86 86 86 86
+87 2017-10-01 88 87 87 87 87 87
+88 2017-10-01 89 88 88 88 88 88
+89 2017-10-01 90 89 89 89 89 89
+90 2017-10-01 91 90 90 90 90 90
+91 2017-10-01 92 91 91 91 91 91
+92 2017-10-01 93 92 92 92 92 92
+93 2017-10-01 94 93 93 93 93 93
+94 2017-10-01 95 94 94 94 94 94
+95 2017-10-01 96 95 95 95 95 95
+96 2017-10-01 97 96 96 96 96 96
+97 2017-10-01 98 97 97 97 97 97
+98 2017-10-01 99 98 98 98 98 98
+99 2017-10-01 100 99 99 99 99 99
+100 2018-10-01 1 1 2 3 4 5
+
+-- !order_qt_select_load1 --
+1 2017-10-01 2 1 1 1 1 1
+2 2017-10-01 3 2 2 2 2 2
+3 2017-10-01 4 3 3 3 3 3
+4 2017-10-01 5 4 4 4 4 4
+5 2017-10-01 6 5 5 5 5 5
+6 2017-10-01 7 6 6 6 6 6
+7 2017-10-01 8 7 7 7 7 7
+8 2017-10-01 9 8 8 8 8 8
+9 2017-10-01 10 9 9 9 9 9
+10 2017-10-01 11 10 10 10 10 10
+11 2017-10-01 12 11 11 11 11 11
+12 2017-10-01 13 12 12 12 12 12
+13 2017-10-01 14 13 13 13 13 13
+14 2017-10-01 15 14 14 14 14 14
+15 2017-10-01 16 15 15 15 15 15
+16 2017-10-01 17 16 16 16 16 16
+17 2017-10-01 18 17 17 17 17 17
+18 2017-10-01 19 18 18 18 18 18
+19 2017-10-01 20 19 19 19 19 19
+20 2017-10-01 21 20 20 20 20 20
+21 2017-10-01 22 21 21 21 21 21
+22 2017-10-01 23 22 22 22 22 22
+23 2017-10-01 24 23 23 23 23 23
+24 2017-10-01 25 24 24 24 24 24
+25 2017-10-01 26 25 25 25 25 25
+26 2017-10-01 27 26 26 26 26 26
+27 2017-10-01 28 27 27 27 27 27
+28 2017-10-01 29 28 28 28 28 28
+29 2017-10-01 30 29 29 29 29 29
+30 2017-10-01 31 30 30 30 30 30
+31 2017-10-01 32 31 31 31 31 31
+32 2017-10-01 33 32 32 32 32 32
+33 2017-10-01 34 33 33 33 33 33
+34 2017-10-01 35 34 34 34 34 34
+35 2017-10-01 36 35 35 35 35 35
+36 2017-10-01 37 36 36 36 36 36
+37 2017-10-01 38 37 37 37 37 37
+38 2017-10-01 39 38 38 38 38 38
+39 2017-10-01 40 39 39 39 39 39
+40 2017-10-01 41 40 40 40 40 40
+41 2017-10-01 42 41 41 41 41 41
+42 2017-10-01 43 42 42 42 42 42
+43 2017-10-01 44 43 43 43 43 43
+44 2017-10-01 45 44 44 44 44 44
+45 2017-10-01 46 45 45 45 45 45
+46 2017-10-01 47 46 46 46 46 46
+47 2017-10-01 48 47 47 47 47 47
+48 2017-10-01 49 48 48 48 48 48
+49 2017-10-01 50 49 49 49 49 49
+50 2017-10-01 51 50 50 50 50 50
+51 2017-10-01 52 51 51 51 51 51
+52 2017-10-01 53 52 52 52 52 52
+53 2017-10-01 54 53 53 53 53 53
+54 2017-10-01 55 54 54 54 54 54
+55 2017-10-01 56 55 55 55 55 55
+56 2017-10-01 57 56 56 56 56 56
+57 2017-10-01 58 57 57 57 57 57
+58 2017-10-01 59 58 58 58 58 58
+59 2017-10-01 60 59 59 59 59 59
+60 2017-10-01 61 60 60 60 60 60
+61 2017-10-01 62 61 61 61 61 61
+62 2017-10-01 63 62 62 62 62 62
+63 2017-10-01 64 63 63 63 63 63
+64 2017-10-01 65 64 64 64 64 64
+65 2017-10-01 66 65 65 65 65 65
+66 2017-10-01 67 66 66 66 66 66
+67 2017-10-01 68 67 67 67 67 67
+68 2017-10-01 69 68 68 68 68 68
+69 2017-10-01 70 69 69 69 69 69
+70 2017-10-01 71 70 70 70 70 70
+71 2017-10-01 72 71 71 71 71 71
+72 2017-10-01 73 72 72 72 72 72
+73 2017-10-01 74 73 73 73 73 73
+74 2017-10-01 75 74 74 74 74 74
+75 2017-10-01 76 75 75 75 75 75
+76 2017-10-01 77 76 76 76 76 76
+77 2017-10-01 78 77 77 77 77 77
+78 2017-10-01 79 78 78 78 78 78
+79 2017-10-01 80 79 79 79 79 79
+80 2017-10-01 81 80 80 80 80 80
+81 2017-10-01 82 81 81 81 81 81
+82 2017-10-01 83 82 82 82 82 82
+83 2017-10-01 84 83 83 83 83 83
+84 2017-10-01 85 84 84 84 84 84
+85 2017-10-01 86 85 85 85 85 85
+86 2017-10-01 87 86 86 86 86 86
+87 2017-10-01 88 87 87 87 87 87
+88 2017-10-01 89 88 88 88 88 88
+89 2017-10-01 90 89 89 89 89 89
+90 2017-10-01 91 90 90 90 90 90
+91 2017-10-01 92 91 91 91 91 91
+92 2017-10-01 93 92 92 92 92 92
+93 2017-10-01 94 93 93 93 93 93
+94 2017-10-01 95 94 94 94 94 94
+95 2017-10-01 96 95 95 95 95 95
+96 2017-10-01 97 96 96 96 96 96
+97 2017-10-01 98 97 97 97 97 97
+98 2017-10-01 99 98 98 98 98 98
+99 2017-10-01 100 99 99 99 99 99
+100 2018-10-01 1 1 2 3 4 5
+
diff --git
a/regression-test/suites/export_p0/test_export_table_with_materialized_view.groovy
b/regression-test/suites/export_p0/test_export_table_with_materialized_view.groovy
new file mode 100644
index 00000000000..5f05d0fd91a
--- /dev/null
+++
b/regression-test/suites/export_p0/test_export_table_with_materialized_view.groovy
@@ -0,0 +1,128 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("test_export_parquet", "p0") {
+ // open nereids
+ sql """ set enable_nereids_planner=true """
+ sql """ set enable_fallback_to_original_planner=false """
+
+
+ String ak = getS3AK()
+ String sk = getS3SK()
+ String s3_endpoint = getS3Endpoint()
+ String region = getS3Region()
+ String bucket = context.config.otherConfigs.get("s3BucketName");
+
+
+ def table_export_name = "test_export_table_with_materialized_view"
+ def table_load_name = "test_load_table_with_rollup"
+ def outfile_path_prefix =
"""${bucket}/export/p0/export_table_with_materialized_view/exp"""
+
+ // create table and insert
+ sql """ DROP TABLE IF EXISTS ${table_export_name} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${table_export_name} (
+ `k1` INT NOT NULL,
+ `k2` DATE NOT NULL COMMENT "数据灌入日期",
+ `k3` BIGINT NOT NULL,
+ `v1` SMALLINT COMMENT "",
+ `v2` TINYINT COMMENT "",
+ `v3` int COMMENT "",
+ `v4` bigint COMMENT "",
+ `v5` largeint COMMENT ""
+ )
+ DISTRIBUTED BY HASH(k3) BUCKETS 10 PROPERTIES("replication_num" = "1");
+ """
+ sql """
+ CREATE MATERIALIZED VIEW export_table_materialized_view AS SELECT k1,
sum(v1) FROM ${table_export_name} GROUP BY k1;
+ """
+ StringBuilder sb = new StringBuilder()
+ int i = 1
+ for (; i < 100; i ++) {
+ sb.append("""
+ (${i}, '2017-10-01', ${i} + 1, ${i % 128}, ${i}, ${i}, ${i}, ${i}),
+ """)
+ }
+ sb.append("""
+ (${i}, '2018-10-01', 1, 1, 2, 3, 4, 5)
+ """)
+ sql """ INSERT INTO ${table_export_name} VALUES
+ ${sb.toString()}
+ """
+ def insert_res = sql "show last insert;"
+ logger.info("insert result: " + insert_res.toString())
+ order_qt_select_export1 """ SELECT * FROM ${table_export_name} t ORDER BY
k1; """
+
+
+ def waiting_export = { export_label ->
+ while (true) {
+ def res = sql """ show export where label = "${export_label}" """
+ logger.info("export state: " + res[0][2])
+ if (res[0][2] == "FINISHED") {
+ def json = parseJson(res[0][11])
+ assert json instanceof List
+ assertEquals("1", json.fileNumber[0][0])
+ log.info("outfile_path: ${json.url[0][0]}")
+ return json.url[0][0];
+ } else if (res[0][2] == "CANCELLED") {
+ throw new IllegalStateException("""export failed:
${res[0][10]}""")
+ } else {
+ sleep(5000)
+ }
+ }
+ }
+
+ def uuid = UUID.randomUUID().toString()
+ def outFilePath = """${outfile_path_prefix}_${uuid}"""
+ def label = "label_${uuid}"
+ try {
+ // exec export
+ sql """
+ EXPORT TABLE ${table_export_name} TO "s3://${outFilePath}/"
+ PROPERTIES(
+ "label" = "${label}",
+ "format" = "parquet",
+ 'columns' = 'k1, k2, k3, v1, v2, v3, v4, v5'
+ )
+ WITH S3(
+ "s3.endpoint" = "${s3_endpoint}",
+ "s3.region" = "${region}",
+ "s3.secret_key"="${sk}",
+ "s3.access_key" = "${ak}"
+ );
+ """
+ def outfile_url = waiting_export.call(label)
+
+ order_qt_select_load1 """ select * from s3(
+ "uri" = "http://${s3_endpoint}${outfile_url.substring(4,
outfile_url.length() - 1)}0.parquet",
+ "s3.access_key"= "${ak}",
+ "s3.secret_key" = "${sk}",
+ "format" = "parquet",
+ "region" = "${region}"
+ ) ORDER BY user_id;
+ """
+
+ } finally {
+ try_sql("DROP TABLE IF EXISTS ${table_export_name}")
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
