This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2511ea0d6e2377f45401bf2f0c271da16d92664e
Author: slothever <18522955+w...@users.noreply.github.com>
AuthorDate: Thu Aug 22 18:55:23 2024 +0800
[fix](multi-catalog)fix max compute array and map type read offset (#39680)
fix for https://github.com/apache/doris/pull/39259
---
.../doris/maxcompute/MaxComputeColumnValue.java | 22 ++--
.../mc/test_max_compute_complex_type.out | 4 +-
.../mc/test_max_compute_complex_type.groovy | 127 +++++++++++++++++++++
3 files changed, 143 insertions(+), 10 deletions(-)
diff --git
a/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeColumnValue.java
b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeColumnValue.java
index 9c4f56068a7..644caf80d97 100644
---
a/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeColumnValue.java
+++
b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeColumnValue.java
@@ -53,6 +53,7 @@ import java.util.List;
public class MaxComputeColumnValue implements ColumnValue {
private static final Logger LOG =
Logger.getLogger(MaxComputeColumnValue.class);
private int idx;
+ private int offset = 0; // for complex type
private ValueVector column;
public MaxComputeColumnValue() {
@@ -67,6 +68,7 @@ public class MaxComputeColumnValue implements ColumnValue {
public void reset(ValueVector column) {
this.column = column;
this.idx = 0;
+ this.offset = 0;
}
@Override
@@ -233,26 +235,30 @@ public class MaxComputeColumnValue implements ColumnValue
{
public void unpackArray(List<ColumnValue> values) {
skippedIfNull();
ListVector listCol = (ListVector) column;
- for (int i = 0; i < listCol.getDataVector().getValueCount(); i++) {
- MaxComputeColumnValue val = new
MaxComputeColumnValue(listCol.getDataVector(), i);
+ int elemSize = listCol.getObject(idx).size();
+ for (int i = 0; i < elemSize; i++) {
+ MaxComputeColumnValue val = new
MaxComputeColumnValue(listCol.getDataVector(), offset);
values.add(val);
+ offset++;
}
+ idx++;
}
@Override
public void unpackMap(List<ColumnValue> keys, List<ColumnValue> values) {
skippedIfNull();
MapVector mapCol = (MapVector) column;
+ int elemSize = mapCol.getObject(idx).size();
FieldVector keyList =
mapCol.getDataVector().getChildrenFromFields().get(0);
- for (int i = 0; i < keyList.getValueCount(); i++) {
- MaxComputeColumnValue val = new MaxComputeColumnValue(keyList, i);
- keys.add(val);
- }
FieldVector valList =
mapCol.getDataVector().getChildrenFromFields().get(1);
- for (int i = 0; i < valList.getValueCount(); i++) {
- MaxComputeColumnValue val = new MaxComputeColumnValue(valList, i);
+ for (int i = 0; i < elemSize; i++) {
+ MaxComputeColumnValue key = new MaxComputeColumnValue(keyList,
offset);
+ keys.add(key);
+ MaxComputeColumnValue val = new MaxComputeColumnValue(valList,
offset);
values.add(val);
+ offset++;
}
+ idx++;
}
@Override
diff --git
a/regression-test/data/external_table_p0/mc/test_max_compute_complex_type.out
b/regression-test/data/external_table_p0/mc/test_max_compute_complex_type.out
index 2b62ebaff75..86df55f16c5 100644
---
a/regression-test/data/external_table_p0/mc/test_max_compute_complex_type.out
+++
b/regression-test/data/external_table_p0/mc/test_max_compute_complex_type.out
@@ -13,5 +13,5 @@
{"phone_number":2345671, "email":"us...@example.com", "addr":"Addr2"}
{"id":"user2", "age":30}
-- !mc_q4 --
-user1 [{"activity_date":"2024-08-01",
"activities":{"cooking":{"details":"Made vegan meal",
"metrics":{"time_spent":1.5, "calories":500, "duration":2, "rating":8.5,
"duration":1, "calories":300, "items":10, "cost":100, "distance":10,
"elevation":500, "photos_taken":100, "time":4, "duration":3, "participants":15,
"items":5, "cost":1500, "duration":30.5, "calories":200, "pages":50, "time":2,
"distance":500, "time":3, "duration":1.5, "participants":5}},
"movie":{"details":"Watched action mo [...]
-user2 [{"activity_date":"2024-08-01",
"activities":{"cooking":{"details":"Made vegan meal",
"metrics":{"time_spent":1.5, "calories":500, "duration":2, "rating":8.5,
"duration":1, "calories":300, "items":10, "cost":100, "distance":10,
"elevation":500, "photos_taken":100, "time":4, "duration":3, "participants":15,
"items":5, "cost":1500, "duration":30.5, "calories":200, "pages":50, "time":2,
"distance":500, "time":3, "duration":1.5, "participants":5}},
"movie":{"details":"Watched action mo [...]
+user1 [{"activity_date":"2024-08-01",
"activities":{"cooking":{"details":"Made vegan meal",
"metrics":{"time_spent":1.5, "calories":500}}, "movie":{"details":"Watched
action movie", "metrics":{"time_spent":1.5, "calories":500}}}},
{"activity_date":"2024-08-02", "activities":{"cooking":{"details":"Made vegan
meal", "metrics":{"time_spent":1.5, "calories":500}},
"movie":{"details":"Watched action movie", "metrics":{"time_spent":1.5,
"calories":500}}}}]
+user2 [{"activity_date":"2024-08-01",
"activities":{"cooking":{"details":"Made vegan meal",
"metrics":{"time_spent":1.5, "calories":500}}, "movie":{"details":"Watched
action movie", "metrics":{"time_spent":1.5, "calories":500}}}},
{"activity_date":"2024-08-02", "activities":{"cooking":{"details":"Made vegan
meal", "metrics":{"time_spent":1.5, "calories":500}},
"movie":{"details":"Watched action movie", "metrics":{"time_spent":1.5,
"calories":500}}}}]
diff --git
a/regression-test/suites/external_table_p0/mc/test_max_compute_complex_type.groovy
b/regression-test/suites/external_table_p0/mc/test_max_compute_complex_type.groovy
index cb6178e0102..a3de3715f91 100644
---
a/regression-test/suites/external_table_p0/mc/test_max_compute_complex_type.groovy
+++
b/regression-test/suites/external_table_p0/mc/test_max_compute_complex_type.groovy
@@ -15,6 +15,133 @@
// specific language governing permissions and limitations
// under the License.
+/*
+ // Test Case DDL
+ create table array_table (
+ id int,
+ arr1 ARRAY<BIGINT>,
+ arr2 ARRAY<VARCHAR(10)>,
+ arr3 ARRAY<DOUBLE>,
+ arr4 ARRAY<DATE>,
+ arr5 ARRAY<DATETIME>
+ );
+ INSERT INTO array_table VALUES(1, array(1, 2, 3), array('a', 'b', 'c'),
array(1.2, 1.3), array(date('2023-05-23')), array(datetime('2023-05-23
13:55:12')));
+ INSERT INTO array_table VALUES(2, array(1, 2, 3), array('a', 'b', 'c'),
array(1.2, 1.3), array(date('2023-05-23')), array(datetime('2023-05-23
13:55:12')));
+ INSERT INTO array_table VALUES(3, array(1, 2, 3), array('a', 'b', 'c'),
array(1.3), array(date('2023-05-23')), array(datetime('2023-05-23 13:55:12')));
+
+ create table map_table (
+ arr1 MAP<BIGINT, DOUBLE>,
+ arr2 MAP<BIGINT, STRING>
+ );
+ INSERT INTO map_table (arr1, arr2)
+ VALUES (
+ MAP(1, 2.5, 2, 3.75),
+ MAP(1, 'example1', 2, 'example2')
+ );
+ INSERT INTO map_table (arr1, arr2)
+ VALUES (
+ MAP(3, 2.5, 99, 3.75),
+ MAP(349, 'asd', 324, 'uid')
+ );
+
+ create table struct_table (
+ user_info STRUCT<id: STRING,age: INT>,
+ contact_info STRUCT<phone_number: BIGINT, email: STRING, addr:
VARCHAR(10)>
+ );
+
+ INSERT INTO struct_table VALUES
+ (
+ named_struct('id', 'user1', 'age', 25),
+ named_struct('phone_number', 123450, 'email', 'us...@example.com',
'addr', 'Addr1')
+ ),
+ (
+ named_struct('id', 'user2', 'age', 30),
+ named_struct('phone_number', 2345671, 'email', 'us...@example.com',
'addr', 'Addr2')
+ ),
+ (
+ named_struct('id', 'user3', 'age', 35),
+ named_struct('phone_number', 3456789, 'email', 'us...@example.com',
'addr', 'Addr3')
+ );
+
+ CREATE TABLE nested_complex_table (
+ user_id STRING,
+ user_profile STRUCT<
+ name: STRING,
+ age: INT,
+ preferences: MAP<
+ STRING,
+ STRUCT<
+ preference_id: INT,
+ preference_values: ARRAY<STRING>
+ >
+ >
+ >,
+ activity_log ARRAY<
+ STRUCT<
+ activity_date: STRING,
+ activities: MAP<
+ STRING,
+ STRUCT<
+ details: STRING,
+ metrics: MAP<STRING, DOUBLE>
+ >
+ >
+ >
+ >
+ );
+ INSERT INTO nested_complex_table VALUES
+ (
+ 'user1',
+ named_struct('name', 'Alice', 'age', 28, 'preferences', map(
+ 'sports', named_struct('preference_id', 101, 'preference_values',
array('soccer', 'tennis')),
+ 'music', named_struct('preference_id', 102, 'preference_values',
array('rock', 'classical'))
+ )),
+ array(
+ named_struct('activity_date', '2024-08-01', 'activities', map(
+ 'workout', named_struct('details', 'Morning run', 'metrics',
map('duration', 30.5, 'calories', 200.0)),
+ 'reading', named_struct('details', 'Read book on Hive',
'metrics', map('pages', 50.0, 'time', 2.0))
+ )),
+ named_struct('activity_date', '2024-08-02', 'activities', map(
+ 'travel', named_struct('details', 'Flight to NY', 'metrics',
map('distance', 500.0, 'time', 3.0)),
+ 'meeting', named_struct('details', 'Project meeting',
'metrics', map('duration', 1.5, 'participants', 5.0))
+ ))
+ )
+ ),
+ (
+ 'user2',
+ named_struct('name', 'Bob', 'age', 32, 'preferences', map(
+ 'books', named_struct('preference_id', 201, 'preference_values',
array('fiction', 'non-fiction')),
+ 'travel', named_struct('preference_id', 202, 'preference_values',
array('beaches', 'mountains'))
+ )),
+ array(
+ named_struct('activity_date', '2024-08-01', 'activities', map(
+ 'hiking', named_struct('details', 'Mountain trail', 'metrics',
map('distance', 10.0, 'elevation', 500.0)),
+ 'photography', named_struct('details', 'Wildlife photoshoot',
'metrics', map('photos_taken', 100.0, 'time', 4.0))
+ )),
+ named_struct('activity_date', '2024-08-02', 'activities', map(
+ 'workshop', named_struct('details', 'Photography workshop',
'metrics', map('duration', 3.0, 'participants', 15.0)),
+ 'shopping', named_struct('details', 'Bought camera gear',
'metrics', map('items', 5.0, 'cost', 1500.0))
+ ))
+ )
+ ),
+ (
+ 'user3',
+ named_struct('name', 'Carol', 'age', 24, 'preferences', map(
+ 'food', named_struct('preference_id', 301, 'preference_values',
array('vegan', 'desserts')),
+ 'movies', named_struct('preference_id', 302, 'preference_values',
array('action', 'comedy'))
+ )),
+ array(
+ named_struct('activity_date', '2024-08-01', 'activities', map(
+ 'cooking', named_struct('details', 'Made vegan meal',
'metrics', map('time_spent', 1.5, 'calories', 500.0)),
+ 'movie', named_struct('details', 'Watched action movie',
'metrics', map('duration', 2.0, 'rating', 8.5))
+ )),
+ named_struct('activity_date', '2024-08-02', 'activities', map(
+ 'gym', named_struct('details', 'Strength training', 'metrics',
map('duration', 1.0, 'calories', 300.0)),
+ 'shopping', named_struct('details', 'Bought groceries',
'metrics', map('items', 10.0, 'cost', 100.0))
+ ))
+ )
+ );
+ */
suite("test_max_compute_complex_type",
"p0,external,doris,external_docker,external_docker_doris") {
String enabled = context.config.otherConfigs.get("enableMaxComputeTest")
if (enabled != null && enabled.equalsIgnoreCase("true")) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
