This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 41a58afba2e [fix](variant) Fix Variant test fixtures broken by #68447
(#69261)
41a58afba2e is described below
commit 41a58afba2e4bdb6c9b53fd9c1fe74e9c3d9494a
Author: lihangyu <[email protected]>
AuthorDate: Mon Oct 19 20:58:49 2026 +0800
[fix](variant) Fix Variant test fixtures broken by #68447 (#69261)
Issue Number: DORIS-29778
Problem Summary:
Test fixtures still assume the behavior that #68447 removed.
1.
`regression-test/suites/variant_p0/test_variant_json_functions.groovy`
builds its
Variant string row with `parse_to_variant('abc')`, which relied on the
old permissive
fallback that stored text that is not a JSON document as a Variant
string. Since
#68447, `parse_to_variant` rejects such input, so the INSERT fails and
the suite
stops before any assertion runs:
```
errCode = 2, detailMessage = [INVALID_ARGUMENT]Parse json document failed
at row 0,
error: [INVALID_ARGUMENT]Failed to parse JSON as Variant
```
Parse the equivalent JSON string document `"abc"` instead: it produces
the same
Variant string value, so the remaining assertions still pass and the
expected
result file is unchanged.
2. BE unit tests still pass the removed
`JsonToVariantOptions::check_duplicate_json_path`
field, so the BE UT build fails to compile:
```
core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp:115:32:
error:
no matching constructor for initialization of 'JsonStringToVariantEncoder'
format/transformer/vparquet_writer_test.cpp:198:32: error:
no matching constructor for initialization of 'JsonStringToVariantEncoder'
```
Remove the field from the six affected initializers in
`data_type_serde_arrow_test.cpp`,
`data_type_variant_v2_serde_jsonb_test.cpp`,
`variant_v2_execution_test.cpp` and
`format/transformer/vparquet_writer_test.cpp`
(the last two were added by #69194) (DORIS-29778).
3. `external_table_p0/paimon/write/test_paimon_write_variant_errors`
expects
`try_parse_to_variant('not-json')` to be stored as the Variant string
`not-json`,
while it now returns NULL, so the row 21 check fails:
```
ExpectRow: [21, false, null, not-json]
RealRow : [21, true, null, null]
```
---
be/test/core/data_type_serde/data_type_serde_arrow_test.cpp | 10 ++++------
.../data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp | 5 ++---
be/test/exprs/variant_v2_execution_test.cpp | 5 ++---
be/test/format/transformer/vparquet_writer_test.cpp | 10 ++++------
.../paimon/write/test_paimon_write_variant_errors.out | 2 +-
.../paimon/write/test_paimon_write_variant_errors.groovy | 3 +--
.../suites/variant_p0/test_variant_json_functions.groovy | 3 ++-
7 files changed, 16 insertions(+), 22 deletions(-)
diff --git a/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp
b/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp
index c32d552788d..902a55bf185 100644
--- a/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp
+++ b/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp
@@ -957,9 +957,8 @@ TEST(DataTypeSerDeArrowTest,
PlainArrowConverterDoesNotInferIcebergUuid) {
TEST(DataTypeSerDeArrowTest, IcebergVariantExtensionAndParquetSchema) {
auto make_variant_column = []() {
- JsonStringToVariantEncoder encoder({.max_json_key_length = 1024,
- .throw_on_invalid_json = true,
- .check_duplicate_json_path =
false});
+ JsonStringToVariantEncoder encoder(
+ {.max_json_key_length = 1024, .throw_on_invalid_json = true});
for (std::string_view json :
{std::string_view {R"({"name":"doris","n":1})"}, std::string_view
{"null"},
std::string_view {R"([1,true,"x"])"}}) {
@@ -1054,9 +1053,8 @@ TEST(DataTypeSerDeArrowTest,
IcebergVariantExtensionAndParquetSchema) {
TEST(DataTypeSerDeArrowTest, NestedIcebergVariantExtensionsAndParquetSchema) {
auto make_variant_column = [](std::initializer_list<std::string_view>
json_values) {
- JsonStringToVariantEncoder encoder({.max_json_key_length = 1024,
- .throw_on_invalid_json = true,
- .check_duplicate_json_path =
false});
+ JsonStringToVariantEncoder encoder(
+ {.max_json_key_length = 1024, .throw_on_invalid_json = true});
for (std::string_view json : json_values) {
encoder.add_json({json.data(), json.size()});
}
diff --git
a/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp
b/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp
index 5cecf9d9cef..a88aeec2d48 100644
--- a/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp
+++ b/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp
@@ -112,9 +112,8 @@ std::string json_at(const DataTypeVariantV2SerDe& serde,
const IColumn& column,
}
ColumnVariantV2::MutablePtr
encoded_json(std::initializer_list<std::string_view> rows) {
- JsonStringToVariantEncoder encoder({.max_json_key_length = 1024,
- .throw_on_invalid_json = true,
- .check_duplicate_json_path = false});
+ JsonStringToVariantEncoder encoder(
+ {.max_json_key_length = 1024, .throw_on_invalid_json = true});
for (std::string_view row : rows) {
encoder.add_json({row.data(), row.size()});
}
diff --git a/be/test/exprs/variant_v2_execution_test.cpp
b/be/test/exprs/variant_v2_execution_test.cpp
index e3db965fc22..75d045cf4b4 100644
--- a/be/test/exprs/variant_v2_execution_test.cpp
+++ b/be/test/exprs/variant_v2_execution_test.cpp
@@ -49,9 +49,8 @@ TEST(VariantV2ExecutionTest,
TypePropertiesDoNotAffectEquality) {
TEST(VariantV2ExecutionTest, CanonicalComparison) {
auto parse = [](std::initializer_list<std::string_view> rows) {
- JsonStringToVariantEncoder encoder({.max_json_key_length = 255,
- .throw_on_invalid_json = true,
- .check_duplicate_json_path =
false});
+ JsonStringToVariantEncoder encoder(
+ {.max_json_key_length = 255, .throw_on_invalid_json = true});
for (auto json : rows) {
encoder.add_json({json.data(), json.size()});
}
diff --git a/be/test/format/transformer/vparquet_writer_test.cpp
b/be/test/format/transformer/vparquet_writer_test.cpp
index c4b05cf045e..acab031b796 100644
--- a/be/test/format/transformer/vparquet_writer_test.cpp
+++ b/be/test/format/transformer/vparquet_writer_test.cpp
@@ -195,9 +195,8 @@ TEST_F(VParquetWriterTest,
WritesIcebergVariantAndCollectsLogicalMetrics) {
options, &schema_json, *schema);
ASSERT_TRUE(transformer.open().ok());
- JsonStringToVariantEncoder encoder({.max_json_key_length = 1024,
- .throw_on_invalid_json = true,
- .check_duplicate_json_path = false});
+ JsonStringToVariantEncoder encoder(
+ {.max_json_key_length = 1024, .throw_on_invalid_json = true});
for (std::string_view json : {std::string_view
{R"({"name":"doris","n":1})"},
std::string_view {"null"}, std::string_view
{R"([1,2,3])"}}) {
encoder.add_json({json.data(), json.size()});
@@ -282,9 +281,8 @@ TEST_F(VParquetWriterTest, WritesNestedIcebergVariant) {
options, &schema_json, *schema);
ASSERT_TRUE(transformer.open().ok());
- JsonStringToVariantEncoder encoder({.max_json_key_length = 1024,
- .throw_on_invalid_json = true,
- .check_duplicate_json_path = false});
+ JsonStringToVariantEncoder encoder(
+ {.max_json_key_length = 1024, .throw_on_invalid_json = true});
for (std::string_view json :
{std::string_view {R"({"kind":"nested"})"}, std::string_view {"null"},
std::string_view {"{}"}, std::string_view {"7"}}) {
diff --git
a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_errors.out
b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_errors.out
index f2333c25116..0c20b7ea6c5 100644
---
a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_errors.out
+++
b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_errors.out
@@ -1,7 +1,7 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !variant_after_errors --
20 false true {"recovered":true}
-21 false \N not-json
+21 true \N \N
22 false \N {"typed":"string"}
23 false \N 7
24 false \N values-string
diff --git
a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_errors.groovy
b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_errors.groovy
index cde2e529c5e..a4e18f3a0e4 100644
---
a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_errors.groovy
+++
b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_errors.groovy
@@ -202,8 +202,7 @@ suite("test_paimon_write_variant_errors",
"p0,external,paimon,nonConcurrent") {
row.collect { value -> value == null ? null : value.toString() }
})
- // Invalid JSON is preserved as a Variant string unless the global
- // throw-on-invalid-JSON option is enabled.
+ // try_parse_to_variant returns NULL for text that is not a JSON
document.
order_qt_variant_after_errors """
SELECT id, payload IS NULL,
CAST(payload['recovered'] AS BOOLEAN),
diff --git
a/regression-test/suites/variant_p0/test_variant_json_functions.groovy
b/regression-test/suites/variant_p0/test_variant_json_functions.groovy
index 45873317030..171f9282e0a 100644
--- a/regression-test/suites/variant_p0/test_variant_json_functions.groovy
+++ b/regression-test/suites/variant_p0/test_variant_json_functions.groovy
@@ -23,6 +23,7 @@ suite("test_variant_json_functions", "p0") {
DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1
PROPERTIES ("replication_num" = "1")
"""
+ // parse_to_variant only accepts JSON documents, so the Variant string row
is a JSON string.
sql """
INSERT INTO test_variant_json_functions
SELECT k, parse_to_variant(doc) FROM (
@@ -30,7 +31,7 @@ suite("test_variant_json_functions", "p0") {
UNION ALL SELECT 2, '{"a":2,"s":"yz","b":{"c":[3]},"n":{"m":true}}'
UNION ALL SELECT 3, '[1,"two",{"three":3}]'
UNION ALL SELECT 4, '42'
- UNION ALL SELECT 5, 'abc'
+ UNION ALL SELECT 5, '"abc"'
UNION ALL SELECT 6, NULL) t
"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
