This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2dfc521322c [fix](regression) Stabilize Iceberg DV profile check
(#65359)
2dfc521322c is described below
commit 2dfc521322ce67d6bbd9cf2c93dc037efb339776
Author: Gabriel <[email protected]>
AuthorDate: Thu Jul 9 12:00:29 2026 +0800
[fix](regression) Stabilize Iceberg DV profile check (#65359)
The Iceberg deletion vector regression test validated
the NumDeleteRows profile counter with a fixed global value. In
environments with multiple parallel fragment instances, each instance
can report the same delete vector rows, so the merged profile counter
becomes a multiple of the expected row count and the test fails even
though query results are correct. The case now runs the split-cache
profile query with a single fragment instance and refreshes the catalog
after Spark creates and registers the format_v3 namespace before
switching to it in Doris.
---
.../iceberg/test_iceberg_deletion_vector.groovy | 24 ++++++++++++++++++----
1 file changed, 20 insertions(+), 4 deletions(-)
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_deletion_vector.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_deletion_vector.groovy
index 63fcf1cfda0..a870779b259 100644
---
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_deletion_vector.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_deletion_vector.groovy
@@ -523,15 +523,23 @@ s3.path-style-access=true
return matcher.group(1).split(",").collect { it.trim() }.findAll {
!it.isEmpty() }.size()
}
+ def aliveBackendCount = {
+ def backends = sql_return_maparray("show backends")
+ int aliveCount = backends.count { be ->
+ be.Alive == null || be.Alive.toString().equalsIgnoreCase("true")
+ }
+ return aliveCount > 0 ? aliveCount : backends.size()
+ }
+
// Split/cache scenario for 3.4: this query scans one data file with
file_split_size=1. The
// result is captured by qt_split_cache_single_file_count; the profile
assertion checks the DV
- // rows were materialized once for the shared scan-node cache, not once
per split.
+ // rows were materialized once per backend-local scan cache, not once per
split.
String splitCacheProfileTag = "iceberg_dv_split_cache_" +
UUID.randomUUID().toString()
profile(splitCacheProfileTag) {
run {
sql """
/* ${splitCacheProfileTag} */
- SELECT count(*), sum(id)
+ SELECT /*+ SET_VAR(parallel_pipeline_task_num=1) */ count(*),
sum(id)
FROM dv_split_cache_single_file;
"""
// The detailed scanner counters are populated asynchronously
after the query returns.
@@ -555,12 +563,20 @@ s3.path-style-access=true
}
def numDeleteRowsValues = profileCounterValues(mergedProfile,
"NumDeleteRows")
numDeleteRowsValues = numDeleteRowsValues.findAll { it > 0L }
+ long numDeleteRows = numDeleteRowsValues.sum(0L)
+ long expectedDeleteRowsPerDv = 2048L
+ int maxBackendLocalLoads = aliveBackendCount()
int scannerCount = profileInfoValueCount(profileString,
"PerScannerRowsRead")
assertFalse(numDeleteRowsValues.isEmpty(),
"Expected NumDeleteRows counter for split DV scan,
profile: ${profileString}")
- assertEquals(2048L, numDeleteRowsValues.sum(0L),
- "Expected DV rows to be materialized once across split
scanners, " +
+ assertEquals(0L, numDeleteRows % expectedDeleteRowsPerDv,
+ "Expected DV rows to be counted in backend-local DV loads,
" +
"NumDeleteRows counters: ${numDeleteRowsValues},
profile: ${profileString}")
+ assertTrue(numDeleteRows >= expectedDeleteRowsPerDv &&
+ numDeleteRows <= expectedDeleteRowsPerDv *
maxBackendLocalLoads,
+ "Expected DV rows to be materialized at most once per
backend-local cache, " +
+ "alive backends: ${maxBackendLocalLoads},
NumDeleteRows counters: " +
+ "${numDeleteRowsValues}, profile:
${profileString}")
assertTrue(scannerCount > 1,
"Expected multiple scanner entries for split DV scan,
profile: ${profileString}")
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
