This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 8cfbca23a fix: run scan metrics test with both native_datafusion and
native_iceberg_compat (#3690)
8cfbca23a is described below
commit 8cfbca23a36424e9bcad5195b2eb815267855496
Author: Andy Grove <[email protected]>
AuthorDate: Fri Mar 13 14:52:33 2026 -0600
fix: run scan metrics test with both native_datafusion and
native_iceberg_compat (#3690)
The "Comet native metrics: scan" test only ran with SCAN_AUTO (which
resolves to native_iceberg_compat), missing coverage for native_datafusion.
Changes:
- Run the test with both SCAN_NATIVE_DATAFUSION and
SCAN_NATIVE_ICEBERG_COMPAT
- Use find() with isDefined assert instead of find().foreach() to fail
explicitly if no scan node is found
- Remove the WHERE clause that returned 0 rows due to predicate pushdown
pruning all rows in native_datafusion mode
- Add scan mode context to assertion messages for easier debugging
---
.../org/apache/comet/exec/CometExecSuite.scala | 57 ++++++++++++++--------
1 file changed, 36 insertions(+), 21 deletions(-)
diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala
b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala
index 91a02d69f..0bf9bbc95 100644
--- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala
@@ -558,35 +558,50 @@ class CometExecSuite extends CometTestBase {
}
test("Comet native metrics: scan") {
- withSQLConf(CometConf.COMET_EXEC_ENABLED.key -> "true") {
- withTempDir { dir =>
- val path = new Path(dir.toURI.toString, "native-scan.parquet")
- makeParquetFileAllPrimitiveTypes(path, dictionaryEnabled = true, 10000)
- withParquetTable(path.toString, "tbl") {
- val df = sql("SELECT * FROM tbl WHERE _2 > _3")
- df.collect()
+ Seq(CometConf.SCAN_NATIVE_DATAFUSION,
CometConf.SCAN_NATIVE_ICEBERG_COMPAT).foreach {
+ scanMode =>
+ withSQLConf(
+ CometConf.COMET_EXEC_ENABLED.key -> "true",
+ CometConf.COMET_NATIVE_SCAN_IMPL.key -> scanMode) {
+ withTempDir { dir =>
+ val path = new Path(dir.toURI.toString, "native-scan.parquet")
+ makeParquetFileAllPrimitiveTypes(path, dictionaryEnabled = true,
10000)
+ withParquetTable(path.toString, "tbl") {
+ val df = sql("SELECT * FROM tbl")
+ df.collect()
- find(df.queryExecution.executedPlan)(s =>
- s.isInstanceOf[CometScanExec] ||
s.isInstanceOf[CometNativeScanExec])
- .foreach(scan => {
- val metrics = scan.metrics
+ val scan = find(df.queryExecution.executedPlan)(s =>
+ s.isInstanceOf[CometScanExec] ||
s.isInstanceOf[CometNativeScanExec])
+ assert(scan.isDefined, s"Expected to find a Comet scan node for
$scanMode")
+ val metrics = scan.get.metrics
- assert(metrics.contains("time_elapsed_scanning_total"))
+ assert(
+ metrics.contains("time_elapsed_scanning_total"),
+ s"[$scanMode] Missing time_elapsed_scanning_total. Available:
${metrics.keys}")
assert(metrics.contains("bytes_scanned"))
assert(metrics.contains("output_rows"))
assert(metrics.contains("time_elapsed_opening"))
assert(metrics.contains("time_elapsed_processing"))
assert(metrics.contains("time_elapsed_scanning_until_data"))
- assert(metrics("time_elapsed_scanning_total").value > 0)
- assert(metrics("bytes_scanned").value > 0)
- assert(metrics("output_rows").value > 0)
- assert(metrics("time_elapsed_opening").value > 0)
- assert(metrics("time_elapsed_processing").value > 0)
- assert(metrics("time_elapsed_scanning_until_data").value > 0)
- })
-
+ assert(
+ metrics("time_elapsed_scanning_total").value > 0,
+ s"[$scanMode] time_elapsed_scanning_total should be > 0")
+ assert(
+ metrics("bytes_scanned").value > 0,
+ s"[$scanMode] bytes_scanned should be > 0")
+ assert(metrics("output_rows").value > 0, s"[$scanMode]
output_rows should be > 0")
+ assert(
+ metrics("time_elapsed_opening").value > 0,
+ s"[$scanMode] time_elapsed_opening should be > 0")
+ assert(
+ metrics("time_elapsed_processing").value > 0,
+ s"[$scanMode] time_elapsed_processing should be > 0")
+ assert(
+ metrics("time_elapsed_scanning_until_data").value > 0,
+ s"[$scanMode] time_elapsed_scanning_until_data should be > 0")
+ }
+ }
}
- }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
