(paimon) branch master updated: [spark] Use the field index instead of the field id for TopN pushdown (#9566)

[lzljs3620320]

This is an automated email from the ASF dual-hosted git repository.

JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new ba5801ba29 [spark] Use the field index instead of the field id for 
TopN pushdown (#9566)
ba5801ba29 is described below

commit ba5801ba2967845017a6b9a4ef225082ba2f650d
Author: jackylee <qcsd2...@gmail.com>
AuthorDate: Thu Sep 3 16:26:27 2026 +0800

    [spark] Use the field index instead of the field id for TopN pushdown 
(#9566)
---
 .../apache/paimon/spark/PaimonScanBuilder.scala    |  2 +-
 .../paimon/spark/sql/PaimonPushDownTestBase.scala  | 22 ++++++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git 
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScanBuilder.scala
 
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScanBuilder.scala
index 4a19890f44..b32a07e37b 100644
--- 
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScanBuilder.scala
+++ 
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScanBuilder.scala
@@ -64,7 +64,7 @@ class PaimonScanBuilder(val table: InnerTable)
           }
 
           val field = rowType.getField(fieldName)
-          val ref = new FieldRef(field.id(), field.name(), field.`type`())
+          val ref = new FieldRef(rowType.getFieldIndex(fieldName), 
field.name(), field.`type`())
 
           val nullOrdering = order.nullOrdering() match {
             case expressions.NullOrdering.NULLS_LAST => NullOrdering.NULLS_LAST
diff --git 
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPushDownTestBase.scala
 
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPushDownTestBase.scala
index 1f29b0365e..38ec748709 100644
--- 
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPushDownTestBase.scala
+++ 
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPushDownTestBase.scala
@@ -683,6 +683,28 @@ abstract class PaimonPushDownTestBase extends 
PaimonSparkTestBase with AdaptiveS
     Assertions.assertTrue(qe1.optimizedPlan.containsPattern(LIMIT))
   }
 
+  test("Paimon pushDown: TopN for append-only tables after dropping a column") 
{
+    assume(gteqSpark3_3)
+    spark.sql("""
+                |CREATE TABLE T (pt INT, dropped INT, id INT, price BIGINT) 
PARTITIONED BY (pt)
+                |TBLPROPERTIES ('file-index.range-bitmap.columns'='id')
+                |""".stripMargin)
+    spark.sql("INSERT INTO T VALUES (1, 0, 10, 100L), (2, 0, 20, 200L), (3, 0, 
30, 300L)")
+    spark.sql("INSERT INTO T VALUES (4, 0, 40, 400L), (5, 0, 50, 500L)")
+    spark.sql("INSERT INTO T VALUES (6, 0, 60, 600L), (7, 0, 70, 700L)")
+
+    // dropping a column does not reassign the remaining field ids, so from 
here on the field id
+    // and the field index of id and price no longer agree
+    spark.sql("ALTER TABLE T DROP COLUMN dropped")
+
+    checkAnswer(
+      spark.sql("SELECT id FROM T ORDER BY id ASC LIMIT 5"),
+      Row(10) :: Row(20) :: Row(30) :: Row(40) :: Row(50) :: Nil)
+    checkAnswer(
+      spark.sql("SELECT price FROM T ORDER BY price DESC LIMIT 3"),
+      Row(700L) :: Row(600L) :: Row(500L) :: Nil)
+  }
+
   test("Paimon pushDown: multi TopN for append-only tables") {
     assume(gteqSpark3_3)
     spark.sql("""