(doris) branch master updated: [pipelineX](file scan) Improve parallel tasks if ignore data distribution (#31328)

[yiguolei]

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 96d29e14429 [pipelineX](file scan) Improve parallel tasks if ignore 
data distribution (#31328)
96d29e14429 is described below

commit 96d29e14429ff5415152333368141c538757a5ab
Author: Gabriel <gabrielleeb...@gmail.com>
AuthorDate: Sat Feb 24 11:34:52 2024 +0800

    [pipelineX](file scan) Improve parallel tasks if ignore data distribution 
(#31328)
---
 .../java/org/apache/doris/datasource/FileQueryScanNode.java    | 10 ++++++++++
 .../org/apache/doris/datasource/tvf/source/TVFScanNode.java    |  5 +++++
 2 files changed, 15 insertions(+)

diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java 
b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
index 6c60692ea21..7afb04831ce 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
@@ -503,6 +503,16 @@ public abstract class FileQueryScanNode extends 
FileScanNode {
         return i;
     }
 
+    @Override
+    public int getNumInstances() {
+        if (ConnectContext.get() != null
+                && 
ConnectContext.get().getSessionVariable().getEnablePipelineXEngine()
+                && 
ConnectContext.get().getSessionVariable().isIgnoreStorageDataDistribution()) {
+            return 
ConnectContext.get().getSessionVariable().getParallelExecInstanceNum();
+        }
+        return scanRangeLocations.size();
+    }
+
     protected abstract TFileType getLocationType() throws UserException;
 
     protected abstract TFileType getLocationType(String location) throws 
UserException;
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
 
b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
index 83aa49f5ab0..dcd248a9782 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
@@ -143,4 +143,9 @@ public class TVFScanNode extends FileQueryScanNode {
         }
         return splits;
     }
+
+    @Override
+    public int getNumInstances() {
+        return scanRangeLocations.size();
+    }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org