morningman commented on code in PR #24070:
URL: https://github.com/apache/doris/pull/24070#discussion_r1320751623
##########
fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java:
##########
@@ -315,11 +299,77 @@ public void generateOutfileLogicalPlans(List<String>
nameParts)
generateExportJobExecutor();
}
- private LogicalPlan generateOneLogicalPlan(List<String> nameParts,
List<Long> tabletIds,
- List<NamedExpression> selectLists) {
+ private void generateOlapTableOutfile(List<String> qualifiedTableName)
throws UserException {
+ // build source columns
+ List<NamedExpression> selectLists = Lists.newArrayList();
+ if (exportColumns.isEmpty()) {
+ selectLists.add(new UnboundStar(ImmutableList.of()));
+ } else {
+ this.exportColumns.stream().forEach(col -> {
+ selectLists.add(new UnboundSlot(this.tableName.getTbl(), col));
+ });
+ }
+
+ // get all tablets
+ List<List<Long>> tabletsListPerParallel = splitTablets();
+
+ // Each Outfile clause responsible for
MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT tablets
+ for (List<Long> tabletsList : tabletsListPerParallel) {
+ List<StatementBase> logicalPlanAdapters = Lists.newArrayList();
+ for (int i = 0; i < tabletsList.size(); i +=
MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT) {
+ int end = i + MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT <
tabletsList.size()
+ ? i + MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT :
tabletsList.size();
+ List<Long> tabletIds = new ArrayList<>(tabletsList.subList(i,
end));
+
+ // generate LogicalPlan
+ LogicalPlan plan = generateOneLogicalPlan(qualifiedTableName,
tabletIds,
+ this.partitionNames, selectLists);
+ // generate LogicalPlanAdapter
+ StatementBase statementBase = generateLogicalPlanAdapter(plan);
+
+ logicalPlanAdapters.add(statementBase);
+ }
+ selectStmtListPerParallel.add(logicalPlanAdapters);
+ }
+ }
+
+ /**
+ * This method used to generate outfile sql for view table or external
table.
+ * @throws UserException
+ */
+ private void generateViewOrExternalTableOutfile(List<String>
qualifiedTableName) {
+ // Because there is no division of tablets in view and external table
+ // we set parallelism = 1;
+ this.parallelism = 1;
+ LOG.info("Because there is no division of tablets in view and external
table, we set parallelism = 1");
Review Comment:
```suggestion
LOG.debug("Because there is no division of tablets in view and
external table, we set parallelism = 1");
```
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java:
##########
@@ -156,40 +159,52 @@ private void checkPropertyKey(Map<String, String>
properties) throws AnalysisExc
}
// check partitions specified by user are belonged to the table.
- private void checkPartitions(Env env, TableName tblName) throws
AnalysisException, UserException {
+ private void checkPartitions(ConnectContext ctx, TableName tblName) throws
AnalysisException, UserException {
if (this.partitionsNames.isEmpty()) {
return;
}
+ CatalogIf catalog =
ctx.getEnv().getCatalogMgr().getCatalogOrAnalysisException(tblName.getCtl());
+ // As for external table, we do not support export PARTITION
+ if (!"internal".equals(catalog.getType())) {
Review Comment:
use `InternalCatalog.INTERNAL_CATALOG_NAME`
##########
fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java:
##########
@@ -264,39 +254,33 @@ public void generateOutfileStatement() throws
UserException {
generateExportJobExecutor();
}
- public void generateOutfileLogicalPlans(List<String> nameParts)
+ /**
+ * For an ExportJob:
+ * The ExportJob is divided into multiple 'ExportTaskExecutor'
+ * according to the 'parallelism' set by the user.
+ * The tablets which will be exported by this ExportJob are divided into
'parallelism' copies,
+ * and each ExportTaskExecutor is responsible for a list of tablets.
+ * The tablets responsible for an ExportTaskExecutor will be assigned to
multiple OutfileStmt
+ * according to the 'TABLETS_NUM_PER_OUTFILE_IN_EXPORT'.
+ *
+ * @throws UserException
+ */
+ public void generateOutfileLogicalPlans(List<String> qualifiedTableName)
throws UserException {
+ String catalogType =
Env.getCurrentEnv().getCatalogMgr().getCatalog(this.tableName.getCtl()).getType();
exportTable.readLock();
try {
- // build source columns
- List<NamedExpression> selectLists = Lists.newArrayList();
- if (exportColumns.isEmpty()) {
- selectLists.add(new UnboundStar(ImmutableList.of()));
- } else {
- this.exportColumns.stream().forEach(col -> {
- selectLists.add(new UnboundSlot(this.tableName.getTbl(),
col));
- });
- }
-
- // get all tablets
- List<List<Long>> tabletsListPerParallel = splitTablets();
-
- // Each Outfile clause responsible for
MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT tablets
- for (List<Long> tabletsList : tabletsListPerParallel) {
- List<StatementBase> logicalPlanAdapters = Lists.newArrayList();
- for (int i = 0; i < tabletsList.size(); i +=
MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT) {
- int end = i + MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT <
tabletsList.size()
- ? i + MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT :
tabletsList.size();
- List<Long> tabletIds = new
ArrayList<>(tabletsList.subList(i, end));
-
- // generate LogicalPlan
- LogicalPlan plan = generateOneLogicalPlan(nameParts,
tabletIds, selectLists);
- // generate LogicalPlanAdapter
- StatementBase statementBase =
generateLogicalPlanAdapter(plan);
-
- logicalPlanAdapters.add(statementBase);
+ if ("internal".equals(catalogType)) {
+ if (exportTable.getType() == TableType.VIEW) {
+ // view table
+ generateViewOrExternalTableOutfile(qualifiedTableName);
+ } else {
Review Comment:
Better use `else if (exportTable.getType() == TableType.OLAP)`,
and add `else` to throw exception for unsupported type.
Because we may add some other type of table later.
##########
fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java:
##########
@@ -264,39 +254,33 @@ public void generateOutfileStatement() throws
UserException {
generateExportJobExecutor();
}
- public void generateOutfileLogicalPlans(List<String> nameParts)
+ /**
+ * For an ExportJob:
+ * The ExportJob is divided into multiple 'ExportTaskExecutor'
+ * according to the 'parallelism' set by the user.
+ * The tablets which will be exported by this ExportJob are divided into
'parallelism' copies,
+ * and each ExportTaskExecutor is responsible for a list of tablets.
+ * The tablets responsible for an ExportTaskExecutor will be assigned to
multiple OutfileStmt
+ * according to the 'TABLETS_NUM_PER_OUTFILE_IN_EXPORT'.
+ *
+ * @throws UserException
+ */
+ public void generateOutfileLogicalPlans(List<String> qualifiedTableName)
throws UserException {
+ String catalogType =
Env.getCurrentEnv().getCatalogMgr().getCatalog(this.tableName.getCtl()).getType();
exportTable.readLock();
try {
- // build source columns
- List<NamedExpression> selectLists = Lists.newArrayList();
- if (exportColumns.isEmpty()) {
- selectLists.add(new UnboundStar(ImmutableList.of()));
- } else {
- this.exportColumns.stream().forEach(col -> {
- selectLists.add(new UnboundSlot(this.tableName.getTbl(),
col));
- });
- }
-
- // get all tablets
- List<List<Long>> tabletsListPerParallel = splitTablets();
-
- // Each Outfile clause responsible for
MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT tablets
- for (List<Long> tabletsList : tabletsListPerParallel) {
- List<StatementBase> logicalPlanAdapters = Lists.newArrayList();
- for (int i = 0; i < tabletsList.size(); i +=
MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT) {
- int end = i + MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT <
tabletsList.size()
- ? i + MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT :
tabletsList.size();
- List<Long> tabletIds = new
ArrayList<>(tabletsList.subList(i, end));
-
- // generate LogicalPlan
- LogicalPlan plan = generateOneLogicalPlan(nameParts,
tabletIds, selectLists);
- // generate LogicalPlanAdapter
- StatementBase statementBase =
generateLogicalPlanAdapter(plan);
-
- logicalPlanAdapters.add(statementBase);
+ if ("internal".equals(catalogType)) {
Review Comment:
Use `InternalCatalog.INTERNAL_CATALOG_NAME`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
