morningman commented on code in PR #23485: URL: https://github.com/apache/doris/pull/23485#discussion_r1311006943
########## fe/fe-core/src/main/java/org/apache/doris/analysis/BulkDesc.java: ########## @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.common.util.PrintableMap; +import org.apache.doris.datasource.property.S3ClientBEProperties; +import org.apache.doris.fs.PersistentFileSystem; +import org.apache.doris.thrift.TFileType; + +import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Map; + +// Broker descriptor +// +// Broker example: +// WITH S3/HDFS +// ( +// "username" = "user0", +// "password" = "password0" +// ) +public class BulkDesc implements Writable { + private static final Logger LOG = LogManager.getLogger(BulkDesc.class); + private String name; + protected BulkType type; Review Comment: Not a good name. Maybe `StorageType` ########## fe/fe-core/src/main/java/org/apache/doris/analysis/BulkDesc.java: ########## @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.common.util.PrintableMap; +import org.apache.doris.datasource.property.S3ClientBEProperties; +import org.apache.doris.fs.PersistentFileSystem; +import org.apache.doris.thrift.TFileType; + +import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Map; + +// Broker descriptor +// +// Broker example: +// WITH S3/HDFS +// ( +// "username" = "user0", +// "password" = "password0" +// ) +public class BulkDesc implements Writable { + private static final Logger LOG = LogManager.getLogger(BulkDesc.class); + private String name; + protected BulkType type; + protected Map<String, String> properties; + private boolean convertedToS3 = false; + + public enum BulkType { + BROKER, + S3, + HDFS, + LOCAL + } + // Only used for recovery + + private BulkDesc() { + this.properties = Maps.newHashMap(); + this.type = BulkType.BROKER; + } + // for empty broker desc + + public BulkDesc(String name) { + this.name = name; + this.properties = Maps.newHashMap(); + this.type = BulkType.LOCAL; + } + + public BulkDesc(String name, Map<String, String> properties) { + this.name = name; + this.properties = properties; + if (this.properties == null) { + this.properties = Maps.newHashMap(); + } + this.type = BulkType.BROKER; + this.properties.putAll(S3ClientBEProperties.getBeFSProperties(this.properties)); + } + + public BulkDesc(String name, BulkType type, Map<String, String> properties) { + this.name = name; + this.properties = properties; + if (this.properties == null) { + this.properties = Maps.newHashMap(); + } + this.type = type; + this.properties.putAll(S3ClientBEProperties.getBeFSProperties(this.properties)); + } + + + public TFileType getFileType() { + switch (type) { + case LOCAL: + return TFileType.FILE_LOCAL; + case S3: + return TFileType.FILE_S3; + case HDFS: + return TFileType.FILE_HDFS; + case BROKER: + default: + return TFileType.FILE_BROKER; + } + } + + public String getFileLocation(String location) { + return location; + } + + public BulkType getType() { + return type; + } + + @Override + public void write(DataOutput out) throws IOException { Review Comment: Use Gson ########## fe/fe-core/src/main/java/org/apache/doris/analysis/BulkDesc.java: ########## @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.common.util.PrintableMap; +import org.apache.doris.datasource.property.S3ClientBEProperties; +import org.apache.doris.fs.PersistentFileSystem; +import org.apache.doris.thrift.TFileType; + +import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Map; + +// Broker descriptor +// +// Broker example: +// WITH S3/HDFS +// ( +// "username" = "user0", +// "password" = "password0" +// ) +public class BulkDesc implements Writable { Review Comment: Not a good name, this class is used to describe storage connection info. Better rename it. ########## fe/fe-core/src/main/java/org/apache/doris/analysis/NereidsLoadStmt.java: ########## @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import com.google.common.base.Preconditions; + +public class NereidsLoadStmt extends DdlStmt { Review Comment: What is this class for? ########## fe/fe-core/src/main/java/org/apache/doris/analysis/BulkLoadDataDesc.java: ########## @@ -0,0 +1,314 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.load.loadv2.LoadTask; +import org.apache.doris.nereids.trees.expressions.Expression; + +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +// used to describe data info which is needed to import. +// +// data_desc: +// DATA INFILE ('file_path', ...) +// [NEGATIVE] +// INTO TABLE tbl_name +// [PARTITION (p1, p2)] +// [COLUMNS TERMINATED BY separator] +// [FORMAT AS format] +// [(tmp_col1, tmp_col2, col3, ...)] +// [COLUMNS FROM PATH AS (col1, ...)] +// [SET (k1=f1(xx), k2=f2(xxx))] +// [where_clause] +// +// DATA FROM TABLE external_hive_tbl_name +// [NEGATIVE] +// INTO TABLE tbl_name +// [PARTITION (p1, p2)] +// [SET (k1=f1(xx), k2=f2(xxx))] +// [where_clause] + +/** + * The transform of columns should be added after the keyword named COLUMNS. + * The transform after the keyword named SET is the old ways which only supports the hadoop function. + * It old way of transform will be removed gradually. It + */ +public class BulkLoadDataDesc { + private static final Logger LOG = LogManager.getLogger(BulkLoadDataDesc.class); + private final String tableName; + private String dbName; + private final PartitionNames partitionNames; + private final List<String> filePaths; + private final Separator columnSeparator; + private String fileFormat; + private final boolean isNegative; + // column names in the path + private final List<String> columnsFromPath; + // save column mapping in SET(xxx = xxx) clause + private final List<Expression> columnMappingList; + private final Expression precedingFilterExpr; + private final Expression whereExpr; + private LoadTask.MergeType mergeType; + private final String srcTableName; + // column names of source files + private List<String> fileFieldNames; + private Separator lineDelimiter; + private String sequenceCol; + + // Merged from fileFieldNames, columnsFromPath and columnMappingList + // ImportColumnDesc: column name to (expr or null) + private List<ImportColumnDesc> parsedColumnExprList = Lists.newArrayList(); + private final Expression deleteCondition; + private final Map<String, String> properties; + private boolean isMysqlLoad = false; + + public BulkLoadDataDesc(List<String> fullTableName, + PartitionNames partitionNames, + List<String> filePaths, + List<String> columns, + Separator columnSeparator, + Separator lineDelimiter, + String fileFormat, + List<String> columnsFromPath, + boolean isNegative, + List<Expression> columnMappingList, + Expression fileFilterExpr, + Expression whereExpr, + LoadTask.MergeType mergeType, + Expression deleteCondition, + String sequenceColName, + Map<String, String> properties) { + this.dbName = Objects.requireNonNull(fullTableName.get(1), "Database name should not null"); + this.tableName = Objects.requireNonNull(fullTableName.get(2), "Table name should not null"); + this.partitionNames = partitionNames; + this.filePaths = filePaths; + this.fileFieldNames = columns; + this.columnSeparator = columnSeparator; + this.lineDelimiter = lineDelimiter; + this.fileFormat = fileFormat; + this.columnsFromPath = columnsFromPath; + this.isNegative = isNegative; + this.columnMappingList = columnMappingList; + this.precedingFilterExpr = fileFilterExpr; + this.whereExpr = whereExpr; + this.mergeType = mergeType; + this.srcTableName = null; + this.deleteCondition = deleteCondition; + this.sequenceCol = sequenceColName; + this.properties = properties; + columnsNameToLowerCase(fileFieldNames); + columnsNameToLowerCase(columnsFromPath); + } + + // data from table external_hive_table + public BulkLoadDataDesc(List<String> fullTableName, Review Comment: No need to implement this now. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
