This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 7200baefeb1 [fix](catalog) fix the that failed to check if input
format is splitable (#35029)
7200baefeb1 is described below
commit 7200baefeb177e9ae932cbae4bd99cc470dd31fa
Author: Mingyu Chen <morning...@163.com>
AuthorDate: Sat May 18 14:06:40 2024 +0800
[fix](catalog) fix the that failed to check if input format is splitable
(#35029)
Introduced from #33242
When we check supported inputformat in a Set<String>, we should use string,
not object
---
.../doris/datasource/hive/HiveMetaStoreCache.java | 44 +++++++++++-----------
.../apache/doris/external/hive/util/HiveUtil.java | 6 +--
2 files changed, 25 insertions(+), 25 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index f6391093569..061f948e5dd 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -348,21 +348,21 @@ public class HiveMetaStoreCache {
StorageDescriptor sd = partition.getSd();
ret.put(new PartitionCacheKey(dbName, tblName,
partition.getValues()),
new HivePartition(dbName, tblName, false,
- sd.getInputFormat(), sd.getLocation(),
partition.getValues()));
+ sd.getInputFormat(), sd.getLocation(),
partition.getValues()));
}
return ret;
}
// Get File Status by using FileSystem API.
- private FileCacheValue getFileCache(String location, InputFormat<?, ?>
inputFormat,
- JobConf jobConf,
- List<String> partitionValues,
- String bindBrokerName) throws
UserException {
+ private FileCacheValue getFileCache(String location, String inputFormat,
+ JobConf jobConf,
+ List<String> partitionValues,
+ String bindBrokerName) throws UserException {
FileCacheValue result = new FileCacheValue();
RemoteFileSystem fs =
Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
new
FileSystemCache.FileSystemCacheKey(LocationPath.getFSIdentity(
- location, bindBrokerName), jobConf, bindBrokerName));
- result.setSplittable(HiveUtil.isSplittable(fs, inputFormat, location,
jobConf));
+ location, bindBrokerName), jobConf, bindBrokerName));
+ result.setSplittable(HiveUtil.isSplittable(fs, inputFormat, location));
try {
// For Tez engine, it may generate subdirectoies for "union" query.
// So there may be files and directories in the table directory at
the same time. eg:
@@ -421,18 +421,18 @@ public class HiveMetaStoreCache {
FileInputFormat.setInputPaths(jobConf, finalLocation.get());
try {
FileCacheValue result;
- InputFormat<?, ?> inputFormat =
HiveUtil.getInputFormat(jobConf, key.inputFormat, false);
// TODO: This is a temp config, will remove it after the
HiveSplitter is stable.
if (key.useSelfSplitter) {
- result = getFileCache(finalLocation.get(), inputFormat,
jobConf,
- key.getPartitionValues(), key.bindBrokerName);
+ result = getFileCache(finalLocation.get(),
key.inputFormat, jobConf,
+ key.getPartitionValues(), key.bindBrokerName);
} else {
+ InputFormat<?, ?> inputFormat =
HiveUtil.getInputFormat(jobConf, key.inputFormat, false);
InputSplit[] splits;
String remoteUser =
jobConf.get(HdfsResource.HADOOP_USER_NAME);
if (!Strings.isNullOrEmpty(remoteUser)) {
UserGroupInformation ugi =
UserGroupInformation.createRemoteUser(remoteUser);
splits = ugi.doAs(
- (PrivilegedExceptionAction<InputSplit[]>) () ->
inputFormat.getSplits(jobConf, 0));
+ (PrivilegedExceptionAction<InputSplit[]>) ()
-> inputFormat.getSplits(jobConf, 0));
} else {
splits = inputFormat.getSplits(jobConf, 0 /* use hdfs
block size as default */);
}
@@ -715,7 +715,7 @@ public class HiveMetaStoreCache {
}
public void dropPartitionsCache(String dbName, String tblName,
List<String> partitionNames,
- boolean invalidPartitionCache) {
+ boolean invalidPartitionCache) {
PartitionValueCacheKey key = new PartitionValueCacheKey(dbName,
tblName, null);
HivePartitionValues partitionValues =
partitionValuesCache.getIfPresent(key);
if (partitionValues == null) {
@@ -839,17 +839,17 @@ public class HiveMetaStoreCache {
RemoteFileSystem fs =
Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
new FileSystemCache.FileSystemCacheKey(
LocationPath.getFSIdentity(location,
bindBrokerName),
- jobConf, bindBrokerName));
+ jobConf, bindBrokerName));
RemoteFiles locatedFiles = fs.listLocatedFiles(location,
true, false);
if (delta.isDeleteDelta()) {
List<String> deleteDeltaFileNames =
locatedFiles.files().stream().map(f -> f.getName()).filter(
name ->
name.startsWith(HIVE_TRANSACTIONAL_ORC_BUCKET_PREFIX))
- .collect(Collectors.toList());
+ .collect(Collectors.toList());
deleteDeltas.add(new DeleteDeltaInfo(location,
deleteDeltaFileNames));
continue;
}
locatedFiles.files().stream().filter(
- f ->
f.getName().startsWith(HIVE_TRANSACTIONAL_ORC_BUCKET_PREFIX))
+ f ->
f.getName().startsWith(HIVE_TRANSACTIONAL_ORC_BUCKET_PREFIX))
.forEach(fileCacheValue::addFile);
}
@@ -859,10 +859,10 @@ public class HiveMetaStoreCache {
RemoteFileSystem fs =
Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
new FileSystemCache.FileSystemCacheKey(
LocationPath.getFSIdentity(location,
bindBrokerName),
- jobConf, bindBrokerName));
+ jobConf, bindBrokerName));
RemoteFiles locatedFiles = fs.listLocatedFiles(location,
true, false);
locatedFiles.files().stream().filter(
- f ->
f.getName().startsWith(HIVE_TRANSACTIONAL_ORC_BUCKET_PREFIX))
+ f ->
f.getName().startsWith(HIVE_TRANSACTIONAL_ORC_BUCKET_PREFIX))
.forEach(fileCacheValue::addFile);
}
fileCacheValue.setAcidInfo(new AcidInfo(partition.getPath(),
deleteDeltas));
@@ -976,8 +976,8 @@ public class HiveMetaStoreCache {
}
public static FileCacheKey createDummyCacheKey(String dbName, String
tblName, String location,
- String inputFormat,
boolean useSelfSplitter,
- String bindBrokerName) {
+ String inputFormat, boolean useSelfSplitter,
+ String bindBrokerName) {
FileCacheKey fileCacheKey = new FileCacheKey(location,
inputFormat, null, bindBrokerName);
fileCacheKey.dummyKey = dbName + "." + tblName;
fileCacheKey.useSelfSplitter = useSelfSplitter;
@@ -996,7 +996,7 @@ public class HiveMetaStoreCache {
return dummyKey.equals(((FileCacheKey) obj).dummyKey);
}
return location.equals(((FileCacheKey) obj).location)
- && Objects.equals(partitionValues, ((FileCacheKey)
obj).partitionValues);
+ && Objects.equals(partitionValues, ((FileCacheKey)
obj).partitionValues);
}
@Override
@@ -1115,10 +1115,10 @@ public class HiveMetaStoreCache {
private Map<Long, List<UniqueId>> idToUniqueIdsMap;
private Map<Long, PartitionItem> idToPartitionItem;
private Map<Long, List<String>> partitionValuesMap;
- //multi pair
+ // multi pair
private Map<UniqueId, Range<PartitionKey>> uidToPartitionRange;
private Map<Range<PartitionKey>, UniqueId> rangeToId;
- //single pair
+ // single pair
private RangeMap<ColumnBound, UniqueId> singleColumnRangeMap;
private Map<UniqueId, Range<ColumnBound>> singleUidToColumnRangeMap;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java
index f0f013fa1f0..55243b9b148 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java
@@ -185,11 +185,11 @@ public final class HiveUtil {
}
}
- public static boolean isSplittable(RemoteFileSystem remoteFileSystem,
InputFormat<?, ?> inputFormat,
- String location, JobConf jobConf) throws UserException {
+ public static boolean isSplittable(RemoteFileSystem remoteFileSystem,
String inputFormat,
+ String location) throws UserException {
if (remoteFileSystem instanceof BrokerFileSystem) {
return ((BrokerFileSystem) remoteFileSystem)
- .isSplittable(location,
inputFormat.getClass().getCanonicalName());
+ .isSplittable(location, inputFormat);
}
return
HMSExternalTable.SUPPORTED_HIVE_FILE_FORMATS.contains(inputFormat);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
