This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 900f01f288 Make getConfigMapWithPrefix auto append tailing dot if
missing (#8522)
900f01f288 is described below
commit 900f01f288c8d13738b782992b0f29c55d5b1a34
Author: Xiang Fu <xiangfu.1...@gmail.com>
AuthorDate: Tue Apr 12 21:03:04 2022 -0700
Make getConfigMapWithPrefix auto append tailing dot if missing (#8522)
---
.../org/apache/pinot/controller/util/FileIngestionHelper.java | 2 +-
.../SegmentGenerationAndPushTaskExecutor.java | 2 +-
.../java/org/apache/pinot/spi/utils/IngestionConfigUtils.java | 11 +++++++----
.../org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java | 8 ++++++++
4 files changed, 17 insertions(+), 6 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionHelper.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionHelper.java
index a180d9cb55..92da871460 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionHelper.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionHelper.java
@@ -176,7 +176,7 @@ public class FileIngestionHelper {
if (!PinotFSFactory.isSchemeSupported(sourceFileURIScheme)) {
PinotFSFactory.register(sourceFileURIScheme,
batchConfigMap.get(BatchConfigProperties.INPUT_FS_CLASS),
IngestionConfigUtils.getInputFsProps(IngestionConfigUtils.getConfigMapWithPrefix(
- batchConfigMap, BatchConfigProperties.INPUT_FS_PROP_PREFIX +
IngestionConfigUtils.DOT_SEPARATOR)));
+ batchConfigMap, BatchConfigProperties.INPUT_FS_PROP_PREFIX)));
}
PinotFSFactory.create(sourceFileURIScheme).copyToLocalFile(sourceFileURI,
destFile);
}
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java
index 25ffea0977..445a37c90c 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java
@@ -306,7 +306,7 @@ public class SegmentGenerationAndPushTaskExecutor extends
BaseTaskExecutor {
SegmentNameGeneratorSpec segmentNameGeneratorSpec = new
SegmentNameGeneratorSpec();
segmentNameGeneratorSpec.setType(taskConfigs.get(BatchConfigProperties.SEGMENT_NAME_GENERATOR_TYPE));
segmentNameGeneratorSpec.setConfigs(IngestionConfigUtils.getConfigMapWithPrefix(taskConfigs,
- BatchConfigProperties.SEGMENT_NAME_GENERATOR_PROP_PREFIX +
IngestionConfigUtils.DOT_SEPARATOR));
+ BatchConfigProperties.SEGMENT_NAME_GENERATOR_PROP_PREFIX));
taskSpec.setSegmentNameGeneratorSpec(segmentNameGeneratorSpec);
taskSpec.setCustomProperty(BatchConfigProperties.INPUT_DATA_FILE_URI_KEY,
inputFileURI.toString());
return taskSpec;
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
index 3bd9e3a6ed..e89b28e831 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
@@ -117,7 +117,7 @@ public final class IngestionConfigUtils {
* Fetch the properties which belong to record reader, by removing the
identifier prefix
*/
public static Map<String, String> getRecordReaderProps(Map<String, String>
batchConfigMap) {
- return getConfigMapWithPrefix(batchConfigMap,
BatchConfigProperties.RECORD_READER_PROP_PREFIX + DOT_SEPARATOR);
+ return getConfigMapWithPrefix(batchConfigMap,
BatchConfigProperties.RECORD_READER_PROP_PREFIX);
}
/**
@@ -125,17 +125,17 @@ public final class IngestionConfigUtils {
*/
public static Map<String, String> getSegmentNameGeneratorProps(Map<String,
String> batchConfigMap) {
return getConfigMapWithPrefix(batchConfigMap,
- BatchConfigProperties.SEGMENT_NAME_GENERATOR_PROP_PREFIX +
DOT_SEPARATOR);
+ BatchConfigProperties.SEGMENT_NAME_GENERATOR_PROP_PREFIX);
}
public static PinotConfiguration getInputFsProps(Map<String, String>
batchConfigMap) {
return new PinotConfiguration(
- getPropsWithPrefix(batchConfigMap,
BatchConfigProperties.INPUT_FS_PROP_PREFIX + DOT_SEPARATOR));
+ getPropsWithPrefix(batchConfigMap,
BatchConfigProperties.INPUT_FS_PROP_PREFIX));
}
public static PinotConfiguration getOutputFsProps(Map<String, String>
batchConfigMap) {
return new PinotConfiguration(
- getPropsWithPrefix(batchConfigMap,
BatchConfigProperties.OUTPUT_FS_PROP_PREFIX + DOT_SEPARATOR));
+ getPropsWithPrefix(batchConfigMap,
BatchConfigProperties.OUTPUT_FS_PROP_PREFIX));
}
/**
@@ -160,6 +160,9 @@ public final class IngestionConfigUtils {
public static Map<String, String> getConfigMapWithPrefix(Map<String, String>
batchConfigMap, String prefix) {
Map<String, String> props = new HashMap<>();
+ if (!prefix.endsWith(DOT_SEPARATOR)) {
+ prefix = prefix + DOT_SEPARATOR;
+ }
for (String configKey : batchConfigMap.keySet()) {
if (configKey.startsWith(prefix)) {
String[] splits = configKey.split(prefix, 2);
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java
index bb6649c934..1121f87f90 100644
---
a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java
+++
b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.spi.utils;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import java.util.HashMap;
import java.util.Map;
@@ -146,4 +147,11 @@ public class IngestionConfigUtilsTest {
segmentsValidationAndRetentionConfig.setSegmentPushType(null);
Assert.assertNull(IngestionConfigUtils.getBatchSegmentIngestionType(tableConfig));
}
+
+ @Test
+ public void testGetConfigMapWithPrefix() {
+ Map<String, String> testMap = ImmutableMap.of("k1", "v1", "k1.k2", "v2",
"k1.k3", "v3", "k4", "v4");
+ Assert.assertEquals(2,
IngestionConfigUtils.getConfigMapWithPrefix(testMap, "k1").size());
+ Assert.assertEquals(2,
IngestionConfigUtils.getConfigMapWithPrefix(testMap, "k1.").size());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
