This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new bf28a83958 Use more efficient variants of URLEncoder::encode and
URLDecoder::decode (#13030)
bf28a83958 is described below
commit bf28a83958e069f868e794f4a5271328d5a5f525
Author: Yash Mayya <yash.ma...@gmail.com>
AuthorDate: Wed May 1 07:35:34 2024 +0530
Use more efficient variants of URLEncoder::encode and URLDecoder::decode
(#13030)
---
.../common/function/scalar/StringFunctions.java | 15 +++++--------
.../org/apache/pinot/common/utils/URIUtils.java | 15 +++----------
.../generation/SegmentGenerationUtilsTest.java | 8 +++----
.../pinot/integration/tests/ClusterTest.java | 8 +++----
.../tests/HybridClusterIntegrationTest.java | 6 ++---
.../batch/hadoop/HadoopSegmentCreationMapper.java | 6 ++---
.../spark/SparkSegmentGenerationJobRunner.java | 6 ++---
.../spark3/SparkSegmentGenerationJobRunner.java | 6 ++---
.../standalone/SegmentGenerationJobRunner.java | 4 ++--
.../pinot/plugin/filesystem/AzurePinotFSUtil.java | 7 +++---
.../filesystem/test/AzurePinotFSUtilTest.java | 6 +++--
.../apache/pinot/plugin/filesystem/S3PinotFS.java | 15 ++-----------
.../parquet/ParquetNativeRecordReaderFullTest.java | 4 +++-
.../segment/local/utils/SegmentPushUtils.java | 4 ++--
.../pinot/server/api/resources/TablesResource.java | 20 +++--------------
.../apache/pinot/spi/filesystem/LocalPinotFS.java | 8 ++-----
.../utils/builder/ControllerRequestURLBuilder.java | 26 +++++++---------------
17 files changed, 56 insertions(+), 108 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/StringFunctions.java
b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/StringFunctions.java
index 31baeb5d2d..21c086ffb7 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/StringFunctions.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/StringFunctions.java
@@ -21,8 +21,6 @@ package org.apache.pinot.common.function.scalar;
import it.unimi.dsi.fastutil.objects.ObjectLinkedOpenHashSet;
import it.unimi.dsi.fastutil.objects.ObjectSet;
import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
-import java.net.URLEncoder;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.text.Normalizer;
@@ -33,6 +31,7 @@ import java.util.regex.Pattern;
import javax.annotation.Nullable;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.common.utils.RegexpPatternConverterUtils;
+import org.apache.pinot.common.utils.URIUtils;
import org.apache.pinot.spi.annotations.ScalarFunction;
import org.apache.pinot.spi.utils.JsonUtils;
@@ -804,24 +803,20 @@ public class StringFunctions {
*
* @param input plaintext string
* @return url encoded string
- * @throws UnsupportedEncodingException
*/
@ScalarFunction
- public static String encodeUrl(String input)
- throws UnsupportedEncodingException {
- return URLEncoder.encode(input, StandardCharsets.UTF_8.toString());
+ public static String encodeUrl(String input) {
+ return URIUtils.encode(input);
}
/**
*
* @param input url encoded string
* @return plaintext string
- * @throws UnsupportedEncodingException
*/
@ScalarFunction
- public static String decodeUrl(String input)
- throws UnsupportedEncodingException {
- return URLDecoder.decode(input, StandardCharsets.UTF_8.toString());
+ public static String decodeUrl(String input) {
+ return URIUtils.decode(input);
}
/**
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/URIUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/URIUtils.java
index 5aff60b07f..a857981f2a 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/URIUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/URIUtils.java
@@ -23,6 +23,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.StringJoiner;
import org.apache.http.client.utils.URIBuilder;
@@ -91,21 +92,11 @@ public class URIUtils {
}
public static String encode(String string) {
- try {
- return URLEncoder.encode(string, "UTF-8");
- } catch (Exception e) {
- // Should never happen
- throw new RuntimeException(e);
- }
+ return URLEncoder.encode(string, StandardCharsets.UTF_8);
}
public static String decode(String string) {
- try {
- return URLDecoder.decode(string, "UTF-8");
- } catch (Exception e) {
- // Should never happen
- throw new RuntimeException(e);
- }
+ return URLDecoder.decode(string, StandardCharsets.UTF_8);
}
/**
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/segment/generation/SegmentGenerationUtilsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/segment/generation/SegmentGenerationUtilsTest.java
index 983d8ededd..d33c046207 100644
---
a/pinot-common/src/test/java/org/apache/pinot/common/segment/generation/SegmentGenerationUtilsTest.java
+++
b/pinot-common/src/test/java/org/apache/pinot/common/segment/generation/SegmentGenerationUtilsTest.java
@@ -22,13 +22,12 @@ package org.apache.pinot.common.segment.generation;
import com.google.common.collect.Lists;
import java.io.File;
import java.io.IOException;
-import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
-import java.net.URLEncoder;
import java.nio.file.Files;
import java.util.List;
import org.apache.commons.io.FileUtils;
+import org.apache.pinot.common.utils.URIUtils;
import org.apache.pinot.spi.filesystem.PinotFS;
import org.apache.pinot.spi.filesystem.PinotFSFactory;
import org.testng.Assert;
@@ -73,7 +72,7 @@ public class SegmentGenerationUtilsTest {
// Invalid segment tar name with space
@Test
public void testInvalidRelativeURIs()
- throws URISyntaxException, UnsupportedEncodingException {
+ throws URISyntaxException {
URI inputDirURI = new URI("hdfs://namenode1:9999/path/to/");
URI inputFileURI = new URI("hdfs://namenode1:9999/path/to/subdir/file");
URI outputDirURI = new URI("hdfs://namenode2/output/dir/");
@@ -85,8 +84,7 @@ public class SegmentGenerationUtilsTest {
Assert.assertTrue(e instanceof URISyntaxException);
}
URI outputSegmentTarURI =
SegmentGenerationUtils.getRelativeOutputPath(inputDirURI, inputFileURI,
outputDirURI)
- .resolve(new URI(
-
URLEncoder.encode("table_OFFLINE_2021-02-01_09:39:00.000_2021-02-01_11:59:00.000_2.tar.gz",
"UTF-8")));
+ .resolve(new
URI(URIUtils.encode("table_OFFLINE_2021-02-01_09:39:00.000_2021-02-01_11:59:00.000_2.tar.gz")));
Assert.assertEquals(outputSegmentTarURI.toString(),
"hdfs://namenode2/output/dir/subdir/table_OFFLINE_2021-02-01_09%3A39%3A00.000_2021-02-01_11%3A59%3A00.000_2"
+ ".tar.gz");
diff --git
a/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterTest.java
b/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterTest.java
index 278f378b1a..49edf86264 100644
---
a/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterTest.java
+++
b/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterTest.java
@@ -25,8 +25,6 @@ import com.google.common.collect.ImmutableMap;
import java.io.File;
import java.io.IOException;
import java.net.URI;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -56,6 +54,7 @@ import org.apache.pinot.broker.broker.helix.BaseBrokerStarter;
import org.apache.pinot.broker.broker.helix.HelixBrokerStarter;
import org.apache.pinot.common.exception.HttpErrorStatusException;
import org.apache.pinot.common.utils.FileUploadDownloadClient;
+import org.apache.pinot.common.utils.URIUtils;
import org.apache.pinot.common.utils.http.HttpClient;
import org.apache.pinot.controller.helix.ControllerTest;
import org.apache.pinot.minion.BaseMinionStarter;
@@ -456,8 +455,9 @@ public abstract class ClusterTest extends ControllerTest {
FileUploadDownloadClient fileUploadDownloadClient, File segmentTarFile)
throws IOException, HttpErrorStatusException {
List<Header> headers = ImmutableList.of(new
BasicHeader(FileUploadDownloadClient.CustomHeaders.DOWNLOAD_URI,
- "file://" + segmentTarFile.getParentFile().getAbsolutePath() + "/" +
URLEncoder.encode(segmentTarFile.getName(),
- StandardCharsets.UTF_8.toString())), new
BasicHeader(FileUploadDownloadClient.CustomHeaders.UPLOAD_TYPE,
+ "file://" + segmentTarFile.getParentFile().getAbsolutePath() + "/"
+ + URIUtils.encode(segmentTarFile.getName())),
+ new BasicHeader(FileUploadDownloadClient.CustomHeaders.UPLOAD_TYPE,
FileUploadDownloadClient.FileUploadType.METADATA.toString()));
// Add table name and table type as request parameters
NameValuePair tableNameValuePair =
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTest.java
index 73d0dcb9bd..f66c194d85 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTest.java
@@ -20,7 +20,6 @@ package org.apache.pinot.integration.tests;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.File;
-import java.net.URLEncoder;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -28,6 +27,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.pinot.broker.broker.helix.BaseBrokerStarter;
+import org.apache.pinot.common.utils.URIUtils;
import org.apache.pinot.common.utils.config.TagNameUtils;
import org.apache.pinot.controller.ControllerConf;
import org.apache.pinot.spi.config.table.TableConfig;
@@ -269,9 +269,9 @@ public class HybridClusterIntegrationTest extends
BaseClusterIntegrationTestSet
String offlineTableName =
TableNameBuilder.OFFLINE.tableNameWithType(tableName);
String realtimeTableName =
TableNameBuilder.REALTIME.tableNameWithType(tableName);
String encodedSQL;
- encodedSQL = URLEncoder.encode("select * from " + realtimeTableName,
"UTF-8");
+ encodedSQL = URIUtils.encode("select * from " + realtimeTableName);
Assert.assertNotNull(getDebugInfo("debug/routingTable/sql?query=" +
encodedSQL));
- encodedSQL = URLEncoder.encode("select * from " + offlineTableName,
"UTF-8");
+ encodedSQL = URIUtils.encode("select * from " + offlineTableName);
Assert.assertNotNull(getDebugInfo("debug/routingTable/sql?query=" +
encodedSQL));
}
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/java/org/apache/pinot/plugin/ingestion/batch/hadoop/HadoopSegmentCreationMapper.java
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/java/org/apache/pinot/plugin/ingestion/batch/hadoop/HadoopSegmentCreationMapper.java
index 7782764340..3efad6b05d 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/java/org/apache/pinot/plugin/ingestion/batch/hadoop/HadoopSegmentCreationMapper.java
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/java/org/apache/pinot/plugin/ingestion/batch/hadoop/HadoopSegmentCreationMapper.java
@@ -22,7 +22,6 @@ import com.google.common.base.Preconditions;
import java.io.File;
import java.io.IOException;
import java.net.URI;
-import java.net.URLEncoder;
import java.nio.file.Files;
import java.util.List;
import java.util.UUID;
@@ -34,6 +33,7 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.pinot.common.segment.generation.SegmentGenerationUtils;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
+import org.apache.pinot.common.utils.URIUtils;
import
org.apache.pinot.plugin.ingestion.batch.common.SegmentGenerationJobUtils;
import
org.apache.pinot.plugin.ingestion.batch.common.SegmentGenerationTaskRunner;
import org.apache.pinot.spi.env.PinotConfiguration;
@@ -175,7 +175,7 @@ public class HadoopSegmentCreationMapper extends
Mapper<LongWritable, Text, Long
// Tar segment directory to compress file
File localSegmentDir = new File(localOutputTempDir, segmentName);
- String segmentTarFileName = URLEncoder.encode(segmentName +
Constants.TAR_GZ_FILE_EXT, "UTF-8");
+ String segmentTarFileName = URIUtils.encode(segmentName +
Constants.TAR_GZ_FILE_EXT);
File localSegmentTarFile = new File(localOutputTempDir,
segmentTarFileName);
LOGGER.info("Tarring segment from: {} to: {}", localSegmentDir,
localSegmentTarFile);
TarGzCompressionUtils.createTarGzFile(localSegmentDir,
localSegmentTarFile);
@@ -190,7 +190,7 @@ public class HadoopSegmentCreationMapper extends
Mapper<LongWritable, Text, Long
_spec.isOverwriteOutput());
// Create and upload segment metadata tar file
- String metadataTarFileName = URLEncoder.encode(segmentName +
Constants.METADATA_TAR_GZ_FILE_EXT, "UTF-8");
+ String metadataTarFileName = URIUtils.encode(segmentName +
Constants.METADATA_TAR_GZ_FILE_EXT);
URI outputMetadataTarURI =
relativeOutputPath.resolve(metadataTarFileName);
if (outputDirFS.exists(outputMetadataTarURI) &&
(_spec.isOverwriteOutput() || !_spec.isCreateMetadataTarGz())) {
LOGGER.info("Deleting existing metadata tar gz file: {}",
outputMetadataTarURI);
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentGenerationJobRunner.java
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentGenerationJobRunner.java
index 204884ab8d..6ae7ff97ab 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentGenerationJobRunner.java
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentGenerationJobRunner.java
@@ -22,7 +22,6 @@ import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.net.URI;
-import java.net.URLEncoder;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
@@ -34,6 +33,7 @@ import java.util.UUID;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.segment.generation.SegmentGenerationUtils;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
+import org.apache.pinot.common.utils.URIUtils;
import
org.apache.pinot.plugin.ingestion.batch.common.SegmentGenerationJobUtils;
import
org.apache.pinot.plugin.ingestion.batch.common.SegmentGenerationTaskRunner;
import org.apache.pinot.spi.env.PinotConfiguration;
@@ -283,7 +283,7 @@ public class SparkSegmentGenerationJobRunner implements
IngestionJobRunner, Seri
// Tar segment directory to compress file
File localSegmentDir = new File(localOutputTempDir, segmentName);
- String segmentTarFileName = URLEncoder.encode(segmentName +
Constants.TAR_GZ_FILE_EXT, "UTF-8");
+ String segmentTarFileName = URIUtils.encode(segmentName +
Constants.TAR_GZ_FILE_EXT);
File localSegmentTarFile = new File(localOutputTempDir,
segmentTarFileName);
LOGGER.info("Tarring segment from: {} to: {}", localSegmentDir,
localSegmentTarFile);
TarGzCompressionUtils.createTarGzFile(localSegmentDir,
localSegmentTarFile);
@@ -299,7 +299,7 @@ public class SparkSegmentGenerationJobRunner implements
IngestionJobRunner, Seri
_spec.isOverwriteOutput());
// Create and upload segment metadata tar file
- String metadataTarFileName = URLEncoder.encode(segmentName +
Constants.METADATA_TAR_GZ_FILE_EXT, "UTF-8");
+ String metadataTarFileName = URIUtils.encode(segmentName +
Constants.METADATA_TAR_GZ_FILE_EXT);
URI outputMetadataTarURI =
relativeOutputPath.resolve(metadataTarFileName);
if (finalOutputDirFS.exists(outputMetadataTarURI) &&
(_spec.isOverwriteOutput()
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentGenerationJobRunner.java
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentGenerationJobRunner.java
index d595da66b5..ef1f6cea5d 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentGenerationJobRunner.java
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentGenerationJobRunner.java
@@ -22,7 +22,6 @@ import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.net.URI;
-import java.net.URLEncoder;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
@@ -34,6 +33,7 @@ import java.util.UUID;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.segment.generation.SegmentGenerationUtils;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
+import org.apache.pinot.common.utils.URIUtils;
import
org.apache.pinot.plugin.ingestion.batch.common.SegmentGenerationJobUtils;
import
org.apache.pinot.plugin.ingestion.batch.common.SegmentGenerationTaskRunner;
import org.apache.pinot.spi.env.PinotConfiguration;
@@ -281,7 +281,7 @@ public class SparkSegmentGenerationJobRunner implements
IngestionJobRunner, Seri
// Tar segment directory to compress file
File localSegmentDir = new File(localOutputTempDir, segmentName);
- String segmentTarFileName = URLEncoder.encode(segmentName +
Constants.TAR_GZ_FILE_EXT, "UTF-8");
+ String segmentTarFileName = URIUtils.encode(segmentName +
Constants.TAR_GZ_FILE_EXT);
File localSegmentTarFile = new File(localOutputTempDir,
segmentTarFileName);
LOGGER.info("Tarring segment from: {} to: {}", localSegmentDir,
localSegmentTarFile);
TarGzCompressionUtils.createTarGzFile(localSegmentDir,
localSegmentTarFile);
@@ -297,7 +297,7 @@ public class SparkSegmentGenerationJobRunner implements
IngestionJobRunner, Seri
_spec.isOverwriteOutput());
// Create and upload segment metadata tar file
- String metadataTarFileName = URLEncoder.encode(segmentName +
Constants.METADATA_TAR_GZ_FILE_EXT, "UTF-8");
+ String metadataTarFileName = URIUtils.encode(segmentName +
Constants.METADATA_TAR_GZ_FILE_EXT);
URI outputMetadataTarURI =
relativeOutputPath.resolve(metadataTarFileName);
if (finalOutputDirFS.exists(outputMetadataTarURI) &&
(_spec.isOverwriteOutput()
|| !_spec.isCreateMetadataTarGz())) {
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/java/org/apache/pinot/plugin/ingestion/batch/standalone/SegmentGenerationJobRunner.java
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/java/org/apache/pinot/plugin/ingestion/batch/standalone/SegmentGenerationJobRunner.java
index 9743166077..a85247fc3d 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/java/org/apache/pinot/plugin/ingestion/batch/standalone/SegmentGenerationJobRunner.java
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/java/org/apache/pinot/plugin/ingestion/batch/standalone/SegmentGenerationJobRunner.java
@@ -22,7 +22,6 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
-import java.net.URLEncoder;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
@@ -37,6 +36,7 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.segment.generation.SegmentGenerationUtils;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
+import org.apache.pinot.common.utils.URIUtils;
import
org.apache.pinot.plugin.ingestion.batch.common.SegmentGenerationJobUtils;
import
org.apache.pinot.plugin.ingestion.batch.common.SegmentGenerationTaskRunner;
import org.apache.pinot.segment.local.utils.ConsistentDataPushUtils;
@@ -265,7 +265,7 @@ public class SegmentGenerationJobRunner implements
IngestionJobRunner {
String segmentName = taskRunner.run();
// Tar segment directory to compress file
localSegmentDir = new File(localOutputTempDir, segmentName);
- String segmentTarFileName = URLEncoder.encode(segmentName +
Constants.TAR_GZ_FILE_EXT, "UTF-8");
+ String segmentTarFileName = URIUtils.encode(segmentName +
Constants.TAR_GZ_FILE_EXT);
localSegmentTarFile = new File(localOutputTempDir, segmentTarFileName);
LOGGER.info("Tarring segment from: {} to: {}", localSegmentDir,
localSegmentTarFile);
TarGzCompressionUtils.createTarGzFile(localSegmentDir,
localSegmentTarFile);
diff --git
a/pinot-plugins/pinot-file-system/pinot-adls/src/main/java/org/apache/pinot/plugin/filesystem/AzurePinotFSUtil.java
b/pinot-plugins/pinot-file-system/pinot-adls/src/main/java/org/apache/pinot/plugin/filesystem/AzurePinotFSUtil.java
index 282b4346d6..c9665e7093 100644
---
a/pinot-plugins/pinot-file-system/pinot-adls/src/main/java/org/apache/pinot/plugin/filesystem/AzurePinotFSUtil.java
+++
b/pinot-plugins/pinot-file-system/pinot-adls/src/main/java/org/apache/pinot/plugin/filesystem/AzurePinotFSUtil.java
@@ -23,6 +23,7 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
/**
@@ -41,12 +42,10 @@ public class AzurePinotFSUtil {
*
* @param uri a uri path
* @return path in Azure Data Lake Gen2 format
- * @throws IOException
*/
- public static String convertUriToAzureStylePath(URI uri)
- throws IOException {
+ public static String convertUriToAzureStylePath(URI uri) {
// Pinot side code uses `URLEncoder` when building uri
- String path = URLDecoder.decode(uri.getRawPath(), "UTF-8");
+ String path = URLDecoder.decode(uri.getRawPath(), StandardCharsets.UTF_8);
if (path.startsWith(DIRECTORY_DELIMITER)) {
path = path.substring(1);
}
diff --git
a/pinot-plugins/pinot-file-system/pinot-adls/src/test/java/org/apache/pinot/plugin/filesystem/test/AzurePinotFSUtilTest.java
b/pinot-plugins/pinot-file-system/pinot-adls/src/test/java/org/apache/pinot/plugin/filesystem/test/AzurePinotFSUtilTest.java
index 80e68a6599..057a481470 100644
---
a/pinot-plugins/pinot-file-system/pinot-adls/src/test/java/org/apache/pinot/plugin/filesystem/test/AzurePinotFSUtilTest.java
+++
b/pinot-plugins/pinot-file-system/pinot-adls/src/test/java/org/apache/pinot/plugin/filesystem/test/AzurePinotFSUtilTest.java
@@ -23,6 +23,7 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import org.apache.pinot.plugin.filesystem.AzurePinotFSUtil;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -67,11 +68,12 @@ public class AzurePinotFSUtilTest {
throws Exception {
// "/encode(dir)/encode(segment)"
String expectedPath = String.join(File.separator, tableName, segmentName);
- URI uri = createUri(URLEncoder.encode(tableName, "UTF-8"),
URLEncoder.encode(segmentName, "UTF-8"));
+ URI uri = createUri(URLEncoder.encode(tableName, StandardCharsets.UTF_8),
URLEncoder.encode(segmentName,
+ StandardCharsets.UTF_8));
checkUri(uri, expectedPath, urlEncoded);
// "/encode(dir/segment)"
- uri = createUri(URLEncoder.encode(String.join(File.separator, tableName,
segmentName), "UTF-8"));
+ uri = createUri(URLEncoder.encode(String.join(File.separator, tableName,
segmentName), StandardCharsets.UTF_8));
checkUri(uri, expectedPath, urlEncoded);
// "/encode(dir/segment)"
diff --git
a/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
b/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
index e3a7daa55e..4fc84f3541 100644
---
a/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
+++
b/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
@@ -26,7 +26,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
@@ -325,12 +324,7 @@ public class S3PinotFS extends BasePinotFS {
private boolean copyFile(URI srcUri, URI dstUri)
throws IOException {
try {
- String encodedUrl = null;
- try {
- encodedUrl = URLEncoder.encode(srcUri.getHost() + srcUri.getPath(),
StandardCharsets.UTF_8.toString());
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- }
+ String encodedUrl = URLEncoder.encode(srcUri.getHost() +
srcUri.getPath(), StandardCharsets.UTF_8);
String dstPath = sanitizePath(dstUri.getPath());
CopyObjectRequest copyReq = generateCopyObjectRequest(encodedUrl,
dstUri, dstPath, null);
@@ -674,12 +668,7 @@ public class S3PinotFS extends BasePinotFS {
throws IOException {
try {
HeadObjectResponse s3ObjectMetadata = getS3ObjectMetadata(uri);
- String encodedUrl = null;
- try {
- encodedUrl = URLEncoder.encode(uri.getHost() + uri.getPath(),
StandardCharsets.UTF_8.toString());
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- }
+ String encodedUrl = URLEncoder.encode(uri.getHost() + uri.getPath(),
StandardCharsets.UTF_8);
String path = sanitizePath(uri.getPath());
CopyObjectRequest request = generateCopyObjectRequest(encodedUrl, uri,
path,
diff --git
a/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordReaderFullTest.java
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordReaderFullTest.java
index f0438a0508..6e84b19c9a 100644
---
a/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordReaderFullTest.java
+++
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordReaderFullTest.java
@@ -20,6 +20,7 @@ package org.apache.pinot.plugin.inputformat.parquet;
import java.io.File;
import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.io.FileUtils;
import org.testng.annotations.Test;
@@ -125,7 +126,8 @@ public class ParquetNativeRecordReaderFullTest {
protected void testParquetFile(String filePath)
throws Exception {
- File dataFile = new
File(URLDecoder.decode(getClass().getClassLoader().getResource(filePath).getFile(),
"UTF-8"));
+ File dataFile = new
File(URLDecoder.decode(getClass().getClassLoader().getResource(filePath).getFile(),
+ StandardCharsets.UTF_8));
ParquetNativeRecordReader recordReader = new ParquetNativeRecordReader();
recordReader.init(dataFile, null, null);
while (recordReader.hasNext()) {
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java
index 0a6be1905b..6ca93f2491 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java
@@ -24,7 +24,6 @@ import java.io.InputStream;
import java.io.Serializable;
import java.net.URI;
import java.net.URISyntaxException;
-import java.net.URLDecoder;
import java.nio.file.FileSystems;
import java.nio.file.PathMatcher;
import java.nio.file.Paths;
@@ -44,6 +43,7 @@ import
org.apache.pinot.common.exception.HttpErrorStatusException;
import org.apache.pinot.common.utils.FileUploadDownloadClient;
import org.apache.pinot.common.utils.SimpleHttpResponse;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
+import org.apache.pinot.common.utils.URIUtils;
import org.apache.pinot.common.utils.http.HttpClient;
import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.name.SegmentNameUtils;
@@ -407,7 +407,7 @@ public class SegmentPushUtils implements Serializable {
try {
if (fileSystem instanceof LocalPinotFS) {
// For local file system, we don't need to copy the tar file.
- tarFile = new File(URLDecoder.decode(tarFileURI.getRawPath(),
"UTF-8"));
+ tarFile = new File(URIUtils.decode(tarFileURI.getRawPath()));
} else {
// For other file systems, we need to download the file to local file
system
fileSystem.copyToLocalFile(tarFileURI, tarFile);
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
index 1baa52ffa4..1c35dc21ec 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
@@ -29,9 +29,7 @@ import io.swagger.annotations.Authorization;
import io.swagger.annotations.SecurityDefinition;
import io.swagger.annotations.SwaggerDefinition;
import java.io.File;
-import java.io.UnsupportedEncodingException;
import java.net.URI;
-import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
@@ -207,11 +205,7 @@ public class TablesResource {
List<String> decodedColumns = new ArrayList<>(columns.size());
for (String column : columns) {
- try {
- decodedColumns.add(URLDecoder.decode(column,
StandardCharsets.UTF_8.name()));
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e.getCause());
- }
+ decodedColumns.add(URIUtils.decode(column));
}
boolean allColumns = false;
@@ -380,19 +374,11 @@ public class TablesResource {
List<String> columns, @Context HttpHeaders headers) {
tableName = DatabaseUtils.translateTableName(tableName, headers);
for (int i = 0; i < columns.size(); i++) {
- try {
- columns.set(i, URLDecoder.decode(columns.get(i),
StandardCharsets.UTF_8.name()));
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e.getCause());
- }
+ columns.set(i, URIUtils.decode(columns.get(i)));
}
TableDataManager tableDataManager =
ServerResourceUtils.checkGetTableDataManager(_serverInstance, tableName);
- try {
- segmentName = URLDecoder.decode(segmentName,
StandardCharsets.UTF_8.name());
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e.getCause());
- }
+ segmentName = URIUtils.decode(segmentName);
SegmentDataManager segmentDataManager =
tableDataManager.acquireSegment(segmentName);
if (segmentDataManager == null) {
throw new WebApplicationException(String.format("Table %s segments %s
does not exist", tableName, segmentName),
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/LocalPinotFS.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/LocalPinotFS.java
index 7fd8ca5906..b82490b5e0 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/LocalPinotFS.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/LocalPinotFS.java
@@ -23,9 +23,9 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -188,11 +188,7 @@ public class LocalPinotFS extends BasePinotFS {
private static File toFile(URI uri) {
// NOTE: Do not use new File(uri) because scheme might not exist and it
does not decode '+' to ' '
// Do not use uri.getPath() because it does not decode '+' to ' '
- try {
- return new File(URLDecoder.decode(uri.getRawPath(), "UTF-8"));
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- }
+ return new File(URLDecoder.decode(uri.getRawPath(),
StandardCharsets.UTF_8));
}
private static void copy(File srcFile, File dstFile, boolean recursive)
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java
index 3b22e04941..f4133fee59 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.spi.utils.builder;
-import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.List;
@@ -488,31 +487,27 @@ public class ControllerRequestURLBuilder {
return url;
}
- public String forIngestFromFile(String tableNameWithType, String
batchConfigMapStr)
- throws UnsupportedEncodingException {
+ public String forIngestFromFile(String tableNameWithType, String
batchConfigMapStr) {
return String.format("%s?tableNameWithType=%s&batchConfigMapStr=%s",
StringUtil.join("/", _baseUrl, "ingestFromFile"), tableNameWithType,
- URLEncoder.encode(batchConfigMapStr,
StandardCharsets.UTF_8.toString()));
+ URLEncoder.encode(batchConfigMapStr, StandardCharsets.UTF_8));
}
- public String forIngestFromFile(String tableNameWithType, Map<String,
String> batchConfigMap)
- throws UnsupportedEncodingException {
+ public String forIngestFromFile(String tableNameWithType, Map<String,
String> batchConfigMap) {
String batchConfigMapStr =
batchConfigMap.entrySet().stream().map(e ->
String.format("\"%s\":\"%s\"", e.getKey(), e.getValue()))
.collect(Collectors.joining(",", "{", "}"));
return forIngestFromFile(tableNameWithType, batchConfigMapStr);
}
- public String forIngestFromURI(String tableNameWithType, String
batchConfigMapStr, String sourceURIStr)
- throws UnsupportedEncodingException {
+ public String forIngestFromURI(String tableNameWithType, String
batchConfigMapStr, String sourceURIStr) {
return
String.format("%s?tableNameWithType=%s&batchConfigMapStr=%s&sourceURIStr=%s",
StringUtil.join("/", _baseUrl, "ingestFromURI"), tableNameWithType,
- URLEncoder.encode(batchConfigMapStr,
StandardCharsets.UTF_8.toString()),
- URLEncoder.encode(sourceURIStr, StandardCharsets.UTF_8.toString()));
+ URLEncoder.encode(batchConfigMapStr, StandardCharsets.UTF_8),
+ URLEncoder.encode(sourceURIStr, StandardCharsets.UTF_8));
}
- public String forIngestFromURI(String tableNameWithType, Map<String, String>
batchConfigMap, String sourceURIStr)
- throws UnsupportedEncodingException {
+ public String forIngestFromURI(String tableNameWithType, Map<String, String>
batchConfigMap, String sourceURIStr) {
String batchConfigMapStr =
batchConfigMap.entrySet().stream().map(e ->
String.format("\"%s\":\"%s\"", e.getKey(), e.getValue()))
.collect(Collectors.joining(",", "{", "}"));
@@ -573,12 +568,7 @@ public class ControllerRequestURLBuilder {
}
private static String encode(String s) {
- try {
- return URLEncoder.encode(s, "UTF-8");
- } catch (Exception e) {
- // Should never happen
- throw new RuntimeException(e);
- }
+ return URLEncoder.encode(s, StandardCharsets.UTF_8);
}
public String forSegmentUpload() {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
