[pinot] branch master updated: Fixing the metadata tar gz file path (#10149)

[xiangfu]

This is an automated email from the ASF dual-hosted git repository.

xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 112893bd6e Fixing the metadata tar gz file path (#10149)
112893bd6e is described below

commit 112893bd6ee1357d08b9f889535c64a4b654ea1a
Author: Xiang Fu <xiangfu.1...@gmail.com>
AuthorDate: Thu Jan 19 03:48:09 2023 -0800

    Fixing the metadata tar gz file path (#10149)
---
 .../segment/local/utils/SegmentPushUtils.java      | 20 +++++++++++++++--
 .../segment/local/utils/SegmentPushUtilsTest.java  | 25 +++++++++++++++++++---
 2 files changed, 40 insertions(+), 5 deletions(-)

diff --git 
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java
 
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java
index b2ed31821c..dabee5dcb6 100644
--- 
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java
+++ 
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java
@@ -290,8 +290,9 @@ public class SegmentPushUtils implements Serializable {
       // Check if there is a segment metadata tar gz file named 
`segmentName.metadata.tar.gz`, already in the remote
       // directory. This is to avoid generating a new segment metadata tar gz 
file every time we push a segment,
       // which requires downloading the entire segment tar gz file.
-      URI metadataTarGzFilePath = URI.create(
-          new File(tarFilePath).getParentFile() + File.separator + segmentName 
+ Constants.METADATA_TAR_GZ_FILE_EXT);
+
+      URI metadataTarGzFilePath = generateSegmentMetadataURI(tarFilePath, 
segmentName);
+      LOGGER.info("Checking if metadata tar gz file {} exists", 
metadataTarGzFilePath);
       if (spec.getPushJobSpec().isPreferMetadataTarGz() && 
fileSystem.exists(metadataTarGzFilePath)) {
         segmentMetadataFile = new File(FileUtils.getTempDirectory(),
             "segmentMetadata-" + UUID.randomUUID() + 
TarGzCompressionUtils.TAR_GZ_FILE_EXTENSION);
@@ -442,4 +443,19 @@ public class SegmentPushUtils implements Serializable {
       FileUtils.deleteQuietly(segmentMetadataDir);
     }
   }
+
+  public static URI generateSegmentMetadataURI(String segmentTarPath, String 
segmentName)
+      throws URISyntaxException {
+    URI segmentTarURI = URI.create(segmentTarPath);
+    URI metadataTarGzFilePath = new URI(
+        segmentTarURI.getScheme(),
+        segmentTarURI.getUserInfo(),
+        segmentTarURI.getHost(),
+        segmentTarURI.getPort(),
+        new File(segmentTarURI.getPath()).getParentFile() + File.separator + 
segmentName
+            + Constants.METADATA_TAR_GZ_FILE_EXT,
+        segmentTarURI.getQuery(),
+        segmentTarURI.getFragment());
+    return metadataTarGzFilePath;
+  }
 }
diff --git 
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/SegmentPushUtilsTest.java
 
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/SegmentPushUtilsTest.java
index dd3a09a7ed..20121a14b1 100644
--- 
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/SegmentPushUtilsTest.java
+++ 
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/SegmentPushUtilsTest.java
@@ -20,21 +20,24 @@ package org.apache.pinot.segment.local.utils;
 
 import java.io.IOException;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.util.Map;
 import org.apache.pinot.spi.ingestion.batch.spec.PushJobSpec;
 import org.testng.annotations.Test;
 
-import static org.testng.Assert.*;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
 
 
 public class SegmentPushUtilsTest {
 
   @Test
-  public void testGetSegmentUriToTarPathMap() throws IOException {
+  public void testGetSegmentUriToTarPathMap()
+      throws IOException {
     URI outputDirURI = Files.createTempDirectory("test").toUri();
 
-    String[] segmentFiles = new String[] {
+    String[] segmentFiles = new String[]{
         outputDirURI.resolve("segment.tar.gz").toString(),
         outputDirURI.resolve("stats_202201.tar.gz").toString(),
         outputDirURI.resolve("/2022/segment.tar.gz").toString(),
@@ -61,4 +64,20 @@ public class SegmentPushUtilsTest {
     assertEquals(result.get(segmentFiles[1]), segmentFiles[1]);
     assertEquals(result.get(segmentFiles[3]), segmentFiles[3]);
   }
+
+  @Test
+  public void testGenerateSegmentMetadataURI()
+      throws URISyntaxException {
+    assertEquals(
+        
SegmentPushUtils.generateSegmentMetadataURI("/a/b/c/my-segment.tar.gz", 
"my-segment"),
+        URI.create("/a/b/c/my-segment.metadata.tar.gz"));
+
+    assertEquals(
+        
SegmentPushUtils.generateSegmentMetadataURI("s3://a/b/c/my-segment.tar.gz", 
"my-segment"),
+        URI.create("s3://a/b/c/my-segment.metadata.tar.gz"));
+
+    assertEquals(
+        
SegmentPushUtils.generateSegmentMetadataURI("hdfs://a/b/c/my-segment.tar.gz", 
"my-segment"),
+        URI.create("hdfs://a/b/c/my-segment.metadata.tar.gz"));
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org