[incubator-pinot] branch master updated: Adding S3PinotFS as one of the default PinotFS for Quickstart (#6379)

[xiangfu]

This is an automated email from the ASF dual-hosted git repository.

xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new e268a79  Adding S3PinotFS as one of the default PinotFS for Quickstart 
(#6379)
e268a79 is described below

commit e268a795a684f18e1876e4530c81ef00baec878c
Author: Xiang Fu <fx19880...@gmail.com>
AuthorDate: Mon Dec 28 00:15:26 2020 -0800

    Adding S3PinotFS as one of the default PinotFS for Quickstart (#6379)
    
    * Adding s3 as default pinot fs for quickstart
    
    * Update BootstrapTableTool.java
---
 .../org/apache/pinot/tools/BootstrapTableTool.java | 16 +++++++------
 .../tools/admin/command/QuickstartRunner.java      | 27 +++++++++++++++++++++-
 2 files changed, 35 insertions(+), 8 deletions(-)

diff --git 
a/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java 
b/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java
index 0723b06..3de51b7 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java
@@ -115,13 +115,15 @@ public class BootstrapTableTool {
         String inputDirURI = spec.getInputDirURI();
         if (!new File(inputDirURI).exists()) {
           URL resolvedInputDirURI = 
BootstrapTableTool.class.getClassLoader().getResource(inputDirURI);
-          if (resolvedInputDirURI.getProtocol().equals("jar")) {
-            String[] splits = resolvedInputDirURI.getFile().split("!");
-            String inputDir = new File(setupTableTmpDir, 
"inputData").toString();
-            JarUtils.copyResourcesToDirectory(splits[0], 
splits[1].substring(1), inputDir);
-            spec.setInputDirURI(inputDir);
-          } else {
-            spec.setInputDirURI(resolvedInputDirURI.toString());
+          if (resolvedInputDirURI != null) {
+            if ("jar".equals(resolvedInputDirURI.getProtocol())) {
+              String[] splits = resolvedInputDirURI.getFile().split("!");
+              String inputDir = new File(setupTableTmpDir, 
"inputData").toString();
+              JarUtils.copyResourcesToDirectory(splits[0], 
splits[1].substring(1), inputDir);
+              spec.setInputDirURI(inputDir);
+            } else {
+              spec.setInputDirURI(resolvedInputDirURI.toString());
+            }
           }
         }
         IngestionJobLauncher.runIngestionJob(spec);
diff --git 
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
 
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
index e2e3e38..1f4f838 100644
--- 
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
+++ 
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
@@ -19,6 +19,7 @@
 package org.apache.pinot.tools.admin.command;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.ImmutableMap;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
@@ -28,21 +29,27 @@ import java.net.InetAddress;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 import org.apache.commons.io.FileUtils;
 import org.apache.pinot.common.utils.CommonConstants;
 import org.apache.pinot.spi.config.table.TableType;
 import org.apache.pinot.spi.config.tenant.TenantRole;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.filesystem.PinotFSFactory;
 import org.apache.pinot.spi.ingestion.batch.IngestionJobLauncher;
 import org.apache.pinot.spi.ingestion.batch.spec.SegmentGenerationJobSpec;
 import org.apache.pinot.spi.utils.JsonUtils;
 import org.apache.pinot.tools.QuickstartTableRequest;
 import org.apache.pinot.tools.BootstrapTableTool;
 import org.apache.pinot.tools.utils.JarUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.yaml.snakeyaml.Yaml;
 
 
 public class QuickstartRunner {
+  private static final Logger LOGGER = 
LoggerFactory.getLogger(QuickstartRunner.class.getName());
   private static final Random RANDOM = new Random();
   private static final String CLUSTER_NAME = "QuickStartCluster";
 
@@ -138,6 +145,7 @@ public class QuickstartRunner {
 
   public void startAll()
       throws Exception {
+    registerDefaultPinotFS();
     startZookeeper();
     startControllers();
     startBrokers();
@@ -222,4 +230,21 @@ public class QuickstartRunner {
     return JsonUtils.stringToJsonNode(new 
PostQueryCommand().setBrokerPort(String.valueOf(brokerPort))
         
.setQueryType(CommonConstants.Broker.Request.SQL).setQuery(query).run());
   }
-}
+
+  public static void registerDefaultPinotFS() {
+    registerPinotFS("s3", "org.apache.pinot.plugin.filesystem.S3PinotFS", 
ImmutableMap.of("region", System.getProperty("AWS_REGION", "us-west-2")));
+  }
+
+  public static void registerPinotFS(String scheme, String fsClassName, 
Map<String, Object> configs) {
+    if (PinotFSFactory.isSchemeSupported(scheme)) {
+      LOGGER.info("PinotFS for scheme: {} is already registered.", scheme);
+      return;
+    }
+    try {
+      PinotFSFactory.register(scheme, fsClassName, new 
PinotConfiguration(configs));
+      LOGGER.info("Registered PinotFS for scheme: {}", scheme);
+    } catch (Exception e) {
+      LOGGER.info("Unable to init PinotFS for scheme: {}, class name: {}, 
configs: {}, Error: {}", scheme, fsClassName, configs, e);
+    }
+  }
+}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org