xiangfu0 commented on a change in pull request #6841:
URL: https://github.com/apache/incubator-pinot/pull/6841#discussion_r620703094
##########
File path:
pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java
##########
@@ -34,104 +35,211 @@
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.net.URI;
-import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
-import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.filesystem.PinotFS;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
-import static java.util.Objects.requireNonNull;
-import static joptsimple.internal.Strings.isNullOrEmpty;
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static java.lang.String.format;
+import static org.apache.pinot.plugin.filesystem.GcsUri.createGcsUri;
-public class GcsPinotFS extends PinotFS {
+public class GcsPinotFS extends PinotFS {
public static final String PROJECT_ID = "projectId";
public static final String GCP_KEY = "gcpKey";
private static final Logger LOGGER =
LoggerFactory.getLogger(GcsPinotFS.class);
- private static final String SCHEME = "gs";
- private static final String DELIMITER = "/";
private static final int BUFFER_SIZE = 128 * 1024;
- private Storage storage;
+ // See https://cloud.google.com/storage/docs/json_api/v1/how-tos/batch
+ private static final int BATCH_LIMIT = 100;
+ private Storage _storage;
@Override
public void init(PinotConfiguration config) {
- Credentials credentials;
+ LOGGER.info("Configs are: {}, {}", PROJECT_ID,
config.getProperty(PROJECT_ID));
+ _storage = initializeStorage(config);
+ }
+ private Storage initializeStorage(PinotConfiguration config) {
+ checkArgument(!isNullOrEmpty(config.getProperty(PROJECT_ID)));
Review comment:
I think we still need to handle the default credential support here:
```
StorageOptions.Builder storageBuilder = StorageOptions.newBuilder();
if (!isNullOrEmpty(config.getProperty(PROJECT_ID)) &&
!isNullOrEmpty(config.getProperty(GCP_KEY))) {
LOGGER.info("Configs are: {}, {}", PROJECT_ID,
config.getProperty(PROJECT_ID));
String projectId = config.getProperty(PROJECT_ID);
String gcpKey = config.getProperty(GCP_KEY);
storageBuilder.setProjectId(projectId);
credentials =
GoogleCredentials.fromStream(Files.newInputStream(Paths.get(gcpKey)));
} else {
LOGGER.info("Configs using default credential");
credentials = GoogleCredentials.getApplicationDefault();
}
storage =
storageBuilder.setCredentials(credentials).build().getService();
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
