This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs-object-store.git
The following commit(s) were added to refs/heads/main by this push:
new de4a30e Allow explicitly specifying GCS base URL (#567)
de4a30e is described below
commit de4a30e24b40ade260f88b7894dbc819124597ee
Author: Renan GEHAN <[email protected]>
AuthorDate: Sat Dec 13 14:21:33 2025 +0100
Allow explicitly specifying GCS base URL (#567)
* Allow explicitly specify GCS base URL
* Add configuration key
* Fix doc (don't refer to private var)
* Reformat
---
src/gcp/builder.rs | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 83 insertions(+), 3 deletions(-)
diff --git a/src/gcp/builder.rs b/src/gcp/builder.rs
index b637c53..48a5246 100644
--- a/src/gcp/builder.rs
+++ b/src/gcp/builder.rs
@@ -98,6 +98,8 @@ pub struct GoogleCloudStorageBuilder {
bucket_name: Option<String>,
/// Url
url: Option<String>,
+ /// Base URL
+ base_url: Option<String>,
/// Path to the service account file
service_account_path: Option<String>,
/// The serialized service account key
@@ -159,6 +161,15 @@ pub enum GoogleConfigKey {
/// - `bucket_name`
Bucket,
+ /// Base URL
+ ///
+ /// See [`GoogleCloudStorageBuilder::with_base_url`] for details.
+ ///
+ /// Supported keys:
+ /// - `google_base_url`
+ /// - `base_url`
+ BaseUrl,
+
/// Application credentials path
///
/// See [`GoogleCloudStorageBuilder::with_application_credentials`].
@@ -185,6 +196,7 @@ impl AsRef<str> for GoogleConfigKey {
Self::ServiceAccount => "google_service_account",
Self::ServiceAccountKey => "google_service_account_key",
Self::Bucket => "google_bucket",
+ Self::BaseUrl => "google_base_url",
Self::ApplicationCredentials => "google_application_credentials",
Self::SkipSignature => "google_skip_signature",
Self::Client(key) => key.as_ref(),
@@ -203,6 +215,7 @@ impl FromStr for GoogleConfigKey {
| "service_account_path" => Ok(Self::ServiceAccount),
"google_service_account_key" | "service_account_key" =>
Ok(Self::ServiceAccountKey),
"google_bucket" | "google_bucket_name" | "bucket" | "bucket_name"
=> Ok(Self::Bucket),
+ "google_base_url" | "base_url" => Ok(Self::BaseUrl),
"google_application_credentials" | "application_credentials" => {
Ok(Self::ApplicationCredentials)
}
@@ -225,6 +238,7 @@ impl Default for GoogleCloudStorageBuilder {
retry_config: Default::default(),
client_options: ClientOptions::new().with_allow_http(true),
url: None,
+ base_url: None,
credentials: None,
skip_signature: Default::default(),
signing_credentials: None,
@@ -304,6 +318,7 @@ impl GoogleCloudStorageBuilder {
GoogleConfigKey::ServiceAccount => self.service_account_path =
Some(value.into()),
GoogleConfigKey::ServiceAccountKey => self.service_account_key =
Some(value.into()),
GoogleConfigKey::Bucket => self.bucket_name = Some(value.into()),
+ GoogleConfigKey::BaseUrl => self.base_url = Some(value.into()),
GoogleConfigKey::ApplicationCredentials => {
self.application_credentials_path = Some(value.into())
}
@@ -331,6 +346,7 @@ impl GoogleCloudStorageBuilder {
GoogleConfigKey::ServiceAccount =>
self.service_account_path.clone(),
GoogleConfigKey::ServiceAccountKey =>
self.service_account_key.clone(),
GoogleConfigKey::Bucket => self.bucket_name.clone(),
+ GoogleConfigKey::BaseUrl => self.base_url.clone(),
GoogleConfigKey::ApplicationCredentials =>
self.application_credentials_path.clone(),
GoogleConfigKey::SkipSignature =>
Some(self.skip_signature.to_string()),
GoogleConfigKey::Client(key) =>
self.client_options.get_config_value(key),
@@ -367,6 +383,25 @@ impl GoogleCloudStorageBuilder {
self
}
+ /// Sets the base URL for communicating with GCS.
+ ///
+ /// If not explicitly set, it will be:
+ /// 1. Derived from the service account credentials, if provided
+ /// 2. Otherwise, uses the default GCS endpoint
+ ///
+ /// # Example
+ /// ```
+ /// use object_store::gcp::GoogleCloudStorageBuilder;
+ ///
+ /// let gcs = GoogleCloudStorageBuilder::from_env()
+ /// .with_base_url("https://localhost:4443";)
+ /// .build();
+ /// ```
+ pub fn with_base_url(mut self, base_url: &str) -> Self {
+ self.base_url = Some(base_url.into());
+ self
+ }
+
/// Set the path to the service account file.
///
/// This or [`GoogleCloudStorageBuilder::with_service_account_key`] must be
@@ -506,9 +541,13 @@ impl GoogleCloudStorageBuilder {
.map(|c| c.disable_oauth)
.unwrap_or(false);
- let gcs_base_url: String = service_account_credentials
- .as_ref()
- .and_then(|c| c.gcs_base_url.clone())
+ let gcs_base_url: String = self
+ .base_url
+ .or_else(|| {
+ service_account_credentials
+ .as_ref()
+ .and_then(|c| c.gcs_base_url.clone())
+ })
.unwrap_or_else(|| DEFAULT_GCS_BASE_URL.to_string());
let credentials = if let Some(credentials) = self.credentials {
@@ -607,6 +646,7 @@ mod tests {
use tempfile::NamedTempFile;
const FAKE_KEY: &str = r#"{"private_key": "private_key", "private_key_id":
"private_key_id", "client_email":"client_email", "disable_oauth":true}"#;
+ const FAKE_KEY_WITH_BASE_URL: &str = r#"{"private_key": "private_key",
"private_key_id": "private_key_id", "client_email":"client_email",
"disable_oauth":true, "gcs_base_url":
"https://base-url-from-credentials:4443"}"#;
#[test]
fn gcs_test_service_account_key_and_path() {
@@ -721,6 +761,46 @@ mod tests {
.unwrap();
}
+ #[test]
+ fn gcs_test_with_base_url() {
+ let no_base_url = GoogleCloudStorageBuilder::new()
+ .with_bucket_name("foo")
+ .build()
+ .unwrap();
+ assert_eq!(no_base_url.client.config().base_url, DEFAULT_GCS_BASE_URL);
+
+ let explicit_override = GoogleCloudStorageBuilder::new()
+ .with_bucket_name("foo")
+ .with_base_url("https://explicitly-overriden:4443";)
+ .build()
+ .unwrap();
+ assert_eq!(
+ explicit_override.client.config().base_url,
+ "https://explicitly-overriden:4443";
+ );
+
+ let url_in_credentials = GoogleCloudStorageBuilder::new()
+ .with_bucket_name("foo")
+ .with_service_account_key(FAKE_KEY_WITH_BASE_URL)
+ .build()
+ .unwrap();
+ assert_eq!(
+ url_in_credentials.client.config().base_url,
+ "https://base-url-from-credentials:4443";
+ );
+
+ let explicit_override_and_credentials =
GoogleCloudStorageBuilder::new()
+ .with_bucket_name("foo")
+ .with_base_url("https://explicitly-overriden:4443";) // this should
take precedence
+ .with_service_account_key(FAKE_KEY_WITH_BASE_URL)
+ .build()
+ .unwrap();
+ assert_eq!(
+ explicit_override_and_credentials.client.config().base_url,
+ "https://explicitly-overriden:4443";
+ );
+ }
+
#[test]
fn gcs_test_config_get_value() {
let google_service_account =
"object_store:fake_service_account".to_string();
