This is an automated email from the ASF dual-hosted git repository.
curth pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new d9d8079a4 fix(csharp/src/Drivers/Databricks): [PECO-2396] Fix
timestamp for dbr 6.6 - Set timestamp configuration on OpenSessionReq (#3327)
d9d8079a4 is described below
commit d9d8079a45a5a38c84631eaffc4b936283e349a9
Author: Todd Meng <[email protected]>
AuthorDate: Wed Aug 27 13:23:03 2025 -0700
fix(csharp/src/Drivers/Databricks): [PECO-2396] Fix timestamp for dbr 6.6 -
Set timestamp configuration on OpenSessionReq (#3327)
### Motivation
ExecuteStatementReq only accepts configuration after dbr 7.0, so
timestampasstring did not apply to dbr < 7.0.
The fix is to move it to Session configuration where possible
### Testing
Verified it works by running StatementTests CanExecuteTimestampQuery on
dbr 6.6 (fails before)
---------
Co-authored-by: Bruce Irschick <[email protected]>
---
csharp/src/Drivers/Apache/Spark/SparkConnection.cs | 5 -----
csharp/src/Drivers/Databricks/DatabricksConnection.cs | 12 ++++++++++--
csharp/src/Drivers/Databricks/DatabricksStatement.cs | 4 ----
3 files changed, 10 insertions(+), 11 deletions(-)
diff --git a/csharp/src/Drivers/Apache/Spark/SparkConnection.cs
b/csharp/src/Drivers/Apache/Spark/SparkConnection.cs
index 151e93f11..584380bce 100644
--- a/csharp/src/Drivers/Apache/Spark/SparkConnection.cs
+++ b/csharp/src/Drivers/Apache/Spark/SparkConnection.cs
@@ -35,11 +35,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
internal static TSparkGetDirectResults sparkGetDirectResults = new
TSparkGetDirectResults(1000);
- internal static readonly Dictionary<string, string> timestampConfig =
new Dictionary<string, string>
- {
- { "spark.thriftserver.arrowBasedRowSet.timestampAsString", "false"
}
- };
-
internal SparkConnection(IReadOnlyDictionary<string, string>
properties)
: base(properties)
{
diff --git a/csharp/src/Drivers/Databricks/DatabricksConnection.cs
b/csharp/src/Drivers/Databricks/DatabricksConnection.cs
index d1595b95c..9e5d05a98 100644
--- a/csharp/src/Drivers/Databricks/DatabricksConnection.cs
+++ b/csharp/src/Drivers/Databricks/DatabricksConnection.cs
@@ -47,6 +47,10 @@ namespace Apache.Arrow.Adbc.Drivers.Databricks
/// </summary>
public const string DefaultConfigEnvironmentVariable =
"DATABRICKS_CONFIG_FILE";
+ internal static readonly Dictionary<string, string> timestampConfig =
new Dictionary<string, string>
+ {
+ { "spark.thriftserver.arrowBasedRowSet.timestampAsString", "false"
},
+ };
private bool _applySSPWithQueries = false;
private bool _enableDirectResults = true;
private bool _enableMultipleCatalogSupport = true;
@@ -582,11 +586,15 @@ namespace Apache.Arrow.Adbc.Drivers.Databricks
{
req.InitialNamespace = _defaultNamespace;
}
-
+ req.Configuration = new Dictionary<string, string>();
+ // merge timestampConfig with serverSideProperties
+ foreach (var kvp in timestampConfig)
+ {
+ req.Configuration[kvp.Key] = kvp.Value;
+ }
// If not using queries to set server-side properties, include
them in Configuration
if (!_applySSPWithQueries)
{
- req.Configuration = new Dictionary<string, string>();
var serverSideProperties = GetServerSideProperties();
foreach (var property in serverSideProperties)
{
diff --git a/csharp/src/Drivers/Databricks/DatabricksStatement.cs
b/csharp/src/Drivers/Databricks/DatabricksStatement.cs
index c9abb006e..322b92bef 100644
--- a/csharp/src/Drivers/Databricks/DatabricksStatement.cs
+++ b/csharp/src/Drivers/Databricks/DatabricksStatement.cs
@@ -97,10 +97,6 @@ namespace Apache.Arrow.Adbc.Drivers.Databricks
statement.EnforceResultPersistenceMode = false;
statement.CanReadArrowResult = true;
-#pragma warning disable CS0618 // Type or member is obsolete
- statement.ConfOverlay = SparkConnection.timestampConfig;
-#pragma warning restore CS0618 // Type or member is obsolete
-
statement.UseArrowNativeTypes = new TSparkArrowTypes
{
TimestampAsArrow = true,
