Repository: spark Updated Branches: refs/heads/master dfdcab00c -> 7bf9b1201
[SPARK-15166][SQL] Move some hive-specific code from SparkSession ## What changes were proposed in this pull request? This also simplifies the code being moved. ## How was this patch tested? Existing tests. Author: Andrew Or <[email protected]> Closes #12941 from andrewor14/move-code. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7bf9b120 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7bf9b120 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7bf9b120 Branch: refs/heads/master Commit: 7bf9b12019bb20470b726a7233d60ce38a9c52cc Parents: dfdcab0 Author: Andrew Or <[email protected]> Authored: Mon May 9 11:24:58 2016 -0700 Committer: Andrew Or <[email protected]> Committed: Mon May 9 11:24:58 2016 -0700 ---------------------------------------------------------------------- .../scala/org/apache/spark/sql/SparkSession.scala | 13 ------------- .../apache/spark/sql/hive/HiveSessionState.scala | 2 -- .../apache/spark/sql/hive/HiveSharedState.scala | 17 +++++++++++++---- 3 files changed, 13 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/7bf9b120/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala index 2a893c6..c7fa8f7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -73,19 +73,6 @@ class SparkSession private( | Session-related state | * ----------------------- */ - { - val defaultWarehousePath = - SQLConf.WAREHOUSE_PATH - .defaultValueString - .replace("${system:user.dir}", System.getProperty("user.dir")) - val warehousePath = sparkContext.conf.get( - SQLConf.WAREHOUSE_PATH.key, - defaultWarehousePath) - sparkContext.conf.set(SQLConf.WAREHOUSE_PATH.key, warehousePath) - sparkContext.conf.set("hive.metastore.warehouse.dir", warehousePath) - logInfo(s"Setting warehouse location to $warehousePath") - } - /** * State shared across sessions, including the [[SparkContext]], cached data, listener, * and a catalog that interacts with external systems. http://git-wip-us.apache.org/repos/asf/spark/blob/7bf9b120/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala index 31f28f2..46579ec 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala @@ -17,8 +17,6 @@ package org.apache.spark.sql.hive -import org.apache.hadoop.hive.conf.HiveConf.ConfVars - import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.analysis.Analyzer import org.apache.spark.sql.execution.SparkPlanner http://git-wip-us.apache.org/repos/asf/spark/blob/7bf9b120/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSharedState.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSharedState.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSharedState.scala index 0ea5ce9..f0d9640 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSharedState.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSharedState.scala @@ -18,8 +18,9 @@ package org.apache.spark.sql.hive import org.apache.spark.SparkContext -import org.apache.spark.sql.hive.client.{HiveClient, HiveClientImpl} -import org.apache.spark.sql.internal.SharedState +import org.apache.spark.internal.Logging +import org.apache.spark.sql.hive.client.HiveClient +import org.apache.spark.sql.internal.{SharedState, SQLConf} /** @@ -27,9 +28,17 @@ import org.apache.spark.sql.internal.SharedState * [[org.apache.spark.sql.SparkSession]] backed by Hive. */ private[hive] class HiveSharedState(override val sparkContext: SparkContext) - extends SharedState(sparkContext) { + extends SharedState(sparkContext) with Logging { - // TODO: just share the IsolatedClientLoader instead of the client instances themselves + // TODO: just share the IsolatedClientLoader instead of the client instance itself + + { + // Set the Hive metastore warehouse path to the one we use + val tempConf = new SQLConf + sparkContext.conf.getAll.foreach { case (k, v) => tempConf.setConfString(k, v) } + sparkContext.conf.set("hive.metastore.warehouse.dir", tempConf.warehousePath) + logInfo(s"Setting Hive metastore warehouse path to '${tempConf.warehousePath}'") + } /** * A Hive client used to interact with the metastore. --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
