Repository: spark
Updated Branches:
refs/heads/branch-2.3 eecd83cb2 -> 8032cf852
[SPARK-22972] Couldn't find corresponding Hive SerDe for data source provider
org.apache.spark.sql.hive.orc
## What changes were proposed in this pull request?
Fix the warning: Couldn't find corresponding Hive SerDe for data source
provider org.apache.spark.sql.hive.orc.
## How was this patch tested?
test("SPARK-22972: hive orc source")
assert(HiveSerDe.sourceToSerDe("org.apache.spark.sql.hive.orc")
.equals(HiveSerDe.sourceToSerDe("orc")))
Author: xubo245 <[email protected]>
Closes #20165 from xubo245/HiveSerDe.
(cherry picked from commit 68ce792b5857f0291154f524ac651036db868bb9)
Signed-off-by: gatorsmile <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8032cf85
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8032cf85
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8032cf85
Branch: refs/heads/branch-2.3
Commit: 8032cf852fccd0ab8754f633affdc9ba8fc99e58
Parents: eecd83c
Author: xubo245 <[email protected]>
Authored: Tue Jan 9 10:15:01 2018 +0800
Committer: gatorsmile <[email protected]>
Committed: Tue Jan 9 10:15:45 2018 +0800
----------------------------------------------------------------------
.../apache/spark/sql/internal/HiveSerDe.scala | 1 +
.../spark/sql/hive/orc/HiveOrcSourceSuite.scala | 29 ++++++++++++++++++++
2 files changed, 30 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/8032cf85/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
----------------------------------------------------------------------
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
b/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
index b9515ec..dac4636 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
@@ -73,6 +73,7 @@ object HiveSerDe {
val key = source.toLowerCase(Locale.ROOT) match {
case s if s.startsWith("org.apache.spark.sql.parquet") => "parquet"
case s if s.startsWith("org.apache.spark.sql.orc") => "orc"
+ case s if s.startsWith("org.apache.spark.sql.hive.orc") => "orc"
case s if s.equals("orcfile") => "orc"
case s if s.equals("parquetfile") => "parquet"
case s if s.equals("avrofile") => "avro"
http://git-wip-us.apache.org/repos/asf/spark/blob/8032cf85/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
index 17b7d8c..d556a03 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
@@ -20,8 +20,10 @@ package org.apache.spark.sql.hive.orc
import java.io.File
import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.datasources.orc.OrcSuite
import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.internal.HiveSerDe
import org.apache.spark.util.Utils
class HiveOrcSourceSuite extends OrcSuite with TestHiveSingleton {
@@ -62,6 +64,33 @@ class HiveOrcSourceSuite extends OrcSuite with
TestHiveSingleton {
""".stripMargin)
}
+ test("SPARK-22972: hive orc source") {
+ val tableName = "normal_orc_as_source_hive"
+ withTable(tableName) {
+ sql(
+ s"""
+ |CREATE TABLE $tableName
+ |USING org.apache.spark.sql.hive.orc
+ |OPTIONS (
+ | PATH '${new File(orcTableAsDir.getAbsolutePath).toURI}'
+ |)
+ """.stripMargin)
+
+ val tableMetadata = spark.sessionState.catalog.getTableMetadata(
+ TableIdentifier(tableName))
+ assert(tableMetadata.storage.inputFormat ==
+ Option("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"))
+ assert(tableMetadata.storage.outputFormat ==
+ Option("org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"))
+ assert(tableMetadata.storage.serde ==
+ Option("org.apache.hadoop.hive.ql.io.orc.OrcSerde"))
+ assert(HiveSerDe.sourceToSerDe("org.apache.spark.sql.hive.orc")
+ .equals(HiveSerDe.sourceToSerDe("orc")))
+ assert(HiveSerDe.sourceToSerDe("org.apache.spark.sql.orc")
+ .equals(HiveSerDe.sourceToSerDe("orc")))
+ }
+ }
+
test("SPARK-19459/SPARK-18220: read char/varchar column written by Hive") {
val location = Utils.createTempDir()
val uri = location.toURI
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
