Repository: spark Updated Branches: refs/heads/branch-2.0 196d62eae -> a7f5e7066
[SPARK-16959][SQL] Rebuild Table Comment when Retrieving Metadata from Hive Metastore ### What changes were proposed in this pull request? The `comment` in `CatalogTable` returned from Hive is always empty. We store it in the table property when creating a table. However, when we try to retrieve the table metadata from Hive metastore, we do not rebuild it. The `comment` is always empty. This PR is to fix the issue. ### How was this patch tested? Fixed the test case to verify the change. Author: gatorsmile <[email protected]> Closes #14550 from gatorsmile/tableComment. (cherry picked from commit bdd537164dcfeec5e9c51d54791ef16997ff2597) Signed-off-by: Wenchen Fan <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a7f5e706 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a7f5e706 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a7f5e706 Branch: refs/heads/branch-2.0 Commit: a7f5e7066f935d58d702a3e86b85aa175291d0fc Parents: 196d62e Author: gatorsmile <[email protected]> Authored: Wed Aug 10 16:25:01 2016 +0800 Committer: Wenchen Fan <[email protected]> Committed: Sun Sep 4 00:10:27 2016 +0800 ---------------------------------------------------------------------- .../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 3 ++- .../org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/a7f5e706/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 7b4483a..ac78676 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -412,7 +412,8 @@ private[hive] class HiveClientImpl( serdeProperties = Option(h.getTTable.getSd.getSerdeInfo.getParameters) .map(_.asScala.toMap).orNull ), - properties = properties, + properties = properties.filter(kv => kv._1 != "comment"), + comment = properties.get("comment"), viewOriginalText = Option(h.getViewOriginalText), viewText = Option(h.getViewExpandedText), unsupportedFeatures = unsupportedFeatures) http://git-wip-us.apache.org/repos/asf/spark/blob/a7f5e706/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index f31215a..1416409 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -135,8 +135,11 @@ class HiveDDLSuite sql(s"CREATE VIEW $viewName COMMENT 'no comment' AS SELECT * FROM $tabName") val tableMetadata = catalog.getTableMetadata(TableIdentifier(tabName, Some("default"))) val viewMetadata = catalog.getTableMetadata(TableIdentifier(viewName, Some("default"))) - assert(tableMetadata.properties.get("comment") == Option("BLABLA")) - assert(viewMetadata.properties.get("comment") == Option("no comment")) + assert(tableMetadata.comment == Option("BLABLA")) + assert(viewMetadata.comment == Option("no comment")) + // Ensure that `comment` is removed from the table property + assert(tableMetadata.properties.get("comment").isEmpty) + assert(viewMetadata.properties.get("comment").isEmpty) } } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
