[SPARK-21803][TEST] Remove the HiveDDLCommandSuite ## What changes were proposed in this pull request? We do not have any Hive-specific parser. It does not make sense to keep a parser-specific test suite `HiveDDLCommandSuite.scala` in the Hive package. This PR is to remove it.
## How was this patch tested? N/A Author: gatorsmile <[email protected]> Closes #19015 from gatorsmile/combineDDL. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/be72b157 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/be72b157 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/be72b157 Branch: refs/heads/master Commit: be72b157ea13ea116c5178a9e41e37ae24090f72 Parents: 5c9b301 Author: gatorsmile <[email protected]> Authored: Tue Aug 22 17:54:39 2017 +0800 Committer: Wenchen Fan <[email protected]> Committed: Tue Aug 22 17:54:39 2017 +0800 ---------------------------------------------------------------------- .../sql/execution/command/DDLCommandSuite.scala | 1049 ------------ .../sql/execution/command/DDLParserSuite.scala | 1615 ++++++++++++++++++ .../spark/sql/hive/HiveDDLCommandSuite.scala | 739 -------- .../apache/spark/sql/hive/TestHiveSuite.scala | 4 + .../spark/sql/hive/execution/HiveDDLSuite.scala | 11 + .../sql/hive/execution/HiveSerDeSuite.scala | 133 +- 6 files changed, 1762 insertions(+), 1789 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/be72b157/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala deleted file mode 100644 index 5643c58..0000000 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala +++ /dev/null @@ -1,1049 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.command - -import java.net.URI -import java.util.Locale - -import scala.reflect.{classTag, ClassTag} - -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.catalog._ -import org.apache.spark.sql.catalyst.parser.ParseException -import org.apache.spark.sql.catalyst.plans.PlanTest -import org.apache.spark.sql.catalyst.plans.logical.Project -import org.apache.spark.sql.execution.SparkSqlParser -import org.apache.spark.sql.execution.datasources.CreateTable -import org.apache.spark.sql.internal.{HiveSerDe, SQLConf} -import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType} - - -// TODO: merge this with DDLSuite (SPARK-14441) -class DDLCommandSuite extends PlanTest { - private lazy val parser = new SparkSqlParser(new SQLConf) - - private def assertUnsupported(sql: String, containsThesePhrases: Seq[String] = Seq()): Unit = { - val e = intercept[ParseException] { - parser.parsePlan(sql) - } - assert(e.getMessage.toLowerCase(Locale.ROOT).contains("operation not allowed")) - containsThesePhrases.foreach { p => - assert(e.getMessage.toLowerCase(Locale.ROOT).contains(p.toLowerCase(Locale.ROOT))) - } - } - - private def parseAs[T: ClassTag](query: String): T = { - parser.parsePlan(query) match { - case t: T => t - case other => - fail(s"Expected to parse ${classTag[T].runtimeClass} from query," + - s"got ${other.getClass.getName}: $query") - } - } - - test("create database") { - val sql = - """ - |CREATE DATABASE IF NOT EXISTS database_name - |COMMENT 'database_comment' LOCATION '/home/user/db' - |WITH DBPROPERTIES ('a'='a', 'b'='b', 'c'='c') - """.stripMargin - val parsed = parser.parsePlan(sql) - val expected = CreateDatabaseCommand( - "database_name", - ifNotExists = true, - Some("/home/user/db"), - Some("database_comment"), - Map("a" -> "a", "b" -> "b", "c" -> "c")) - comparePlans(parsed, expected) - } - - test("create database - property values must be set") { - assertUnsupported( - sql = "CREATE DATABASE my_db WITH DBPROPERTIES('key_without_value', 'key_with_value'='x')", - containsThesePhrases = Seq("key_without_value")) - } - - test("drop database") { - val sql1 = "DROP DATABASE IF EXISTS database_name RESTRICT" - val sql2 = "DROP DATABASE IF EXISTS database_name CASCADE" - val sql3 = "DROP SCHEMA IF EXISTS database_name RESTRICT" - val sql4 = "DROP SCHEMA IF EXISTS database_name CASCADE" - // The default is restrict=true - val sql5 = "DROP DATABASE IF EXISTS database_name" - // The default is ifExists=false - val sql6 = "DROP DATABASE database_name" - val sql7 = "DROP DATABASE database_name CASCADE" - - val parsed1 = parser.parsePlan(sql1) - val parsed2 = parser.parsePlan(sql2) - val parsed3 = parser.parsePlan(sql3) - val parsed4 = parser.parsePlan(sql4) - val parsed5 = parser.parsePlan(sql5) - val parsed6 = parser.parsePlan(sql6) - val parsed7 = parser.parsePlan(sql7) - - val expected1 = DropDatabaseCommand( - "database_name", - ifExists = true, - cascade = false) - val expected2 = DropDatabaseCommand( - "database_name", - ifExists = true, - cascade = true) - val expected3 = DropDatabaseCommand( - "database_name", - ifExists = false, - cascade = false) - val expected4 = DropDatabaseCommand( - "database_name", - ifExists = false, - cascade = true) - - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - comparePlans(parsed3, expected1) - comparePlans(parsed4, expected2) - comparePlans(parsed5, expected1) - comparePlans(parsed6, expected3) - comparePlans(parsed7, expected4) - } - - test("alter database set dbproperties") { - // ALTER (DATABASE|SCHEMA) database_name SET DBPROPERTIES (property_name=property_value, ...) - val sql1 = "ALTER DATABASE database_name SET DBPROPERTIES ('a'='a', 'b'='b', 'c'='c')" - val sql2 = "ALTER SCHEMA database_name SET DBPROPERTIES ('a'='a')" - - val parsed1 = parser.parsePlan(sql1) - val parsed2 = parser.parsePlan(sql2) - - val expected1 = AlterDatabasePropertiesCommand( - "database_name", - Map("a" -> "a", "b" -> "b", "c" -> "c")) - val expected2 = AlterDatabasePropertiesCommand( - "database_name", - Map("a" -> "a")) - - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - } - - test("alter database - property values must be set") { - assertUnsupported( - sql = "ALTER DATABASE my_db SET DBPROPERTIES('key_without_value', 'key_with_value'='x')", - containsThesePhrases = Seq("key_without_value")) - } - - test("describe database") { - // DESCRIBE DATABASE [EXTENDED] db_name; - val sql1 = "DESCRIBE DATABASE EXTENDED db_name" - val sql2 = "DESCRIBE DATABASE db_name" - - val parsed1 = parser.parsePlan(sql1) - val parsed2 = parser.parsePlan(sql2) - - val expected1 = DescribeDatabaseCommand( - "db_name", - extended = true) - val expected2 = DescribeDatabaseCommand( - "db_name", - extended = false) - - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - } - - test("create function") { - val sql1 = - """ - |CREATE TEMPORARY FUNCTION helloworld as - |'com.matthewrathbone.example.SimpleUDFExample' USING JAR '/path/to/jar1', - |JAR '/path/to/jar2' - """.stripMargin - val sql2 = - """ - |CREATE FUNCTION hello.world as - |'com.matthewrathbone.example.SimpleUDFExample' USING ARCHIVE '/path/to/archive', - |FILE '/path/to/file' - """.stripMargin - val sql3 = - """ - |CREATE OR REPLACE TEMPORARY FUNCTION helloworld3 as - |'com.matthewrathbone.example.SimpleUDFExample' USING JAR '/path/to/jar1', - |JAR '/path/to/jar2' - """.stripMargin - val sql4 = - """ - |CREATE OR REPLACE FUNCTION hello.world1 as - |'com.matthewrathbone.example.SimpleUDFExample' USING ARCHIVE '/path/to/archive', - |FILE '/path/to/file' - """.stripMargin - val sql5 = - """ - |CREATE FUNCTION IF NOT EXISTS hello.world2 as - |'com.matthewrathbone.example.SimpleUDFExample' USING ARCHIVE '/path/to/archive', - |FILE '/path/to/file' - """.stripMargin - val parsed1 = parser.parsePlan(sql1) - val parsed2 = parser.parsePlan(sql2) - val parsed3 = parser.parsePlan(sql3) - val parsed4 = parser.parsePlan(sql4) - val parsed5 = parser.parsePlan(sql5) - val expected1 = CreateFunctionCommand( - None, - "helloworld", - "com.matthewrathbone.example.SimpleUDFExample", - Seq( - FunctionResource(FunctionResourceType.fromString("jar"), "/path/to/jar1"), - FunctionResource(FunctionResourceType.fromString("jar"), "/path/to/jar2")), - isTemp = true, ifNotExists = false, replace = false) - val expected2 = CreateFunctionCommand( - Some("hello"), - "world", - "com.matthewrathbone.example.SimpleUDFExample", - Seq( - FunctionResource(FunctionResourceType.fromString("archive"), "/path/to/archive"), - FunctionResource(FunctionResourceType.fromString("file"), "/path/to/file")), - isTemp = false, ifNotExists = false, replace = false) - val expected3 = CreateFunctionCommand( - None, - "helloworld3", - "com.matthewrathbone.example.SimpleUDFExample", - Seq( - FunctionResource(FunctionResourceType.fromString("jar"), "/path/to/jar1"), - FunctionResource(FunctionResourceType.fromString("jar"), "/path/to/jar2")), - isTemp = true, ifNotExists = false, replace = true) - val expected4 = CreateFunctionCommand( - Some("hello"), - "world1", - "com.matthewrathbone.example.SimpleUDFExample", - Seq( - FunctionResource(FunctionResourceType.fromString("archive"), "/path/to/archive"), - FunctionResource(FunctionResourceType.fromString("file"), "/path/to/file")), - isTemp = false, ifNotExists = false, replace = true) - val expected5 = CreateFunctionCommand( - Some("hello"), - "world2", - "com.matthewrathbone.example.SimpleUDFExample", - Seq( - FunctionResource(FunctionResourceType.fromString("archive"), "/path/to/archive"), - FunctionResource(FunctionResourceType.fromString("file"), "/path/to/file")), - isTemp = false, ifNotExists = true, replace = false) - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - comparePlans(parsed3, expected3) - comparePlans(parsed4, expected4) - comparePlans(parsed5, expected5) - } - - test("drop function") { - val sql1 = "DROP TEMPORARY FUNCTION helloworld" - val sql2 = "DROP TEMPORARY FUNCTION IF EXISTS helloworld" - val sql3 = "DROP FUNCTION hello.world" - val sql4 = "DROP FUNCTION IF EXISTS hello.world" - - val parsed1 = parser.parsePlan(sql1) - val parsed2 = parser.parsePlan(sql2) - val parsed3 = parser.parsePlan(sql3) - val parsed4 = parser.parsePlan(sql4) - - val expected1 = DropFunctionCommand( - None, - "helloworld", - ifExists = false, - isTemp = true) - val expected2 = DropFunctionCommand( - None, - "helloworld", - ifExists = true, - isTemp = true) - val expected3 = DropFunctionCommand( - Some("hello"), - "world", - ifExists = false, - isTemp = false) - val expected4 = DropFunctionCommand( - Some("hello"), - "world", - ifExists = true, - isTemp = false) - - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - comparePlans(parsed3, expected3) - comparePlans(parsed4, expected4) - } - - test("create hive table - table file format") { - val allSources = Seq("parquet", "parquetfile", "orc", "orcfile", "avro", "avrofile", - "sequencefile", "rcfile", "textfile") - - allSources.foreach { s => - val query = s"CREATE TABLE my_tab STORED AS $s" - val ct = parseAs[CreateTable](query) - val hiveSerde = HiveSerDe.sourceToSerDe(s) - assert(hiveSerde.isDefined) - assert(ct.tableDesc.storage.serde == - hiveSerde.get.serde.orElse(Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))) - assert(ct.tableDesc.storage.inputFormat == hiveSerde.get.inputFormat) - assert(ct.tableDesc.storage.outputFormat == hiveSerde.get.outputFormat) - } - } - - test("create hive table - row format and table file format") { - val createTableStart = "CREATE TABLE my_tab ROW FORMAT" - val fileFormat = s"STORED AS INPUTFORMAT 'inputfmt' OUTPUTFORMAT 'outputfmt'" - val query1 = s"$createTableStart SERDE 'anything' $fileFormat" - val query2 = s"$createTableStart DELIMITED FIELDS TERMINATED BY ' ' $fileFormat" - - // No conflicting serdes here, OK - val parsed1 = parseAs[CreateTable](query1) - assert(parsed1.tableDesc.storage.serde == Some("anything")) - assert(parsed1.tableDesc.storage.inputFormat == Some("inputfmt")) - assert(parsed1.tableDesc.storage.outputFormat == Some("outputfmt")) - - val parsed2 = parseAs[CreateTable](query2) - assert(parsed2.tableDesc.storage.serde == - Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")) - assert(parsed2.tableDesc.storage.inputFormat == Some("inputfmt")) - assert(parsed2.tableDesc.storage.outputFormat == Some("outputfmt")) - } - - test("create hive table - row format serde and generic file format") { - val allSources = Seq("parquet", "orc", "avro", "sequencefile", "rcfile", "textfile") - val supportedSources = Set("sequencefile", "rcfile", "textfile") - - allSources.foreach { s => - val query = s"CREATE TABLE my_tab ROW FORMAT SERDE 'anything' STORED AS $s" - if (supportedSources.contains(s)) { - val ct = parseAs[CreateTable](query) - val hiveSerde = HiveSerDe.sourceToSerDe(s) - assert(hiveSerde.isDefined) - assert(ct.tableDesc.storage.serde == Some("anything")) - assert(ct.tableDesc.storage.inputFormat == hiveSerde.get.inputFormat) - assert(ct.tableDesc.storage.outputFormat == hiveSerde.get.outputFormat) - } else { - assertUnsupported(query, Seq("row format serde", "incompatible", s)) - } - } - } - - test("create hive table - row format delimited and generic file format") { - val allSources = Seq("parquet", "orc", "avro", "sequencefile", "rcfile", "textfile") - val supportedSources = Set("textfile") - - allSources.foreach { s => - val query = s"CREATE TABLE my_tab ROW FORMAT DELIMITED FIELDS TERMINATED BY ' ' STORED AS $s" - if (supportedSources.contains(s)) { - val ct = parseAs[CreateTable](query) - val hiveSerde = HiveSerDe.sourceToSerDe(s) - assert(hiveSerde.isDefined) - assert(ct.tableDesc.storage.serde == - hiveSerde.get.serde.orElse(Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))) - assert(ct.tableDesc.storage.inputFormat == hiveSerde.get.inputFormat) - assert(ct.tableDesc.storage.outputFormat == hiveSerde.get.outputFormat) - } else { - assertUnsupported(query, Seq("row format delimited", "only compatible with 'textfile'", s)) - } - } - } - - test("create hive external table - location must be specified") { - assertUnsupported( - sql = "CREATE EXTERNAL TABLE my_tab", - containsThesePhrases = Seq("create external table", "location")) - val query = "CREATE EXTERNAL TABLE my_tab LOCATION '/something/anything'" - val ct = parseAs[CreateTable](query) - assert(ct.tableDesc.tableType == CatalogTableType.EXTERNAL) - assert(ct.tableDesc.storage.locationUri == Some(new URI("/something/anything"))) - } - - test("create hive table - property values must be set") { - assertUnsupported( - sql = "CREATE TABLE my_tab TBLPROPERTIES('key_without_value', 'key_with_value'='x')", - containsThesePhrases = Seq("key_without_value")) - assertUnsupported( - sql = "CREATE TABLE my_tab ROW FORMAT SERDE 'serde' " + - "WITH SERDEPROPERTIES('key_without_value', 'key_with_value'='x')", - containsThesePhrases = Seq("key_without_value")) - } - - test("create hive table - location implies external") { - val query = "CREATE TABLE my_tab LOCATION '/something/anything'" - val ct = parseAs[CreateTable](query) - assert(ct.tableDesc.tableType == CatalogTableType.EXTERNAL) - assert(ct.tableDesc.storage.locationUri == Some(new URI("/something/anything"))) - } - - test("create table - with partitioned by") { - val query = "CREATE TABLE my_tab(a INT comment 'test', b STRING) " + - "USING parquet PARTITIONED BY (a)" - - val expectedTableDesc = CatalogTable( - identifier = TableIdentifier("my_tab"), - tableType = CatalogTableType.MANAGED, - storage = CatalogStorageFormat.empty, - schema = new StructType() - .add("a", IntegerType, nullable = true, "test") - .add("b", StringType), - provider = Some("parquet"), - partitionColumnNames = Seq("a") - ) - - parser.parsePlan(query) match { - case CreateTable(tableDesc, _, None) => - assert(tableDesc == expectedTableDesc.copy(createTime = tableDesc.createTime)) - case other => - fail(s"Expected to parse ${classOf[CreateTableCommand].getClass.getName} from query," + - s"got ${other.getClass.getName}: $query") - } - } - - test("create table - with bucket") { - val query = "CREATE TABLE my_tab(a INT, b STRING) USING parquet " + - "CLUSTERED BY (a) SORTED BY (b) INTO 5 BUCKETS" - - val expectedTableDesc = CatalogTable( - identifier = TableIdentifier("my_tab"), - tableType = CatalogTableType.MANAGED, - storage = CatalogStorageFormat.empty, - schema = new StructType().add("a", IntegerType).add("b", StringType), - provider = Some("parquet"), - bucketSpec = Some(BucketSpec(5, Seq("a"), Seq("b"))) - ) - - parser.parsePlan(query) match { - case CreateTable(tableDesc, _, None) => - assert(tableDesc == expectedTableDesc.copy(createTime = tableDesc.createTime)) - case other => - fail(s"Expected to parse ${classOf[CreateTableCommand].getClass.getName} from query," + - s"got ${other.getClass.getName}: $query") - } - } - - test("create table - with comment") { - val sql = "CREATE TABLE my_tab(a INT, b STRING) USING parquet COMMENT 'abc'" - - val expectedTableDesc = CatalogTable( - identifier = TableIdentifier("my_tab"), - tableType = CatalogTableType.MANAGED, - storage = CatalogStorageFormat.empty, - schema = new StructType().add("a", IntegerType).add("b", StringType), - provider = Some("parquet"), - comment = Some("abc")) - - parser.parsePlan(sql) match { - case CreateTable(tableDesc, _, None) => - assert(tableDesc == expectedTableDesc.copy(createTime = tableDesc.createTime)) - case other => - fail(s"Expected to parse ${classOf[CreateTableCommand].getClass.getName} from query," + - s"got ${other.getClass.getName}: $sql") - } - } - - test("create table - with location") { - val v1 = "CREATE TABLE my_tab(a INT, b STRING) USING parquet LOCATION '/tmp/file'" - - val expectedTableDesc = CatalogTable( - identifier = TableIdentifier("my_tab"), - tableType = CatalogTableType.EXTERNAL, - storage = CatalogStorageFormat.empty.copy(locationUri = Some(new URI("/tmp/file"))), - schema = new StructType().add("a", IntegerType).add("b", StringType), - provider = Some("parquet")) - - parser.parsePlan(v1) match { - case CreateTable(tableDesc, _, None) => - assert(tableDesc == expectedTableDesc.copy(createTime = tableDesc.createTime)) - case other => - fail(s"Expected to parse ${classOf[CreateTableCommand].getClass.getName} from query," + - s"got ${other.getClass.getName}: $v1") - } - - val v2 = - """ - |CREATE TABLE my_tab(a INT, b STRING) - |USING parquet - |OPTIONS (path '/tmp/file') - |LOCATION '/tmp/file' - """.stripMargin - val e = intercept[ParseException] { - parser.parsePlan(v2) - } - assert(e.message.contains("you can only specify one of them.")) - } - - // ALTER TABLE table_name RENAME TO new_table_name; - // ALTER VIEW view_name RENAME TO new_view_name; - test("alter table/view: rename table/view") { - val sql_table = "ALTER TABLE table_name RENAME TO new_table_name" - val sql_view = sql_table.replace("TABLE", "VIEW") - val parsed_table = parser.parsePlan(sql_table) - val parsed_view = parser.parsePlan(sql_view) - val expected_table = AlterTableRenameCommand( - TableIdentifier("table_name"), - TableIdentifier("new_table_name"), - isView = false) - val expected_view = AlterTableRenameCommand( - TableIdentifier("table_name"), - TableIdentifier("new_table_name"), - isView = true) - comparePlans(parsed_table, expected_table) - comparePlans(parsed_view, expected_view) - } - - test("alter table: rename table with database") { - val query = "ALTER TABLE db1.tbl RENAME TO db1.tbl2" - val plan = parseAs[AlterTableRenameCommand](query) - assert(plan.oldName == TableIdentifier("tbl", Some("db1"))) - assert(plan.newName == TableIdentifier("tbl2", Some("db1"))) - } - - // ALTER TABLE table_name SET TBLPROPERTIES ('comment' = new_comment); - // ALTER TABLE table_name UNSET TBLPROPERTIES [IF EXISTS] ('comment', 'key'); - // ALTER VIEW view_name SET TBLPROPERTIES ('comment' = new_comment); - // ALTER VIEW view_name UNSET TBLPROPERTIES [IF EXISTS] ('comment', 'key'); - test("alter table/view: alter table/view properties") { - val sql1_table = "ALTER TABLE table_name SET TBLPROPERTIES ('test' = 'test', " + - "'comment' = 'new_comment')" - val sql2_table = "ALTER TABLE table_name UNSET TBLPROPERTIES ('comment', 'test')" - val sql3_table = "ALTER TABLE table_name UNSET TBLPROPERTIES IF EXISTS ('comment', 'test')" - val sql1_view = sql1_table.replace("TABLE", "VIEW") - val sql2_view = sql2_table.replace("TABLE", "VIEW") - val sql3_view = sql3_table.replace("TABLE", "VIEW") - - val parsed1_table = parser.parsePlan(sql1_table) - val parsed2_table = parser.parsePlan(sql2_table) - val parsed3_table = parser.parsePlan(sql3_table) - val parsed1_view = parser.parsePlan(sql1_view) - val parsed2_view = parser.parsePlan(sql2_view) - val parsed3_view = parser.parsePlan(sql3_view) - - val tableIdent = TableIdentifier("table_name", None) - val expected1_table = AlterTableSetPropertiesCommand( - tableIdent, Map("test" -> "test", "comment" -> "new_comment"), isView = false) - val expected2_table = AlterTableUnsetPropertiesCommand( - tableIdent, Seq("comment", "test"), ifExists = false, isView = false) - val expected3_table = AlterTableUnsetPropertiesCommand( - tableIdent, Seq("comment", "test"), ifExists = true, isView = false) - val expected1_view = expected1_table.copy(isView = true) - val expected2_view = expected2_table.copy(isView = true) - val expected3_view = expected3_table.copy(isView = true) - - comparePlans(parsed1_table, expected1_table) - comparePlans(parsed2_table, expected2_table) - comparePlans(parsed3_table, expected3_table) - comparePlans(parsed1_view, expected1_view) - comparePlans(parsed2_view, expected2_view) - comparePlans(parsed3_view, expected3_view) - } - - test("alter table - property values must be set") { - assertUnsupported( - sql = "ALTER TABLE my_tab SET TBLPROPERTIES('key_without_value', 'key_with_value'='x')", - containsThesePhrases = Seq("key_without_value")) - } - - test("alter table unset properties - property values must NOT be set") { - assertUnsupported( - sql = "ALTER TABLE my_tab UNSET TBLPROPERTIES('key_without_value', 'key_with_value'='x')", - containsThesePhrases = Seq("key_with_value")) - } - - test("alter table: SerDe properties") { - val sql1 = "ALTER TABLE table_name SET SERDE 'org.apache.class'" - val sql2 = - """ - |ALTER TABLE table_name SET SERDE 'org.apache.class' - |WITH SERDEPROPERTIES ('columns'='foo,bar', 'field.delim' = ',') - """.stripMargin - val sql3 = - """ - |ALTER TABLE table_name SET SERDEPROPERTIES ('columns'='foo,bar', - |'field.delim' = ',') - """.stripMargin - val sql4 = - """ - |ALTER TABLE table_name PARTITION (test=1, dt='2008-08-08', - |country='us') SET SERDE 'org.apache.class' WITH SERDEPROPERTIES ('columns'='foo,bar', - |'field.delim' = ',') - """.stripMargin - val sql5 = - """ - |ALTER TABLE table_name PARTITION (test=1, dt='2008-08-08', - |country='us') SET SERDEPROPERTIES ('columns'='foo,bar', 'field.delim' = ',') - """.stripMargin - val parsed1 = parser.parsePlan(sql1) - val parsed2 = parser.parsePlan(sql2) - val parsed3 = parser.parsePlan(sql3) - val parsed4 = parser.parsePlan(sql4) - val parsed5 = parser.parsePlan(sql5) - val tableIdent = TableIdentifier("table_name", None) - val expected1 = AlterTableSerDePropertiesCommand( - tableIdent, Some("org.apache.class"), None, None) - val expected2 = AlterTableSerDePropertiesCommand( - tableIdent, - Some("org.apache.class"), - Some(Map("columns" -> "foo,bar", "field.delim" -> ",")), - None) - val expected3 = AlterTableSerDePropertiesCommand( - tableIdent, None, Some(Map("columns" -> "foo,bar", "field.delim" -> ",")), None) - val expected4 = AlterTableSerDePropertiesCommand( - tableIdent, - Some("org.apache.class"), - Some(Map("columns" -> "foo,bar", "field.delim" -> ",")), - Some(Map("test" -> "1", "dt" -> "2008-08-08", "country" -> "us"))) - val expected5 = AlterTableSerDePropertiesCommand( - tableIdent, - None, - Some(Map("columns" -> "foo,bar", "field.delim" -> ",")), - Some(Map("test" -> "1", "dt" -> "2008-08-08", "country" -> "us"))) - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - comparePlans(parsed3, expected3) - comparePlans(parsed4, expected4) - comparePlans(parsed5, expected5) - } - - test("alter table - SerDe property values must be set") { - assertUnsupported( - sql = "ALTER TABLE my_tab SET SERDE 'serde' " + - "WITH SERDEPROPERTIES('key_without_value', 'key_with_value'='x')", - containsThesePhrases = Seq("key_without_value")) - } - - // ALTER TABLE table_name ADD [IF NOT EXISTS] PARTITION partition_spec - // [LOCATION 'location1'] partition_spec [LOCATION 'location2'] ...; - test("alter table: add partition") { - val sql1 = - """ - |ALTER TABLE table_name ADD IF NOT EXISTS PARTITION - |(dt='2008-08-08', country='us') LOCATION 'location1' PARTITION - |(dt='2009-09-09', country='uk') - """.stripMargin - val sql2 = "ALTER TABLE table_name ADD PARTITION (dt='2008-08-08') LOCATION 'loc'" - - val parsed1 = parser.parsePlan(sql1) - val parsed2 = parser.parsePlan(sql2) - - val expected1 = AlterTableAddPartitionCommand( - TableIdentifier("table_name", None), - Seq( - (Map("dt" -> "2008-08-08", "country" -> "us"), Some("location1")), - (Map("dt" -> "2009-09-09", "country" -> "uk"), None)), - ifNotExists = true) - val expected2 = AlterTableAddPartitionCommand( - TableIdentifier("table_name", None), - Seq((Map("dt" -> "2008-08-08"), Some("loc"))), - ifNotExists = false) - - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - } - - test("alter table: recover partitions") { - val sql = "ALTER TABLE table_name RECOVER PARTITIONS" - val parsed = parser.parsePlan(sql) - val expected = AlterTableRecoverPartitionsCommand( - TableIdentifier("table_name", None)) - comparePlans(parsed, expected) - } - - test("alter view: add partition (not supported)") { - assertUnsupported( - """ - |ALTER VIEW view_name ADD IF NOT EXISTS PARTITION - |(dt='2008-08-08', country='us') PARTITION - |(dt='2009-09-09', country='uk') - """.stripMargin) - } - - test("alter table: rename partition") { - val sql = - """ - |ALTER TABLE table_name PARTITION (dt='2008-08-08', country='us') - |RENAME TO PARTITION (dt='2008-09-09', country='uk') - """.stripMargin - val parsed = parser.parsePlan(sql) - val expected = AlterTableRenamePartitionCommand( - TableIdentifier("table_name", None), - Map("dt" -> "2008-08-08", "country" -> "us"), - Map("dt" -> "2008-09-09", "country" -> "uk")) - comparePlans(parsed, expected) - } - - test("alter table: exchange partition (not supported)") { - assertUnsupported( - """ - |ALTER TABLE table_name_1 EXCHANGE PARTITION - |(dt='2008-08-08', country='us') WITH TABLE table_name_2 - """.stripMargin) - } - - // ALTER TABLE table_name DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...] - // ALTER VIEW table_name DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...] - test("alter table/view: drop partitions") { - val sql1_table = - """ - |ALTER TABLE table_name DROP IF EXISTS PARTITION - |(dt='2008-08-08', country='us'), PARTITION (dt='2009-09-09', country='uk') - """.stripMargin - val sql2_table = - """ - |ALTER TABLE table_name DROP PARTITION - |(dt='2008-08-08', country='us'), PARTITION (dt='2009-09-09', country='uk') - """.stripMargin - val sql1_view = sql1_table.replace("TABLE", "VIEW") - val sql2_view = sql2_table.replace("TABLE", "VIEW") - - val parsed1_table = parser.parsePlan(sql1_table) - val parsed2_table = parser.parsePlan(sql2_table) - val parsed1_purge = parser.parsePlan(sql1_table + " PURGE") - assertUnsupported(sql1_view) - assertUnsupported(sql2_view) - - val tableIdent = TableIdentifier("table_name", None) - val expected1_table = AlterTableDropPartitionCommand( - tableIdent, - Seq( - Map("dt" -> "2008-08-08", "country" -> "us"), - Map("dt" -> "2009-09-09", "country" -> "uk")), - ifExists = true, - purge = false, - retainData = false) - val expected2_table = expected1_table.copy(ifExists = false) - val expected1_purge = expected1_table.copy(purge = true) - - comparePlans(parsed1_table, expected1_table) - comparePlans(parsed2_table, expected2_table) - comparePlans(parsed1_purge, expected1_purge) - } - - test("alter table: archive partition (not supported)") { - assertUnsupported("ALTER TABLE table_name ARCHIVE PARTITION (dt='2008-08-08', country='us')") - } - - test("alter table: unarchive partition (not supported)") { - assertUnsupported("ALTER TABLE table_name UNARCHIVE PARTITION (dt='2008-08-08', country='us')") - } - - test("alter table: set file format (not allowed)") { - assertUnsupported( - "ALTER TABLE table_name SET FILEFORMAT INPUTFORMAT 'test' OUTPUTFORMAT 'test'") - assertUnsupported( - "ALTER TABLE table_name PARTITION (dt='2008-08-08', country='us') " + - "SET FILEFORMAT PARQUET") - } - - test("alter table: set location") { - val sql1 = "ALTER TABLE table_name SET LOCATION 'new location'" - val sql2 = "ALTER TABLE table_name PARTITION (dt='2008-08-08', country='us') " + - "SET LOCATION 'new location'" - val parsed1 = parser.parsePlan(sql1) - val parsed2 = parser.parsePlan(sql2) - val tableIdent = TableIdentifier("table_name", None) - val expected1 = AlterTableSetLocationCommand( - tableIdent, - None, - "new location") - val expected2 = AlterTableSetLocationCommand( - tableIdent, - Some(Map("dt" -> "2008-08-08", "country" -> "us")), - "new location") - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - } - - test("alter table: change column name/type/comment") { - val sql1 = "ALTER TABLE table_name CHANGE COLUMN col_old_name col_new_name INT" - val sql2 = "ALTER TABLE table_name CHANGE COLUMN col_name col_name INT COMMENT 'new_comment'" - val parsed1 = parser.parsePlan(sql1) - val parsed2 = parser.parsePlan(sql2) - val tableIdent = TableIdentifier("table_name", None) - val expected1 = AlterTableChangeColumnCommand( - tableIdent, - "col_old_name", - StructField("col_new_name", IntegerType)) - val expected2 = AlterTableChangeColumnCommand( - tableIdent, - "col_name", - StructField("col_name", IntegerType).withComment("new_comment")) - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - } - - test("alter table: change column position (not supported)") { - assertUnsupported("ALTER TABLE table_name CHANGE COLUMN col_old_name col_new_name INT FIRST") - assertUnsupported( - "ALTER TABLE table_name CHANGE COLUMN col_old_name col_new_name INT AFTER other_col") - } - - test("alter table: change column in partition spec") { - assertUnsupported("ALTER TABLE table_name PARTITION (a='1', a='2') CHANGE COLUMN a new_a INT") - } - - test("alter table: touch (not supported)") { - assertUnsupported("ALTER TABLE table_name TOUCH") - assertUnsupported("ALTER TABLE table_name TOUCH PARTITION (dt='2008-08-08', country='us')") - } - - test("alter table: compact (not supported)") { - assertUnsupported("ALTER TABLE table_name COMPACT 'compaction_type'") - assertUnsupported( - """ - |ALTER TABLE table_name PARTITION (dt='2008-08-08', country='us') - |COMPACT 'MAJOR' - """.stripMargin) - } - - test("alter table: concatenate (not supported)") { - assertUnsupported("ALTER TABLE table_name CONCATENATE") - assertUnsupported( - "ALTER TABLE table_name PARTITION (dt='2008-08-08', country='us') CONCATENATE") - } - - test("alter table: cluster by (not supported)") { - assertUnsupported( - "ALTER TABLE table_name CLUSTERED BY (col_name) SORTED BY (col2_name) INTO 3 BUCKETS") - assertUnsupported("ALTER TABLE table_name CLUSTERED BY (col_name) INTO 3 BUCKETS") - assertUnsupported("ALTER TABLE table_name NOT CLUSTERED") - assertUnsupported("ALTER TABLE table_name NOT SORTED") - } - - test("alter table: skewed by (not supported)") { - assertUnsupported("ALTER TABLE table_name NOT SKEWED") - assertUnsupported("ALTER TABLE table_name NOT STORED AS DIRECTORIES") - assertUnsupported("ALTER TABLE table_name SET SKEWED LOCATION (col_name1=\"location1\"") - assertUnsupported("ALTER TABLE table_name SKEWED BY (key) ON (1,5,6) STORED AS DIRECTORIES") - } - - test("alter table: replace columns (not allowed)") { - assertUnsupported( - """ - |ALTER TABLE table_name REPLACE COLUMNS (new_col1 INT - |COMMENT 'test_comment', new_col2 LONG COMMENT 'test_comment2') RESTRICT - """.stripMargin) - } - - test("show databases") { - val sql1 = "SHOW DATABASES" - val sql2 = "SHOW DATABASES LIKE 'defau*'" - val parsed1 = parser.parsePlan(sql1) - val expected1 = ShowDatabasesCommand(None) - val parsed2 = parser.parsePlan(sql2) - val expected2 = ShowDatabasesCommand(Some("defau*")) - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - } - - test("show tblproperties") { - val parsed1 = parser.parsePlan("SHOW TBLPROPERTIES tab1") - val expected1 = ShowTablePropertiesCommand(TableIdentifier("tab1", None), None) - val parsed2 = parser.parsePlan("SHOW TBLPROPERTIES tab1('propKey1')") - val expected2 = ShowTablePropertiesCommand(TableIdentifier("tab1", None), Some("propKey1")) - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - } - - test("SPARK-14383: DISTRIBUTE and UNSET as non-keywords") { - val sql = "SELECT distribute, unset FROM x" - val parsed = parser.parsePlan(sql) - assert(parsed.isInstanceOf[Project]) - } - - test("duplicate keys in table properties") { - val e = intercept[ParseException] { - parser.parsePlan("ALTER TABLE dbx.tab1 SET TBLPROPERTIES ('key1' = '1', 'key1' = '2')") - }.getMessage - assert(e.contains("Found duplicate keys 'key1'")) - } - - test("duplicate columns in partition specs") { - val e = intercept[ParseException] { - parser.parsePlan( - "ALTER TABLE dbx.tab1 PARTITION (a='1', a='2') RENAME TO PARTITION (a='100', a='200')") - }.getMessage - assert(e.contains("Found duplicate keys 'a'")) - } - - test("empty values in non-optional partition specs") { - val e = intercept[ParseException] { - parser.parsePlan( - "SHOW PARTITIONS dbx.tab1 PARTITION (a='1', b)") - }.getMessage - assert(e.contains("Found an empty partition key 'b'")) - } - - test("drop table") { - val tableName1 = "db.tab" - val tableName2 = "tab" - - val parsed = Seq( - s"DROP TABLE $tableName1", - s"DROP TABLE IF EXISTS $tableName1", - s"DROP TABLE $tableName2", - s"DROP TABLE IF EXISTS $tableName2", - s"DROP TABLE $tableName2 PURGE", - s"DROP TABLE IF EXISTS $tableName2 PURGE" - ).map(parser.parsePlan) - - val expected = Seq( - DropTableCommand(TableIdentifier("tab", Option("db")), ifExists = false, isView = false, - purge = false), - DropTableCommand(TableIdentifier("tab", Option("db")), ifExists = true, isView = false, - purge = false), - DropTableCommand(TableIdentifier("tab", None), ifExists = false, isView = false, - purge = false), - DropTableCommand(TableIdentifier("tab", None), ifExists = true, isView = false, - purge = false), - DropTableCommand(TableIdentifier("tab", None), ifExists = false, isView = false, - purge = true), - DropTableCommand(TableIdentifier("tab", None), ifExists = true, isView = false, - purge = true)) - - parsed.zip(expected).foreach { case (p, e) => comparePlans(p, e) } - } - - test("drop view") { - val viewName1 = "db.view" - val viewName2 = "view" - - val parsed1 = parser.parsePlan(s"DROP VIEW $viewName1") - val parsed2 = parser.parsePlan(s"DROP VIEW IF EXISTS $viewName1") - val parsed3 = parser.parsePlan(s"DROP VIEW $viewName2") - val parsed4 = parser.parsePlan(s"DROP VIEW IF EXISTS $viewName2") - - val expected1 = - DropTableCommand(TableIdentifier("view", Option("db")), ifExists = false, isView = true, - purge = false) - val expected2 = - DropTableCommand(TableIdentifier("view", Option("db")), ifExists = true, isView = true, - purge = false) - val expected3 = - DropTableCommand(TableIdentifier("view", None), ifExists = false, isView = true, - purge = false) - val expected4 = - DropTableCommand(TableIdentifier("view", None), ifExists = true, isView = true, - purge = false) - - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - comparePlans(parsed3, expected3) - comparePlans(parsed4, expected4) - } - - test("show columns") { - val sql1 = "SHOW COLUMNS FROM t1" - val sql2 = "SHOW COLUMNS IN db1.t1" - val sql3 = "SHOW COLUMNS FROM t1 IN db1" - val sql4 = "SHOW COLUMNS FROM db1.t1 IN db2" - - val parsed1 = parser.parsePlan(sql1) - val expected1 = ShowColumnsCommand(None, TableIdentifier("t1", None)) - val parsed2 = parser.parsePlan(sql2) - val expected2 = ShowColumnsCommand(None, TableIdentifier("t1", Some("db1"))) - val parsed3 = parser.parsePlan(sql3) - val expected3 = ShowColumnsCommand(Some("db1"), TableIdentifier("t1", None)) - val parsed4 = parser.parsePlan(sql4) - val expected4 = ShowColumnsCommand(Some("db2"), TableIdentifier("t1", Some("db1"))) - - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - comparePlans(parsed3, expected3) - comparePlans(parsed4, expected4) - } - - - test("show partitions") { - val sql1 = "SHOW PARTITIONS t1" - val sql2 = "SHOW PARTITIONS db1.t1" - val sql3 = "SHOW PARTITIONS t1 PARTITION(partcol1='partvalue', partcol2='partvalue')" - - val parsed1 = parser.parsePlan(sql1) - val expected1 = - ShowPartitionsCommand(TableIdentifier("t1", None), None) - val parsed2 = parser.parsePlan(sql2) - val expected2 = - ShowPartitionsCommand(TableIdentifier("t1", Some("db1")), None) - val expected3 = - ShowPartitionsCommand(TableIdentifier("t1", None), - Some(Map("partcol1" -> "partvalue", "partcol2" -> "partvalue"))) - val parsed3 = parser.parsePlan(sql3) - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - comparePlans(parsed3, expected3) - } - - test("support for other types in DBPROPERTIES") { - val sql = - """ - |CREATE DATABASE database_name - |LOCATION '/home/user/db' - |WITH DBPROPERTIES ('a'=1, 'b'=0.1, 'c'=TRUE) - """.stripMargin - val parsed = parser.parsePlan(sql) - val expected = CreateDatabaseCommand( - "database_name", - ifNotExists = false, - Some("/home/user/db"), - None, - Map("a" -> "1", "b" -> "0.1", "c" -> "true")) - - comparePlans(parsed, expected) - } - - test("support for other types in TBLPROPERTIES") { - val sql = - """ - |ALTER TABLE table_name - |SET TBLPROPERTIES ('a' = 1, 'b' = 0.1, 'c' = TRUE) - """.stripMargin - val parsed = parser.parsePlan(sql) - val expected = AlterTableSetPropertiesCommand( - TableIdentifier("table_name"), - Map("a" -> "1", "b" -> "0.1", "c" -> "true"), - isView = false) - - comparePlans(parsed, expected) - } - - test("support for other types in OPTIONS") { - val sql = - """ - |CREATE TABLE table_name USING json - |OPTIONS (a 1, b 0.1, c TRUE) - """.stripMargin - - val expectedTableDesc = CatalogTable( - identifier = TableIdentifier("table_name"), - tableType = CatalogTableType.MANAGED, - storage = CatalogStorageFormat.empty.copy( - properties = Map("a" -> "1", "b" -> "0.1", "c" -> "true") - ), - schema = new StructType, - provider = Some("json") - ) - - parser.parsePlan(sql) match { - case CreateTable(tableDesc, _, None) => - assert(tableDesc == expectedTableDesc.copy(createTime = tableDesc.createTime)) - case other => - fail(s"Expected to parse ${classOf[CreateTableCommand].getClass.getName} from query," + - s"got ${other.getClass.getName}: $sql") - } - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
