This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 0601fc7 [SPARK-33118][SQL] CREATE TEMPORARY TABLE fails with location
0601fc7 is described below
commit 0601fc7bfc85b76be960c0b4ce642fd39675926c
Author: Pablo <[email protected]>
AuthorDate: Mon Oct 12 14:18:34 2020 -0700
[SPARK-33118][SQL] CREATE TEMPORARY TABLE fails with location
We have a problem when you use CREATE TEMPORARY TABLE with LOCATION
```scala
spark.range(3).write.parquet("/tmp/testspark1")
sql("CREATE TEMPORARY TABLE t USING parquet OPTIONS (path
'/tmp/testspark1')")
sql("CREATE TEMPORARY TABLE t USING parquet LOCATION '/tmp/testspark1'")
```
```scala
org.apache.spark.sql.AnalysisException: Unable to infer schema for Parquet.
It must be specified manually.;
at
org.apache.spark.sql.execution.datasources.DataSource.$anonfun$getOrInferFileFormatSchema$12(DataSource.scala:200)
at scala.Option.getOrElse(Option.scala:189)
at
org.apache.spark.sql.execution.datasources.DataSource.getOrInferFileFormatSchema(DataSource.scala:200)
at
org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:408)
at
org.apache.spark.sql.execution.datasources.CreateTempViewUsing.run(ddl.scala:94)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:79)
at org.apache.spark.sql.Dataset.$anonfun$logicalPlan$1(Dataset.scala:229)
at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3618)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)
at
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3616)
at org.apache.spark.sql.Dataset.<init>(Dataset.scala:229)
at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:100)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)
at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:97)
at
org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:607)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)
at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:602)
```
This bug was introduced by SPARK-30507.
sparksqlparser --> visitCreateTable --> visitCreateTableClauses -->
cleanTableOptions extract the path from the options but in this case
CreateTempViewUsing need the path in the options map.
To fix the problem
No
Unit testing and manual testing
Closes #30014 from planga82/bugfix/SPARK-33118_create_temp_table_location.
Authored-by: Pablo <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 819f12ee2fe3cce0c59221c2b02831274c769b23)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../org/apache/spark/sql/execution/SparkSqlParser.scala | 6 ++++--
.../apache/spark/sql/execution/SparkSqlParserSuite.scala | 13 +++++++++++--
2 files changed, 15 insertions(+), 4 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
index 44069f3..3e4bc48 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
@@ -196,7 +196,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends
AstBuilder(conf) {
operationNotAllowed("CREATE TEMPORARY TABLE IF NOT EXISTS", ctx)
}
- val (_, _, _, options, _, _) =
visitCreateTableClauses(ctx.createTableClauses())
+ val (_, _, _, options, location, _) =
visitCreateTableClauses(ctx.createTableClauses())
val provider =
Option(ctx.tableProvider).map(_.multipartIdentifier.getText).getOrElse(
throw new ParseException("CREATE TEMPORARY TABLE without a provider is
not allowed.", ctx))
val schema = Option(ctx.colTypeList()).map(createSchema)
@@ -205,7 +205,9 @@ class SparkSqlAstBuilder(conf: SQLConf) extends
AstBuilder(conf) {
"CREATE TEMPORARY VIEW ... USING ... instead")
val table = tableIdentifier(ident, "CREATE TEMPORARY VIEW", ctx)
- CreateTempViewUsing(table, schema, replace = false, global = false,
provider, options)
+ val optionsWithLocation = location.map(l => options + ("path" ->
l)).getOrElse(options)
+ CreateTempViewUsing(table, schema, replace = false, global = false,
provider,
+ optionsWithLocation)
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
index 6a58c8f..343d3c1 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
@@ -25,8 +25,8 @@ import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.expressions.{Ascending,
AttributeReference, Concat, SortOrder}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.datasources.{CreateTable,
RefreshResource}
-import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
+import org.apache.spark.sql.execution.datasources.{CreateTable,
CreateTempViewUsing, RefreshResource}
+import org.apache.spark.sql.internal.{HiveSerDe, SQLConf, StaticSQLConf}
import org.apache.spark.sql.types.{IntegerType, LongType, StringType,
StructType}
/**
@@ -81,6 +81,15 @@ class SparkSqlParserSuite extends AnalysisTest {
intercept("REFRESH", "Resource paths cannot be empty in REFRESH
statements")
}
+ test("SPARK-33118 CREATE TMEPORARY TABLE with LOCATION") {
+ assertEqual("CREATE TEMPORARY TABLE t USING parquet OPTIONS (path
'/data/tmp/testspark1')",
+ CreateTempViewUsing(TableIdentifier("t", None), None, false, false,
"parquet",
+ Map("path" -> "/data/tmp/testspark1")))
+ assertEqual("CREATE TEMPORARY TABLE t USING parquet LOCATION
'/data/tmp/testspark1'",
+ CreateTempViewUsing(TableIdentifier("t", None), None, false, false,
"parquet",
+ Map("path" -> "/data/tmp/testspark1")))
+ }
+
private def createTableUsing(
table: String,
database: Option[String] = None,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
