This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1e9b0f6da96 [SPARK-43836][BUILD] Make Scala 2.13 as default in Spark
3.5
1e9b0f6da96 is described below
commit 1e9b0f6da967647f6f9f8125e1cf4b45736d346c
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sun May 28 11:21:52 2023 -0700
[SPARK-43836][BUILD] Make Scala 2.13 as default in Spark 3.5
### What changes were proposed in this pull request?
This PR aims to make Scala 2.13 the default Scala version in Apache Spark
3.5.
### Why are the changes needed?
The current releases of Scala community are `Scala 3.2.2` and `Scala
2.13.10`.
- https://scala-lang.org/download/all.html
Although the Apache Spark community has been using Scala 2.12 by default
since Apache Spark 3.0 and Scala community will release Scala 2.12.18 for Java
21 support, we had better focus on `Scala 2.13+` more from Apache Spark 3.5
timeline to adopt Scala community's activity.
Since SPARK-25075 added Scala 2.13 at Apache Spark 3.2.0, the Apache Spark
community has been using it as a second Scala version. This PR aims to switch
only the default Scala version from 2.12 to 2.13. Apache Spark will support
both Scala 2.12 and 2.13 still.
### Does this PR introduce _any_ user-facing change?
Yes, but we still have Scala 2.12.
### How was this patch tested?
Pass the CIs.
Closes #41344 from dongjoon-hyun/SPARK-43836.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
assembly/pom.xml | 4 +-
common/kvstore/pom.xml | 4 +-
common/network-common/pom.xml | 4 +-
common/network-shuffle/pom.xml | 4 +-
common/network-yarn/pom.xml | 4 +-
common/sketch/pom.xml | 4 +-
common/tags/pom.xml | 4 +-
common/unsafe/pom.xml | 4 +-
common/utils/pom.xml | 4 +-
connector/avro/pom.xml | 8 +-
connector/connect/client/jvm/pom.xml | 4 +-
connector/connect/common/pom.xml | 4 +-
connector/connect/server/pom.xml | 8 +-
connector/docker-integration-tests/pom.xml | 4 +-
connector/kafka-0-10-assembly/pom.xml | 4 +-
connector/kafka-0-10-sql/pom.xml | 8 +-
connector/kafka-0-10-token-provider/pom.xml | 4 +-
connector/kafka-0-10/pom.xml | 8 +-
connector/kinesis-asl-assembly/pom.xml | 4 +-
connector/kinesis-asl/pom.xml | 4 +-
connector/protobuf/pom.xml | 8 +-
connector/spark-ganglia-lgpl/pom.xml | 4 +-
core/pom.xml | 8 +-
dev/deps/spark-deps-hadoop-3-hive-2.3 | 43 +++----
dev/mima | 8 +-
dev/test-dependencies.sh | 2 +-
docs/_plugins/copy_api_dirs.rb | 14 +--
examples/pom.xml | 4 +-
graphx/pom.xml | 4 +-
hadoop-cloud/pom.xml | 4 +-
launcher/pom.xml | 4 +-
mllib-local/pom.xml | 4 +-
mllib/pom.xml | 8 +-
pom.xml | 125 ++++++++++-----------
repl/pom.xml | 4 +-
resource-managers/kubernetes/core/pom.xml | 4 +-
.../kubernetes/integration-tests/pom.xml | 4 +-
resource-managers/mesos/pom.xml | 4 +-
resource-managers/yarn/pom.xml | 4 +-
sql/catalyst/pom.xml | 8 +-
sql/core/pom.xml | 8 +-
sql/hive-thriftserver/pom.xml | 8 +-
sql/hive/pom.xml | 8 +-
streaming/pom.xml | 8 +-
tools/pom.xml | 4 +-
45 files changed, 197 insertions(+), 203 deletions(-)
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 09d6bd8a33f..d4d7a1db4a2 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>spark-assembly_2.12</artifactId>
+ <artifactId>spark-assembly_2.13</artifactId>
<name>Spark Project Assembly</name>
<url>https://spark.apache.org/</url>
<packaging>pom</packaging>
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index bef8303874b..69f9f186e08 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-kvstore_2.12</artifactId>
+ <artifactId>spark-kvstore_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Local DB</name>
<url>https://spark.apache.org/</url>
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index 8a63e999c53..9f90d12216e 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-network-common_2.12</artifactId>
+ <artifactId>spark-network-common_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Networking</name>
<url>https://spark.apache.org/</url>
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index a8bde14a259..864f1cc2d37 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-network-shuffle_2.12</artifactId>
+ <artifactId>spark-network-shuffle_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Shuffle Streaming Service</name>
<url>https://spark.apache.org/</url>
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index 671d5cb7e01..c19ac33afa5 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-network-yarn_2.12</artifactId>
+ <artifactId>spark-network-yarn_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project YARN Shuffle Service</name>
<url>https://spark.apache.org/</url>
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 4cc597519c3..6cf1a4fb83e 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-sketch_2.12</artifactId>
+ <artifactId>spark-sketch_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Sketch</name>
<url>https://spark.apache.org/</url>
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index 9a44c847d8a..1eb8352e32d 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-tags_2.12</artifactId>
+ <artifactId>spark-tags_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Tags</name>
<url>https://spark.apache.org/</url>
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index a61f00084eb..e37f1b80a50 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-unsafe_2.12</artifactId>
+ <artifactId>spark-unsafe_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Unsafe</name>
<url>https://spark.apache.org/</url>
diff --git a/common/utils/pom.xml b/common/utils/pom.xml
index 8d24d888e6d..53df20b646d 100644
--- a/common/utils/pom.xml
+++ b/common/utils/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-common-utils_2.12</artifactId>
+ <artifactId>spark-common-utils_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Common Utils</name>
<url>https://spark.apache.org/</url>
diff --git a/connector/avro/pom.xml b/connector/avro/pom.xml
index 597e3c2235f..7087fdbccd0 100644
--- a/connector/avro/pom.xml
+++ b/connector/avro/pom.xml
@@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-avro_2.12</artifactId>
+ <artifactId>spark-avro_2.13</artifactId>
<properties>
<sbt.project.name>avro</sbt.project.name>
</properties>
@@ -70,12 +70,12 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
- <!-- #if scala-2.13 --><!--
+ <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- --><!-- #endif scala-2.13 -->
+ <!-- #endif scala-2.13 -->
<dependency>
<groupId>org.tukaani</groupId>
<artifactId>xz</artifactId>
diff --git a/connector/connect/client/jvm/pom.xml
b/connector/connect/client/jvm/pom.xml
index 4d0a4379329..8dc4c3b7a19 100644
--- a/connector/connect/client/jvm/pom.xml
+++ b/connector/connect/client/jvm/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../../../pom.xml</relativePath>
</parent>
- <artifactId>spark-connect-client-jvm_2.12</artifactId>
+ <artifactId>spark-connect-client-jvm_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Connect Client</name>
<url>https://spark.apache.org/</url>
diff --git a/connector/connect/common/pom.xml b/connector/connect/common/pom.xml
index 06076646df7..fcd72480e0e 100644
--- a/connector/connect/common/pom.xml
+++ b/connector/connect/common/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
- <artifactId>spark-connect-common_2.12</artifactId>
+ <artifactId>spark-connect-common_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Connect Common</name>
<url>https://spark.apache.org/</url>
diff --git a/connector/connect/server/pom.xml b/connector/connect/server/pom.xml
index f9dc40ce85a..f3b43e14cda 100644
--- a/connector/connect/server/pom.xml
+++ b/connector/connect/server/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
- <artifactId>spark-connect_2.12</artifactId>
+ <artifactId>spark-connect_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Connect Server</name>
<url>https://spark.apache.org/</url>
@@ -156,12 +156,12 @@
</exclusion>
</exclusions>
</dependency>
- <!-- #if scala-2.13 --><!--
+ <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- --><!-- #endif scala-2.13 -->
+ <!-- #endif scala-2.13 -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
diff --git a/connector/docker-integration-tests/pom.xml
b/connector/docker-integration-tests/pom.xml
index cc549487a8b..e4026983852 100644
--- a/connector/docker-integration-tests/pom.xml
+++ b/connector/docker-integration-tests/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-docker-integration-tests_2.12</artifactId>
+ <artifactId>spark-docker-integration-tests_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Docker Integration Tests</name>
<url>https://spark.apache.org/</url>
diff --git a/connector/kafka-0-10-assembly/pom.xml
b/connector/kafka-0-10-assembly/pom.xml
index 340974cc789..f339e8c2e4f 100644
--- a/connector/kafka-0-10-assembly/pom.xml
+++ b/connector/kafka-0-10-assembly/pom.xml
@@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-streaming-kafka-0-10-assembly_2.12</artifactId>
+ <artifactId>spark-streaming-kafka-0-10-assembly_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Integration for Kafka 0.10 Assembly</name>
<url>https://spark.apache.org/</url>
diff --git a/connector/kafka-0-10-sql/pom.xml b/connector/kafka-0-10-sql/pom.xml
index fdd1196cd44..f5a12b61c2b 100644
--- a/connector/kafka-0-10-sql/pom.xml
+++ b/connector/kafka-0-10-sql/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql-kafka-0-10_2.12</artifactId>
+ <artifactId>spark-sql-kafka-0-10_2.13</artifactId>
<properties>
<sbt.project.name>sql-kafka-0-10</sbt.project.name>
</properties>
@@ -74,12 +74,12 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <!-- #if scala-2.13 --><!--
+ <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- --><!-- #endif scala-2.13 -->
+ <!-- #endif scala-2.13 -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
diff --git a/connector/kafka-0-10-token-provider/pom.xml
b/connector/kafka-0-10-token-provider/pom.xml
index 3256130c50f..b3c0889d947 100644
--- a/connector/kafka-0-10-token-provider/pom.xml
+++ b/connector/kafka-0-10-token-provider/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-token-provider-kafka-0-10_2.12</artifactId>
+ <artifactId>spark-token-provider-kafka-0-10_2.13</artifactId>
<properties>
<sbt.project.name>token-provider-kafka-0-10</sbt.project.name>
</properties>
diff --git a/connector/kafka-0-10/pom.xml b/connector/kafka-0-10/pom.xml
index 706eb2dd2c3..f1820bb595a 100644
--- a/connector/kafka-0-10/pom.xml
+++ b/connector/kafka-0-10/pom.xml
@@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-streaming-kafka-0-10_2.12</artifactId>
+ <artifactId>spark-streaming-kafka-0-10_2.13</artifactId>
<properties>
<sbt.project.name>streaming-kafka-0-10</sbt.project.name>
</properties>
@@ -59,12 +59,12 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <!-- #if scala-2.13 --><!--
+ <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- --><!-- #endif scala-2.13 -->
+ <!-- #endif scala-2.13 -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
diff --git a/connector/kinesis-asl-assembly/pom.xml
b/connector/kinesis-asl-assembly/pom.xml
index cd5c0393f6f..2cba2668f04 100644
--- a/connector/kinesis-asl-assembly/pom.xml
+++ b/connector/kinesis-asl-assembly/pom.xml
@@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-streaming-kinesis-asl-assembly_2.12</artifactId>
+ <artifactId>spark-streaming-kinesis-asl-assembly_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Kinesis Assembly</name>
<url>https://spark.apache.org/</url>
diff --git a/connector/kinesis-asl/pom.xml b/connector/kinesis-asl/pom.xml
index c70a073e734..af9cd4b7ec9 100644
--- a/connector/kinesis-asl/pom.xml
+++ b/connector/kinesis-asl/pom.xml
@@ -19,13 +19,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<!-- Kinesis integration is not included by default due to ASL-licensed
code. -->
- <artifactId>spark-streaming-kinesis-asl_2.12</artifactId>
+ <artifactId>spark-streaming-kinesis-asl_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Kinesis Integration</name>
diff --git a/connector/protobuf/pom.xml b/connector/protobuf/pom.xml
index e85f07841df..c99b9a06087 100644
--- a/connector/protobuf/pom.xml
+++ b/connector/protobuf/pom.xml
@@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-protobuf_2.12</artifactId>
+ <artifactId>spark-protobuf_2.13</artifactId>
<properties>
<sbt.project.name>protobuf</sbt.project.name>
</properties>
@@ -70,12 +70,12 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
- <!-- #if scala-2.13 --><!--
+ <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- --><!-- #endif scala-2.13 -->
+ <!-- #endif scala-2.13 -->
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
diff --git a/connector/spark-ganglia-lgpl/pom.xml
b/connector/spark-ganglia-lgpl/pom.xml
index c0dcde13558..00f4769fd60 100644
--- a/connector/spark-ganglia-lgpl/pom.xml
+++ b/connector/spark-ganglia-lgpl/pom.xml
@@ -19,13 +19,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<!-- Ganglia integration is not included by default due to LGPL-licensed
code -->
- <artifactId>spark-ganglia-lgpl_2.12</artifactId>
+ <artifactId>spark-ganglia-lgpl_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Ganglia Integration</name>
diff --git a/core/pom.xml b/core/pom.xml
index 09b0a2af96f..4413ba6b634 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>spark-core_2.12</artifactId>
+ <artifactId>spark-core_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Core</name>
<url>https://spark.apache.org/</url>
@@ -35,12 +35,12 @@
</properties>
<dependencies>
- <!-- #if scala-2.13 --><!--
+ <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- --><!-- #endif scala-2.13 -->
+ <!-- #endif scala-2.13 -->
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3
b/dev/deps/spark-deps-hadoop-3-hive-2.3
index d275c827ef1..bed3be0b086 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -5,7 +5,7 @@ RoaringBitmap/0.9.44//RoaringBitmap-0.9.44.jar
ST4/4.0.4//ST4-4.0.4.jar
activation/1.1.1//activation-1.1.1.jar
aircompressor/0.21//aircompressor-0.21.jar
-algebra_2.12/2.0.1//algebra_2.12-2.0.1.jar
+algebra_2.13/2.8.0//algebra_2.13-2.8.0.jar
aliyun-java-sdk-core/4.5.10//aliyun-java-sdk-core-4.5.10.jar
aliyun-java-sdk-kms/2.11.0//aliyun-java-sdk-kms-2.11.0.jar
aliyun-java-sdk-ram/3.1.0//aliyun-java-sdk-ram-3.1.0.jar
@@ -30,11 +30,11 @@ azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar
azure-storage/7.0.1//azure-storage-7.0.1.jar
blas/3.0.3//blas-3.0.3.jar
bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
-breeze-macros_2.12/2.1.0//breeze-macros_2.12-2.1.0.jar
-breeze_2.12/2.1.0//breeze_2.12-2.1.0.jar
-cats-kernel_2.12/2.1.1//cats-kernel_2.12-2.1.1.jar
+breeze-macros_2.13/2.1.0//breeze-macros_2.13-2.1.0.jar
+breeze_2.13/2.1.0//breeze_2.13-2.1.0.jar
+cats-kernel_2.13/2.8.0//cats-kernel_2.13-2.8.0.jar
chill-java/0.10.0//chill-java-0.10.0.jar
-chill_2.12/0.10.0//chill_2.12-0.10.0.jar
+chill_2.13/0.10.0//chill_2.13-0.10.0.jar
commons-cli/1.5.0//commons-cli-1.5.0.jar
commons-codec/1.15//commons-codec-1.15.jar
commons-collections/3.2.2//commons-collections-3.2.2.jar
@@ -104,7 +104,7 @@ jackson-databind/2.15.1//jackson-databind-2.15.1.jar
jackson-dataformat-cbor/2.15.1//jackson-dataformat-cbor-2.15.1.jar
jackson-dataformat-yaml/2.15.1//jackson-dataformat-yaml-2.15.1.jar
jackson-datatype-jsr310/2.15.1//jackson-datatype-jsr310-2.15.1.jar
-jackson-module-scala_2.12/2.15.1//jackson-module-scala_2.12-2.15.1.jar
+jackson-module-scala_2.13/2.15.1//jackson-module-scala_2.13-2.15.1.jar
jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar
@@ -129,14 +129,16 @@ jettison/1.5.3//jettison-1.5.3.jar
jetty-util-ajax/9.4.51.v20230217//jetty-util-ajax-9.4.51.v20230217.jar
jetty-util/9.4.51.v20230217//jetty-util-9.4.51.v20230217.jar
jline/2.14.6//jline-2.14.6.jar
+jline/3.21.0//jline-3.21.0.jar
+jna/5.9.0//jna-5.9.0.jar
joda-time/2.12.5//joda-time-2.12.5.jar
jodd-core/3.5.2//jodd-core-3.5.2.jar
jpam/1.1//jpam-1.1.jar
json/1.8//json-1.8.jar
-json4s-ast_2.12/3.7.0-M11//json4s-ast_2.12-3.7.0-M11.jar
-json4s-core_2.12/3.7.0-M11//json4s-core_2.12-3.7.0-M11.jar
-json4s-jackson_2.12/3.7.0-M11//json4s-jackson_2.12-3.7.0-M11.jar
-json4s-scalap_2.12/3.7.0-M11//json4s-scalap_2.12-3.7.0-M11.jar
+json4s-ast_2.13/3.7.0-M11//json4s-ast_2.13-3.7.0-M11.jar
+json4s-core_2.13/3.7.0-M11//json4s-core_2.13-3.7.0-M11.jar
+json4s-jackson_2.13/3.7.0-M11//json4s-jackson_2.13-3.7.0-M11.jar
+json4s-scalap_2.13/3.7.0-M11//json4s-scalap_2.13-3.7.0-M11.jar
jsr305/3.0.0//jsr305-3.0.0.jar
jta/1.1//jta-1.1.jar
jul-to-slf4j/2.0.7//jul-to-slf4j-2.0.7.jar
@@ -224,21 +226,22 @@ pickle/1.3//pickle-1.3.jar
py4j/0.10.9.7//py4j-0.10.9.7.jar
remotetea-oncrpc/1.1.2//remotetea-oncrpc-1.1.2.jar
rocksdbjni/8.1.1.1//rocksdbjni-8.1.1.1.jar
-scala-collection-compat_2.12/2.7.0//scala-collection-compat_2.12-2.7.0.jar
-scala-compiler/2.12.17//scala-compiler-2.12.17.jar
-scala-library/2.12.17//scala-library-2.12.17.jar
-scala-parser-combinators_2.12/2.2.0//scala-parser-combinators_2.12-2.2.0.jar
-scala-reflect/2.12.17//scala-reflect-2.12.17.jar
-scala-xml_2.12/2.1.0//scala-xml_2.12-2.1.0.jar
+scala-collection-compat_2.13/2.7.0//scala-collection-compat_2.13-2.7.0.jar
+scala-compiler/2.13.8//scala-compiler-2.13.8.jar
+scala-library/2.13.8//scala-library-2.13.8.jar
+scala-parallel-collections_2.13/1.0.4//scala-parallel-collections_2.13-1.0.4.jar
+scala-parser-combinators_2.13/2.2.0//scala-parser-combinators_2.13-2.2.0.jar
+scala-reflect/2.13.8//scala-reflect-2.13.8.jar
+scala-xml_2.13/2.1.0//scala-xml_2.13-2.1.0.jar
shims/0.9.44//shims-0.9.44.jar
slf4j-api/2.0.7//slf4j-api-2.0.7.jar
snakeyaml-engine/2.6//snakeyaml-engine-2.6.jar
snakeyaml/2.0//snakeyaml-2.0.jar
snappy-java/1.1.10.0//snappy-java-1.1.10.0.jar
-spire-macros_2.12/0.17.0//spire-macros_2.12-0.17.0.jar
-spire-platform_2.12/0.17.0//spire-platform_2.12-0.17.0.jar
-spire-util_2.12/0.17.0//spire-util_2.12-0.17.0.jar
-spire_2.12/0.17.0//spire_2.12-0.17.0.jar
+spire-macros_2.13/0.18.0//spire-macros_2.13-0.18.0.jar
+spire-platform_2.13/0.18.0//spire-platform_2.13-0.18.0.jar
+spire-util_2.13/0.18.0//spire-util_2.13-0.18.0.jar
+spire_2.13/0.18.0//spire_2.13-0.18.0.jar
stax-api/1.0.1//stax-api-1.0.1.jar
stream/2.9.6//stream-2.9.6.jar
super-csv/2.2.0//super-csv-2.2.0.jar
diff --git a/dev/mima b/dev/mima
index 4a9e343b0a7..32c3718e4cc 100755
--- a/dev/mima
+++ b/dev/mima
@@ -24,9 +24,9 @@ set -e
FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
cd "$FWDIR"
-SPARK_PROFILES=${1:-"-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl
-Pkinesis-asl -Phive-thriftserver -Phive"}
-TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export
tools/fullClasspath" | grep jar | tail -n1)"
-OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES
"export oldDeps/fullClasspath" | grep jar | tail -n1)"
+SPARK_PROFILES=${1:-"-Pscala-2.13 -Pmesos -Pkubernetes -Pyarn
-Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"}
+TOOLS_CLASSPATH="$(build/sbt -Pscala-2.13 -DcopyDependencies=false "export
tools/fullClasspath" | grep jar | tail -n1)"
+OLD_DEPS_CLASSPATH="$(build/sbt -Pscala-2.13 -DcopyDependencies=false
$SPARK_PROFILES "export oldDeps/fullClasspath" | grep jar | tail -n1)"
rm -f .generated-mima*
@@ -42,7 +42,7 @@ $JAVA_CMD \
-cp "$TOOLS_CLASSPATH:$OLD_DEPS_CLASSPATH" \
org.apache.spark.tools.GenerateMIMAIgnore
-echo -e "q\n" | build/sbt -mem 5120 -DcopyDependencies=false "$@"
mimaReportBinaryIssues | grep -v -e "info.*Resolving"
+echo -e "q\n" | build/sbt -Pscala-2.13 -mem 5120 -DcopyDependencies=false "$@"
mimaReportBinaryIssues | grep -v -e "info.*Resolving"
ret_val=$?
if [ $ret_val != 0 ]; then
diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh
index d7967ac3afa..07da6497b97 100755
--- a/dev/test-dependencies.sh
+++ b/dev/test-dependencies.sh
@@ -62,7 +62,7 @@ SCALA_BINARY_VERSION=$($MVN -q \
-Dexec.args='${scala.binary.version}' \
--non-recursive \
org.codehaus.mojo:exec-maven-plugin:1.6.0:exec | grep -E '[0-9]+\.[0-9]+')
-if [[ "$SCALA_BINARY_VERSION" != "2.12" ]]; then
+if [[ "$SCALA_BINARY_VERSION" != "2.13" ]]; then
echo "Skip dependency testing on $SCALA_BINARY_VERSION"
exit 0
fi
diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb
index 28d5e0d82c9..9cb073ef1e0 100644
--- a/docs/_plugins/copy_api_dirs.rb
+++ b/docs/_plugins/copy_api_dirs.rb
@@ -26,8 +26,8 @@ if not (ENV['SKIP_API'] == '1')
curr_dir = pwd
cd("..")
- puts "Running 'build/sbt -Pkinesis-asl clean compile unidoc' from " + pwd
+ "; this may take a few minutes..."
- system("build/sbt -Pkinesis-asl clean compile unidoc") || raise("Unidoc
generation failed")
+ puts "Running 'build/sbt -Pscala-2.13 -Pkinesis-asl clean compile unidoc'
from " + pwd + "; this may take a few minutes..."
+ system("build/sbt -Pscala-2.13 -Pkinesis-asl clean compile unidoc") ||
raise("Unidoc generation failed")
puts "Moving back into docs dir."
cd("docs")
@@ -37,7 +37,7 @@ if not (ENV['SKIP_API'] == '1')
# Copy over the unified ScalaDoc for all projects to api/scala.
# This directory will be copied over to _site when `jekyll` command is run.
- source = "../target/scala-2.12/unidoc"
+ source = "../target/scala-2.13/unidoc"
dest = "api/scala"
puts "Making directory " + dest
@@ -119,8 +119,8 @@ if not (ENV['SKIP_API'] == '1')
puts "Moving to project root and building API docs."
cd("..")
- puts "Running 'build/sbt clean package -Phive' from " + pwd + "; this may
take a few minutes..."
- system("build/sbt clean package -Phive") || raise("PySpark doc generation
failed")
+ puts "Running 'build/sbt -Pscala-2.13 clean package -Phive' from " + pwd +
"; this may take a few minutes..."
+ system("build/sbt -Pscala-2.13 clean package -Phive") || raise("PySpark
doc generation failed")
puts "Moving back into docs dir."
cd("docs")
@@ -165,8 +165,8 @@ if not (ENV['SKIP_API'] == '1')
puts "Moving to project root and building API docs."
cd("..")
- puts "Running 'build/sbt clean package -Phive' from " + pwd + "; this
may take a few minutes..."
- system("build/sbt clean package -Phive") || raise("SQL doc generation
failed")
+ puts "Running 'build/sbt -Pscala-2.13 clean package -Phive' from " + pwd
+ "; this may take a few minutes..."
+ system("build/sbt -Pscala-2.13 clean package -Phive") || raise("SQL doc
generation failed")
puts "Moving back into docs dir."
cd("docs")
diff --git a/examples/pom.xml b/examples/pom.xml
index e8f22b995fd..57e41724bdc 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>spark-examples_2.12</artifactId>
+ <artifactId>spark-examples_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Examples</name>
<url>https://spark.apache.org/</url>
diff --git a/graphx/pom.xml b/graphx/pom.xml
index 48baeb9a875..5d01dd06c0e 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>spark-graphx_2.12</artifactId>
+ <artifactId>spark-graphx_2.13</artifactId>
<properties>
<sbt.project.name>graphx</sbt.project.name>
</properties>
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index e213052dbc1..7a3e601553d 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>spark-hadoop-cloud_2.12</artifactId>
+ <artifactId>spark-hadoop-cloud_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Hadoop Cloud Integration</name>
<description>
diff --git a/launcher/pom.xml b/launcher/pom.xml
index aba7ee82d53..0bc3ae20ee1 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>spark-launcher_2.12</artifactId>
+ <artifactId>spark-launcher_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Launcher</name>
<url>https://spark.apache.org/</url>
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index 00c16a8b6a5..83ca643f43b 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>spark-mllib-local_2.12</artifactId>
+ <artifactId>spark-mllib-local_2.13</artifactId>
<properties>
<sbt.project.name>mllib-local</sbt.project.name>
</properties>
diff --git a/mllib/pom.xml b/mllib/pom.xml
index 73af83c7586..07290124273 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>spark-mllib_2.12</artifactId>
+ <artifactId>spark-mllib_2.13</artifactId>
<properties>
<sbt.project.name>mllib</sbt.project.name>
</properties>
@@ -91,12 +91,12 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <!-- #if scala-2.13 --><!--
+ <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- --><!-- #endif scala-2.13 -->
+ <!-- #endif scala-2.13 -->
<dependency>
<groupId>org.scalanlp</groupId>
<artifactId>breeze_${scala.binary.version}</artifactId>
diff --git a/pom.xml b/pom.xml
index 2394e429218..cc61c72ed75 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
<version>18</version>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Spark Project Parent POM</name>
@@ -167,8 +167,8 @@
<!-- managed up from 3.2.1 for SPARK-11652 -->
<commons.collections.version>3.2.2</commons.collections.version>
<commons.collections4.version>4.4</commons.collections4.version>
- <scala.version>2.12.17</scala.version>
- <scala.binary.version>2.12</scala.binary.version>
+ <scala.version>2.13.8</scala.version>
+ <scala.binary.version>2.13</scala.binary.version>
<scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version>
<!-- dont update scala-maven-plugin to version 4.8.1 SPARK-42809 and
SPARK-43595 -->
<scala-maven-plugin.version>4.8.0</scala-maven-plugin.version>
@@ -422,13 +422,13 @@
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
- <!-- #if scala-2.13 --><!--
+ <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
<version>1.0.4</version>
</dependency>
- --><!-- #endif scala-2.13 -->
+ <!-- #endif scala-2.13 -->
<dependency>
<groupId>com.twitter</groupId>
<artifactId>chill_${scala.binary.version}</artifactId>
@@ -1073,7 +1073,7 @@
<exclusions>
<exclusion>
<groupId>org.scala-lang.modules</groupId>
- <artifactId>scala-xml_2.12</artifactId>
+ <artifactId>scala-xml_2.13</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -2854,9 +2854,43 @@
<arg>-feature</arg>
<arg>-explaintypes</arg>
<arg>-target:jvm-1.8</arg>
- <arg>-Xfatal-warnings</arg>
- <arg>-Ywarn-unused:imports</arg>
- <arg>-P:silencer:globalFilters=.*deprecated.*</arg>
+ <arg>-Wconf:cat=deprecation:wv,any:e</arg>
+ <!--
+ TODO(SPARK-33805): Undo the corresponding deprecated usage
suppression rule after fixed
+ <arg>-Wunused:imports</arg>
+ -->
+ <arg>-Wconf:cat=scaladoc:wv</arg>
+ <arg>-Wconf:cat=lint-multiarg-infix:wv</arg>
+ <arg>-Wconf:cat=other-nullary-override:wv</arg>
+
<arg>-Wconf:cat=other-match-analysis&site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv</arg>
+
<arg>-Wconf:cat=other-pure-statement&site=org.apache.spark.streaming.util.FileBasedWriteAheadLog.readAll.readFile:wv</arg>
+
<arg>-Wconf:cat=other-pure-statement&site=org.apache.spark.scheduler.OutputCommitCoordinatorSuite.<local
OutputCommitCoordinatorSuite>.futureAction:wv</arg>
+ <!--
+ SPARK-33775 Suppress compilation warnings that contain the
following contents.
+ TODO(SPARK-33805): Undo the corresponding deprecated usage
suppression rule after fixed.
+ -->
+
<arg>-Wconf:msg=^(?=.*?method|value|type|object|trait|inheritance)(?=.*?deprecated)(?=.*?since
2.13).+$:s</arg>
+ <arg>-Wconf:msg=^(?=.*?Widening conversion from)(?=.*?is
deprecated because it loses precision).+$:s</arg>
+ <arg>-Wconf:msg=Auto-application to \`\(\)\` is
deprecated:s</arg>
+ <arg>-Wconf:msg=method with a single empty parameter list
overrides method without any parameter list:s</arg>
+ <arg>-Wconf:msg=method without a parameter list overrides a
method with a single empty one:s</arg>
+ <!--
+ SPARK-35574 Prevent the recurrence of compilation warnings
related to
+ `procedure syntax is deprecated`
+ -->
+ <arg>-Wconf:cat=deprecation&msg=procedure syntax is
deprecated:e</arg>
+ <!--
+ SPARK-35496 Upgrade Scala to 2.13.7 and suppress:
+ 1. `The outer reference in this type test cannot be checked at
run time`
+ 2. `the type test for pattern TypeA cannot be checked at
runtime because it
+ has type parameters eliminated by erasure`
+ 3. `abstract type TypeA in type pattern Seq[TypeA] (the
underlying of
+ Seq[TypeA]) is unchecked since it is eliminated by erasure`
+ 4. `fruitless type test: a value of TypeA cannot also be a
TypeB`
+ -->
+ <arg>-Wconf:cat=unchecked&msg=outer reference:s</arg>
+ <arg>-Wconf:cat=unchecked&msg=eliminated by erasure:s</arg>
+ <arg>-Wconf:msg=^(?=.*?a value of type)(?=.*?cannot also
be).+$:s</arg>
</args>
<jvmArgs>
<jvmArg>-Xss128m</jvmArg>
@@ -2872,13 +2906,6 @@
<javacArg>${java.version}</javacArg>
<javacArg>-Xlint:all,-serial,-path,-try</javacArg>
</javacArgs>
- <compilerPlugins>
- <compilerPlugin>
- <groupId>com.github.ghik</groupId>
- <artifactId>silencer-plugin_${scala.version}</artifactId>
- <version>1.7.10</version>
- </compilerPlugin>
- </compilerPlugins>
</configuration>
</plugin>
<plugin>
@@ -3529,6 +3556,13 @@
</properties>
</profile>
+ <profile>
+ <id>scala-2.13</id>
+ <properties>
+ <scala.version>2.13.8</scala.version>
+ </properties>
+ </profile>
+
<profile>
<id>scala-2.12</id>
<properties>
@@ -3538,20 +3572,6 @@
-->
<scala.version>2.12.17</scala.version>
</properties>
- <build>
- <pluginManagement>
- <plugins>
- </plugins>
- </pluginManagement>
- </build>
- </profile>
-
- <profile>
- <id>scala-2.13</id>
- <properties>
- <scala.version>2.13.8</scala.version>
- <scala.binary.version>2.13</scala.binary.version>
- </properties>
<build>
<pluginManagement>
<plugins>
@@ -3565,45 +3585,16 @@
<arg>-feature</arg>
<arg>-explaintypes</arg>
<arg>-target:jvm-1.8</arg>
- <arg>-Wconf:cat=deprecation:wv,any:e</arg>
- <!--
- TODO(SPARK-33805): Undo the corresponding deprecated usage
suppression rule after fixed
- <arg>-Wunused:imports</arg>
- -->
- <arg>-Wconf:cat=scaladoc:wv</arg>
- <arg>-Wconf:cat=lint-multiarg-infix:wv</arg>
- <arg>-Wconf:cat=other-nullary-override:wv</arg>
-
<arg>-Wconf:cat=other-match-analysis&site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv</arg>
-
<arg>-Wconf:cat=other-pure-statement&site=org.apache.spark.streaming.util.FileBasedWriteAheadLog.readAll.readFile:wv</arg>
-
<arg>-Wconf:cat=other-pure-statement&site=org.apache.spark.scheduler.OutputCommitCoordinatorSuite.<local
OutputCommitCoordinatorSuite>.futureAction:wv</arg>
- <!--
- SPARK-33775 Suppress compilation warnings that contain the
following contents.
- TODO(SPARK-33805): Undo the corresponding deprecated usage
suppression rule after fixed.
- -->
-
<arg>-Wconf:msg=^(?=.*?method|value|type|object|trait|inheritance)(?=.*?deprecated)(?=.*?since
2.13).+$:s</arg>
- <arg>-Wconf:msg=^(?=.*?Widening conversion from)(?=.*?is
deprecated because it loses precision).+$:s</arg>
- <arg>-Wconf:msg=Auto-application to \`\(\)\` is
deprecated:s</arg>
- <arg>-Wconf:msg=method with a single empty parameter list
overrides method without any parameter list:s</arg>
- <arg>-Wconf:msg=method without a parameter list overrides a
method with a single empty one:s</arg>
- <!--
- SPARK-35574 Prevent the recurrence of compilation warnings
related to
- `procedure syntax is deprecated`
- -->
- <arg>-Wconf:cat=deprecation&msg=procedure syntax is
deprecated:e</arg>
- <!--
- SPARK-35496 Upgrade Scala to 2.13.7 and suppress:
- 1. `The outer reference in this type test cannot be
checked at run time`
- 2. `the type test for pattern TypeA cannot be checked at
runtime because it
- has type parameters eliminated by erasure`
- 3. `abstract type TypeA in type pattern Seq[TypeA] (the
underlying of
- Seq[TypeA]) is unchecked since it is eliminated by erasure`
- 4. `fruitless type test: a value of TypeA cannot also be a
TypeB`
- -->
- <arg>-Wconf:cat=unchecked&msg=outer reference:s</arg>
- <arg>-Wconf:cat=unchecked&msg=eliminated by
erasure:s</arg>
- <arg>-Wconf:msg=^(?=.*?a value of type)(?=.*?cannot also
be).+$:s</arg>
+ <arg>-Xfatal-warnings</arg>
+ <arg>-Ywarn-unused:imports</arg>
+ <arg>-P:silencer:globalFilters=.*deprecated.*</arg>
</args>
<compilerPlugins combine.self="override">
+ <compilerPlugin>
+ <groupId>com.github.ghik</groupId>
+ <artifactId>silencer-plugin_${scala.version}</artifactId>
+ <version>1.7.10</version>
+ </compilerPlugin>
</compilerPlugins>
</configuration>
</plugin>
diff --git a/repl/pom.xml b/repl/pom.xml
index 8c0f9f989c1..74ac775100c 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>spark-repl_2.12</artifactId>
+ <artifactId>spark-repl_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project REPL</name>
<url>https://spark.apache.org/</url>
diff --git a/resource-managers/kubernetes/core/pom.xml
b/resource-managers/kubernetes/core/pom.xml
index 9dab5496184..72c7f1f12f4 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -19,12 +19,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
- <artifactId>spark-kubernetes_2.12</artifactId>
+ <artifactId>spark-kubernetes_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Kubernetes</name>
<properties>
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml
b/resource-managers/kubernetes/integration-tests/pom.xml
index 02894f82eec..3e25e705370 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -19,12 +19,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
- <artifactId>spark-kubernetes-integration-tests_2.12</artifactId>
+ <artifactId>spark-kubernetes-integration-tests_2.13</artifactId>
<properties>
<sbt.project.name>kubernetes-integration-tests</sbt.project.name>
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index 7510ecac3e7..267d6c7d84f 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -19,12 +19,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-mesos_2.12</artifactId>
+ <artifactId>spark-mesos_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Mesos</name>
<properties>
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index dcc7bcdd1af..2cda552a9c4 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -19,12 +19,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-yarn_2.12</artifactId>
+ <artifactId>spark-yarn_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project YARN</name>
<properties>
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 9057cc8a20e..cf01c0af4a4 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-catalyst_2.12</artifactId>
+ <artifactId>spark-catalyst_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Catalyst</name>
<url>https://spark.apache.org/</url>
@@ -87,12 +87,12 @@
<artifactId>spark-sketch_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
- <!-- #if scala-2.13 --><!--
+ <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- --><!-- #endif scala-2.13 -->
+ <!-- #endif scala-2.13 -->
<dependency>
<groupId>org.scalacheck</groupId>
<artifactId>scalacheck_${scala.binary.version}</artifactId>
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 7f4c2a4cfa5..5d4f7572d00 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-sql_2.12</artifactId>
+ <artifactId>spark-sql_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project SQL</name>
<url>https://spark.apache.org/</url>
@@ -89,12 +89,12 @@
<scope>test</scope>
</dependency>
- <!-- #if scala-2.13 --><!--
+ <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- --><!-- #endif scala-2.13 -->
+ <!-- #endif scala-2.13 -->
<dependency>
<groupId>org.apache.orc</groupId>
<artifactId>orc-core</artifactId>
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index ad7fc0d2ac4..4bbb92d1376 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-hive-thriftserver_2.12</artifactId>
+ <artifactId>spark-hive-thriftserver_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Hive Thrift Server</name>
<url>https://spark.apache.org/</url>
@@ -61,12 +61,12 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <!-- #if scala-2.13 --><!--
+ <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- --><!-- #endif scala-2.13 -->
+ <!-- #endif scala-2.13 -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 16d915c233e..b267830a3ad 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>spark-hive_2.12</artifactId>
+ <artifactId>spark-hive_2.13</artifactId>
<packaging>jar</packaging>
<name>Spark Project Hive</name>
<url>https://spark.apache.org/</url>
@@ -79,12 +79,12 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <!-- #if scala-2.13 --><!--
+ <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- --><!-- #endif scala-2.13 -->
+ <!-- #endif scala-2.13 -->
<dependency>
<groupId>${hive.group}</groupId>
<artifactId>hive-common</artifactId>
diff --git a/streaming/pom.xml b/streaming/pom.xml
index bebfd3abcce..a36370a1e8b 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>spark-streaming_2.12</artifactId>
+ <artifactId>spark-streaming_2.13</artifactId>
<properties>
<sbt.project.name>streaming</sbt.project.name>
</properties>
@@ -50,12 +50,12 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
- <!-- #if scala-2.13 --><!--
+ <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- --><!-- #endif scala-2.13 -->
+ <!-- #endif scala-2.13 -->
<!--
This spark-tags test-dep is needed even though it isn't used in this
module, otherwise testing-cmds that exclude
diff --git a/tools/pom.xml b/tools/pom.xml
index 9dc136d4268..9a088e84bc0 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -19,12 +19,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.12</artifactId>
+ <artifactId>spark-parent_2.13</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>spark-tools_2.12</artifactId>
+ <artifactId>spark-tools_2.13</artifactId>
<properties>
<sbt.project.name>tools</sbt.project.name>
</properties>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
