This is an automated email from the ASF dual-hosted git repository.
pdallig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new fa6e3ee3b8 [ZEPPELIN-5969] Remove Hadoop2 and move to Hadoop3 shaded
client (#4691)
fa6e3ee3b8 is described below
commit fa6e3ee3b83aa5ffc070557780ec111386b8c951
Author: Cheng Pan <cheng...@apache.org>
AuthorDate: Wed Apr 3 15:44:52 2024 +0800
[ZEPPELIN-5969] Remove Hadoop2 and move to Hadoop3 shaded client (#4691)
* Drop hadoop2 in github actions
* Update docs
* Drop hadoop2 support
* Remove hadoop2 integration tests
* findbugs use the same version in all modules
* Use hadoop3.3 for tests
* Move to scala 2.12
* Try to fix flink
* Usage of metals
* Remove duplicate version and groupid
* Fix Flink with Hadoop3
* fix log
* R
* fix
* fix
* fix
* fix
* hadoop-3.3
* fix
* fix
* Address comments
* address comments
---------
Co-authored-by: Philipp Dallig <philipp.dal...@gmail.com>
---
.github/workflows/core.yml | 20 +-
.github/workflows/frontend.yml | 8 +-
.github/workflows/quick.yml | 2 +-
.gitignore | 4 +
alluxio/pom.xml | 16 +-
docs/interpreter/flink.md | 2 +-
docs/setup/basics/how_to_build.md | 1 -
docs/setup/deployment/flink_and_spark_cluster.md | 16 +-
flink-cmd/pom.xml | 9 +-
flink/flink-scala-2.12/pom.xml | 695 ++++++++++++---
.../src/main/resources/interpreter-setting.json | 2 +-
.../zeppelin/flink/FlinkScalaInterpreter.scala | 2 +-
.../zeppelin/flink/FlinkSqlInterpreterTest.java | 2 +-
flink/flink1.15-shims/pom.xml | 2 -
flink/flink1.16-shims/pom.xml | 2 -
hbase/pom.xml | 41 -
jdbc/pom.xml | 208 +----
livy/pom.xml | 120 +--
pom.xml | 978 +--------------------
rlang/pom.xml | 21 +-
spark/interpreter/pom.xml | 30 -
spark/spark-scala-parent/pom.xml | 15 -
spark/spark-shims/pom.xml | 3 +-
submarine/pom.xml | 53 +-
zeppelin-interpreter-integration/pom.xml | 22 +-
.../zeppelin/integration/FlinkIntegrationTest.java | 7 +-
.../integration/SparkIntegrationTest32.java | 11 -
.../integration/SparkIntegrationTest33.java | 10 -
.../integration/SparkIntegrationTest34.java | 12 +-
.../integration/ZeppelinFlinkClusterTest.java | 2 -
.../integration/ZeppelinSparkClusterTest32.java | 10 -
.../integration/ZeppelinSparkClusterTest33.java | 10 -
zeppelin-interpreter/pom.xml | 61 +-
zeppelin-plugins/launcher/yarn/pom.xml | 32 +-
zeppelin-plugins/notebookrepo/filesystem/pom.xml | 110 +--
zeppelin-plugins/notebookrepo/s3/pom.xml | 24 -
zeppelin-server/pom.xml | 20 +-
zeppelin-zengine/pom.xml | 67 +-
.../interpreter/integration/DownloadUtils.java | 7 +-
39 files changed, 693 insertions(+), 1964 deletions(-)
diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index b197fd16eb..e441c82384 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -40,7 +40,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- hadoop: [hadoop2, hadoop3]
+ hadoop: [hadoop3]
java: [ 8, 11 ]
steps:
- name: Checkout
@@ -183,7 +183,7 @@ jobs:
R -e "IRkernel::installspec()"
- name: install environment
run: |
- ./mvnw install -DskipTests -pl
python,rlang,zeppelin-jupyter-interpreter -am -Phadoop2 ${MAVEN_ARGS}
+ ./mvnw install -DskipTests -pl
python,rlang,zeppelin-jupyter-interpreter -am -Phadoop3 ${MAVEN_ARGS}
- name: run tests with ${{ matrix.python }}
run: |
./mvnw test -pl python,rlang,zeppelin-jupyter-interpreter
-DfailIfNoTests=false ${MAVEN_ARGS}
@@ -221,7 +221,7 @@ jobs:
${{ runner.os }}-zeppelin-
- name: install environment
run: |
- ./mvnw install -DskipTests -Phadoop2 -Pintegration -pl
zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown,flink-cmd,flink/flink-scala-2.12,jdbc,shell
-am -Pflink-117 ${MAVEN_ARGS}
+ ./mvnw install -DskipTests -Phadoop3 -Pintegration -pl
zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown,flink-cmd,flink/flink-scala-2.12,jdbc,shell
-am -Pflink-117 ${MAVEN_ARGS}
./mvnw package -pl zeppelin-plugins -amd -DskipTests ${MAVEN_ARGS}
- name: Setup conda environment with python 3.9 and R
uses: conda-incubator/setup-miniconda@v2
@@ -238,7 +238,7 @@ jobs:
run: |
R -e "IRkernel::installspec()"
- name: run tests
- run: ./mvnw test -pl zeppelin-interpreter-integration -Phadoop2
-Pintegration -DfailIfNoTests=false
-Dtest=ZeppelinClientIntegrationTest,ZeppelinClientWithAuthIntegrationTest,ZSessionIntegrationTest,ShellIntegrationTest,JdbcIntegrationTest
+ run: ./mvnw test -pl zeppelin-interpreter-integration -Phadoop3
-Pintegration -DfailIfNoTests=false
-Dtest=ZeppelinClientIntegrationTest,ZeppelinClientWithAuthIntegrationTest,ZSessionIntegrationTest,ShellIntegrationTest,JdbcIntegrationTest
- name: Print zeppelin logs
if: always()
run: if [ -d "logs" ]; then cat logs/*; fi
@@ -278,7 +278,7 @@ jobs:
${{ runner.os }}-zeppelin-
- name: install environment for flink
run: |
- ./mvnw install -DskipTests -am -pl
flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{
matrix.flink }} -Phadoop2 -Pintegration ${MAVEN_ARGS}
+ ./mvnw install -DskipTests -am -pl
flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{
matrix.flink }} -Phadoop3 -Pintegration ${MAVEN_ARGS}
./mvnw clean package -pl zeppelin-plugins -amd -DskipTests
${MAVEN_ARGS}
- name: Setup conda environment with python ${{ matrix.python }} and R
uses: conda-incubator/setup-miniconda@v2
@@ -292,7 +292,7 @@ jobs:
auto-activate-base: false
use-mamba: true
- name: run tests for flink
- run: ./mvnw verify -pl
flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{
matrix.flink }} -am -Phadoop2 -Pintegration -DfailIfNoTests=false
-Dtest=org.apache.zeppelin.flink.*Test,FlinkIntegrationTest${{ matrix.flink }}
${MAVEN_ARGS}
+ run: ./mvnw verify -pl
flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{
matrix.flink }} -am -Phadoop3 -Pintegration -DfailIfNoTests=false
-Dtest=org.apache.zeppelin.flink.*Test,FlinkIntegrationTest${{ matrix.flink }}
${MAVEN_ARGS}
- name: Print zeppelin logs
if: always()
run: if [ -d "logs" ]; then cat logs/*; fi
@@ -328,7 +328,7 @@ jobs:
${{ runner.os }}-zeppelin-
- name: install environment
run: |
- ./mvnw install -DskipTests -pl
zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown
-am -Phadoop2 -Pintegration ${MAVEN_ARGS}
+ ./mvnw install -DskipTests -pl
zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown
-am -Phadoop3 -Pintegration ${MAVEN_ARGS}
./mvnw clean package -pl zeppelin-plugins -amd -DskipTests
${MAVEN_ARGS}
- name: Setup conda environment with python 3.9 and R
uses: conda-incubator/setup-miniconda@v2
@@ -382,7 +382,7 @@ jobs:
restore-keys: |
${{ runner.os }}-zeppelin-
- name: install environment
- run: ./mvnw install -DskipTests -pl
spark-submit,spark/scala-2.12,spark/scala-2.13 -am -Phadoop2 ${MAVEN_ARGS}
+ run: ./mvnw install -DskipTests -pl
spark-submit,spark/scala-2.12,spark/scala-2.13 -am -Phadoop3 ${MAVEN_ARGS}
- name: Setup conda environment with python ${{ matrix.python }} and R
uses: conda-incubator/setup-miniconda@v2
with:
@@ -400,11 +400,11 @@ jobs:
- name: run spark-3.2 tests with scala-2.12 and python-${{ matrix.python
}}
run: |
rm -rf spark/interpreter/metastore_db
- ./mvnw verify -pl spark-submit,spark/interpreter -am
-Dtest=org/apache/zeppelin/spark/* -Pspark-3.2 -Pspark-scala-2.12 -Phadoop2
-Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
+ ./mvnw verify -pl spark-submit,spark/interpreter -am
-Dtest=org/apache/zeppelin/spark/* -Pspark-3.2 -Pspark-scala-2.12 -Phadoop3
-Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.2 tests with scala-2.13 and python-${{ matrix.python
}}
run: |
rm -rf spark/interpreter/metastore_db
- ./mvnw verify -pl spark-submit,spark/interpreter -am
-Dtest=org/apache/zeppelin/spark/* -Pspark-3.2 -Pspark-scala-2.13 -Phadoop2
-Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
+ ./mvnw verify -pl spark-submit,spark/interpreter -am
-Dtest=org/apache/zeppelin/spark/* -Pspark-3.2 -Pspark-scala-2.13 -Phadoop3
-Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.3 tests with scala-2.12 and python-${{ matrix.python
}}
run: |
rm -rf spark/interpreter/metastore_db
diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml
index d7fcbc2fa7..75175e4559 100644
--- a/.github/workflows/frontend.yml
+++ b/.github/workflows/frontend.yml
@@ -53,9 +53,9 @@ jobs:
restore-keys: |
${{ runner.os }}-zeppelin-
- name: Install application
- run: ./mvnw clean install -DskipTests -am -pl zeppelin-web
-Pspark-scala-2.12 -Pspark-3.4 -Phadoop2 -Pweb-dist ${MAVEN_ARGS}
+ run: ./mvnw clean install -DskipTests -am -pl zeppelin-web
-Pspark-scala-2.12 -Pspark-3.4 -Phadoop3 -Pweb-dist ${MAVEN_ARGS}
- name: Run headless test
- run: xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24"
./mvnw verify -pl zeppelin-web -Pspark-scala-2.12 -Pspark-3.4 -Phadoop2
-Pweb-dist -Pweb-e2e ${MAVEN_ARGS}
+ run: xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24"
./mvnw verify -pl zeppelin-web -Pspark-scala-2.12 -Pspark-3.4 -Phadoop3
-Pweb-dist -Pweb-e2e ${MAVEN_ARGS}
- name: Print zeppelin logs
if: always()
run: if [ -d "logs" ]; then cat logs/*; fi
@@ -128,10 +128,10 @@ jobs:
R -e "IRkernel::installspec()"
- name: Install Environment
run: |
- ./mvnw clean install -DskipTests -am -pl zeppelin-integration
-Pintegration -Pspark-scala-2.12 -Pspark-3.4 -Phadoop2 -Pweb-dist ${MAVEN_ARGS}
+ ./mvnw clean install -DskipTests -am -pl zeppelin-integration
-Pintegration -Pspark-scala-2.12 -Pspark-3.4 -Phadoop3 -Pweb-dist ${MAVEN_ARGS}
- name: run tests
run: |
- source ./testing/downloadSpark.sh "3.4.1" "3" && echo "SPARK_HOME:
${SPARK_HOME}" && xvfb-run --auto-servernum --server-args="-screen 0
1600x1024x16" ./mvnw verify -DfailIfNoTests=false -pl zeppelin-integration
-Pintegration -Pspark-scala-2.12 -Pspark-3.4 -Phadoop2 -Pweb-dist
-Pusing-source-tree ${MAVEN_ARGS}
+ source ./testing/downloadSpark.sh "3.4.1" "3" && echo "SPARK_HOME:
${SPARK_HOME}" && xvfb-run --auto-servernum --server-args="-screen 0
1600x1024x16" ./mvnw verify -DfailIfNoTests=false -pl zeppelin-integration
-Pintegration -Pspark-scala-2.12 -Pspark-3.4 -Phadoop3 -Pweb-dist
-Pusing-source-tree ${MAVEN_ARGS}
- name: Print zeppelin logs
if: always()
run: if [ -d "logs" ]; then cat logs/*; fi
diff --git a/.github/workflows/quick.yml b/.github/workflows/quick.yml
index 5bc8ac1735..1b01389e55 100644
--- a/.github/workflows/quick.yml
+++ b/.github/workflows/quick.yml
@@ -41,7 +41,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- hadoop: [hadoop2, hadoop3]
+ hadoop: [hadoop3]
steps:
- name: Checkout
uses: actions/checkout@v3
diff --git a/.gitignore b/.gitignore
index 29bb190ebc..6784673088 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,6 +10,10 @@
/interpreter/*
!/interpreter/lib
+# metals
+.bloop
+.metals
+
# interpreter temp files
derby.log
spark/metastore_db
diff --git a/alluxio/pom.xml b/alluxio/pom.xml
index 55f1a34a6a..87d374db3e 100644
--- a/alluxio/pom.xml
+++ b/alluxio/pom.xml
@@ -68,15 +68,27 @@
<artifactId>alluxio-minicluster</artifactId>
<version>${alluxio.version}</version>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>3.2.4</version>
+ <artifactId>hadoop-client-api</artifactId>
+ <version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/docs/interpreter/flink.md b/docs/interpreter/flink.md
index cc40d03a7a..df272cbdb9 100644
--- a/docs/interpreter/flink.md
+++ b/docs/interpreter/flink.md
@@ -301,7 +301,7 @@ You can also add and set other Flink properties which are
not listed in the tabl
</tr>
<tr>
<td>zeppelin.flink.hive.version</td>
- <td>2.3.4</td>
+ <td>2.3.7</td>
<td>Hive version that you would like to connect</td>
</tr>
<tr>
diff --git a/docs/setup/basics/how_to_build.md
b/docs/setup/basics/how_to_build.md
index b131ec57d1..8c8cd947f8 100644
--- a/docs/setup/basics/how_to_build.md
+++ b/docs/setup/basics/how_to_build.md
@@ -123,7 +123,6 @@ Set hadoop major version (default hadoop3).
Available profiles are
```
--Phadoop2
-Phadoop3
```
diff --git a/docs/setup/deployment/flink_and_spark_cluster.md
b/docs/setup/deployment/flink_and_spark_cluster.md
index d395ccab67..76f9063cf1 100644
--- a/docs/setup/deployment/flink_and_spark_cluster.md
+++ b/docs/setup/deployment/flink_and_spark_cluster.md
@@ -225,16 +225,16 @@ Building from source is recommended where possible, for
simplicity in this tuto
To download the Flink Binary use `wget`
```bash
-wget
"http://mirror.cogentco.com/pub/apache/flink/flink-1.1.3/flink-1.1.3-bin-hadoop24-scala_2.10.tgz";
-tar -xzvf flink-1.1.3-bin-hadoop24-scala_2.10.tgz
+wget
"http://mirror.cogentco.com/pub/apache/flink/flink-1.16.2/flink-1.16.2-bin-scala_2.12.tgz";
+tar -xzvf flink-1.16.2-bin-scala_2.12.tgz
```
-This will download Flink 1.1.3, compatible with Hadoop 2.4. You do not have
to install Hadoop for this binary to work, but if you are using Hadoop, please
change `24` to your appropriate version.
+This will download Flink 1.16.2.
Start the Flink Cluster.
```bash
-flink-1.1.3/bin/start-cluster.sh
+flink-1.16.2/bin/start-cluster.sh
```
###### Building From source
@@ -295,12 +295,12 @@ Using binaries is also
To download the Spark Binary use `wget`
```bash
-wget "http://d3kbcqa49mib13.cloudfront.net/spark-1.6.3-bin-hadoop2.6.tgz";
-tar -xzvf spark-1.6.3-bin-hadoop2.6.tgz
-mv spark-1.6.3-bin-hadoop2.6 spark
+wget "https://dlcdn.apache.org/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3.tgz";
+tar -xzvf spark-3.4.1-bin-hadoop3.tgz
+mv spark-3.4.1-bin-hadoop3 spark
```
-This will download Spark 1.6.3, compatible with Hadoop 2.6. You do not have
to install Hadoop for this binary to work, but if you are using Hadoop, please
change `2.6` to your appropriate version.
+This will download Spark 3.4.1, compatible with Hadoop 3. You do not have to
install Hadoop for this binary to work, but if you are using Hadoop, please
change `3` to your appropriate version.
###### Building From source
diff --git a/flink-cmd/pom.xml b/flink-cmd/pom.xml
index 0e82f08357..4f4456f8dc 100644
--- a/flink-cmd/pom.xml
+++ b/flink-cmd/pom.xml
@@ -44,14 +44,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
+ <artifactId>hadoop-client-runtime</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
diff --git a/flink/flink-scala-2.12/pom.xml b/flink/flink-scala-2.12/pom.xml
index 43b822f57b..d711c9cc44 100644
--- a/flink/flink-scala-2.12/pom.xml
+++ b/flink/flink-scala-2.12/pom.xml
@@ -38,14 +38,14 @@
<flink.scala.version>2.12.7</flink.scala.version>
<flink.scala.binary.version>2.12</flink.scala.binary.version>
<flink.scala.compile.version>${flink.scala.version}</flink.scala.compile.version>
- <flink.hadoop.version>${hadoop2.7.version}</flink.hadoop.version>
- <hive.version>2.3.4</hive.version>
- <hiverunner.version>4.0.0</hiverunner.version>
+ <hive.version>2.3.7</hive.version>
+ <hive.guava.version>14.0.1</hive.guava.version>
+ <derby.version>10.14.2.0</derby.version>
+ <hiverunner.version>5.3.0</hiverunner.version>
<grpc.version>1.15.0</grpc.version>
<flink.bin.download.url>https://archive.apache.org/dist/flink/flink-${flink.version}/flink-${flink.version}-bin-scala_${flink.scala.binary.version}.tgz</flink.bin.download.url>
</properties>
-
<dependencies>
@@ -160,10 +160,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>*</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>*</artifactId>
- </exclusion>
</exclusions>
</dependency>
@@ -289,22 +285,12 @@
<version>${flink.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${flink.hadoop.version}</version>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${hive.guava.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
@@ -314,40 +300,6 @@
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${flink.hadoop.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- <version>${flink.hadoop.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- <version>${flink.hadoop.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${flink.hadoop.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <scope>test</scope>
- </dependency>
-
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId>
@@ -355,32 +307,156 @@
<scope>provided</scope>
<exclusions>
<exclusion>
- <artifactId>hadoop-auth</artifactId>
- <groupId>org.apache.hadoop</groupId>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-shims</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javolution</groupId>
+ <artifactId>javolution</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
</exclusion>
<exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
</exclusion>
<exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-hbase-compat-1.0</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libfb303</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.transaction</groupId>
+ <artifactId>transaction-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.orc</groupId>
+ <artifactId>orc-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
<exclusion>
- <groupId>jdk.tools</groupId>
- <artifactId>jdk.tools</artifactId>
+ <groupId>org.apache.ant</groupId>
+ <artifactId>ant</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.tdunning</groupId>
+ <artifactId>json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty.aggregate</groupId>
+ <artifactId>jetty-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty.orbit</groupId>
+ <artifactId>javax.servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-web</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.github.joshelser</groupId>
+ <artifactId>dropwizard-metrics-hadoop-metrics2-reporter</artifactId>
+ </exclusion>
+
+ <!-- org.apache.hive:hive-service-rpc -->
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </exclusion>
+
+ <!-- org.apache.hive:hive-serde -->
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.sf.opencsv</groupId>
+ <artifactId>opencsv</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-hadoop-bundle</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derby</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.pentaho</groupId>
+ <artifactId>pentaho-aggdesigner-algorithm</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -392,16 +468,16 @@
<scope>provided</scope>
<exclusions>
<exclusion>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-core</artifactId>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-vector-code-gen</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-druid</artifactId>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-llap-tez</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.calcite.avatica</groupId>
- <artifactId>avatica</artifactId>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-shims</artifactId>
</exclusion>
<exclusion>
<groupId>commons-codec</groupId>
@@ -412,20 +488,28 @@
<artifactId>commons-httpclient</artifactId>
</exclusion>
<exclusion>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr-runtime</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
+ <groupId>org.antlr</groupId>
+ <artifactId>ST4</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
+ <groupId>org.apache.ant</groupId>
+ <artifactId>ant</artifactId>
</exclusion>
<exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.ivy</groupId>
+ <artifactId>ivy</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
@@ -433,35 +517,63 @@
</exclusion>
<exclusion>
<groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
+ <artifactId>apache-curator</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.curator</groupId>
- <artifactId>apache-curator</artifactId>
+ <artifactId>curator-framework</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-druid</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.calcite.avatica</groupId>
+ <artifactId>avatica</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-avatica</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</exclusion>
<exclusion>
- <groupId>jline</groupId>
- <artifactId>jline</artifactId>
+ <groupId>stax</groupId>
+ <artifactId>stax-api</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
</exclusion>
<exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
+ <groupId>log4j</groupId>
+ <artifactId>apache-log4j-extras</artifactId>
</exclusion>
<exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.pentaho</groupId>
@@ -471,36 +583,222 @@
</dependency>
<dependency>
- <groupId>org.apache.hive.hcatalog</groupId>
- <artifactId>hive-webhcat-java-client</artifactId>
- <version>${hive.version}</version>
+ <groupId>com.klarna</groupId>
+ <artifactId>hiverunner</artifactId>
+ <version>${hiverunner.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
- <groupId>org.apache.calcite</groupId>
- <artifactId>*</artifactId>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-serde</artifactId>
</exclusion>
<exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
</exclusion>
<exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-service</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-contrib</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-hcatalog-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-webhcat-java-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- This dependency is no longer shipped with the JDK since Java
9.-->
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-auth</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-annotations</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-hdfs</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-shim</artifactId>
+ <groupId>org.apache.tez</groupId>
</exclusion>
<exclusion>
- <groupId>javax.jms</groupId>
<artifactId>jms</artifactId>
+ <groupId>javax.jms</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-contrib</artifactId>
+ <artifactId>hive-service</artifactId>
<version>${hive.version}</version>
<scope>test</scope>
<exclusions>
+ <exclusion>
+ <groupId>org.jamon</groupId>
+ <artifactId>jamon-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-metastore</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- This dependency is no longer shipped with the JDK since Java
9.-->
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-auth</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-client</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-annotations</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-hdfs</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-registry</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.pentaho</groupId>
<artifactId>pentaho-aggdesigner-algorithm</artifactId>
@@ -513,27 +811,62 @@
<artifactId>hive-hcatalog-core</artifactId>
<version>${hive.version}</version>
<scope>test</scope>
- <type>test-jar</type>
<exclusions>
<exclusion>
- <groupId>jline</groupId>
- <artifactId>jline</artifactId>
+ <groupId>org.jamon</groupId>
+ <artifactId>jamon-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
+ <artifactId>hadoop-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
+ <artifactId>hadoop-archives</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
+ <artifactId>hadoop-annotations</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-hdfs</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>apache-log4j-extras</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.pentaho</groupId>
@@ -543,21 +876,91 @@
</dependency>
<dependency>
- <groupId>net.jodah</groupId>
- <artifactId>concurrentunit</artifactId>
- <version>0.4.6</version>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-webhcat-java-client</artifactId>
+ <version>${hive.version}</version>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.jamon</groupId>
+ <artifactId>jamon-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jms</artifactId>
+ <groupId>javax.jms</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.pentaho</groupId>
+ <artifactId>pentaho-aggdesigner-algorithm</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
+ <!-- The derby we use suffers some security vulnerabilities. Explicitly
set it to test scope here. -->
<dependency>
- <groupId>com.klarna</groupId>
- <artifactId>hiverunner</artifactId>
- <version>${hiverunner.version}</version>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derby</artifactId>
+ <version>${derby.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <version>${hive.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.jamon</groupId>
+ <artifactId>jamon-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.pentaho</groupId>
@@ -567,9 +970,27 @@
</dependency>
<dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_${flink.scala.binary.version}</artifactId>
- <version>3.0.8</version>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>net.jodah</groupId>
+ <artifactId>concurrentunit</artifactId>
+ <version>0.4.6</version>
<scope>test</scope>
</dependency>
@@ -578,6 +999,14 @@
<artifactId>slf4j-api</artifactId>
</dependency>
+ <!-- hive enables log4j2 async log by default, which requires disruptor
available on classpath-->
+ <dependency>
+ <groupId>com.lmax</groupId>
+ <artifactId>disruptor</artifactId>
+ <version>3.4.4</version>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
@@ -904,31 +1333,5 @@
</dependency>
</dependencies>
</profile>
-
- <profile>
- <id>hive2</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
- <properties>
- <hive.version>2.3.4</hive.version>
- <hiverunner.version>4.0.0</hiverunner.version>
- </properties>
- </profile>
- <profile>
- <id>hive1</id>
- <properties>
- <hive.version>1.2.1</hive.version>
- <hiverunner.version>3.2.1</hiverunner.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>2.7.5</version>
- <scope>provided</scope>
- </dependency>
- </dependencies>
- </profile>
</profiles>
</project>
diff --git a/flink/flink-scala-2.12/src/main/resources/interpreter-setting.json
b/flink/flink-scala-2.12/src/main/resources/interpreter-setting.json
index 9da270aace..f5968526c7 100644
--- a/flink/flink-scala-2.12/src/main/resources/interpreter-setting.json
+++ b/flink/flink-scala-2.12/src/main/resources/interpreter-setting.json
@@ -155,7 +155,7 @@
"zeppelin.flink.hive.version": {
"envName": null,
"propertyName": null,
- "defaultValue": "2.3.4",
+ "defaultValue": "2.3.7",
"description": "Hive version that you would like to connect",
"type": "string"
},
diff --git
a/flink/flink-scala-2.12/src/main/scala/org/apache/zeppelin/flink/FlinkScalaInterpreter.scala
b/flink/flink-scala-2.12/src/main/scala/org/apache/zeppelin/flink/FlinkScalaInterpreter.scala
index c6ce85c985..16d38f0b43 100644
---
a/flink/flink-scala-2.12/src/main/scala/org/apache/zeppelin/flink/FlinkScalaInterpreter.scala
+++
b/flink/flink-scala-2.12/src/main/scala/org/apache/zeppelin/flink/FlinkScalaInterpreter.scala
@@ -491,7 +491,7 @@ abstract class FlinkScalaInterpreter(val properties:
Properties,
throw new InterpreterException("HIVE_CONF_DIR is not specified");
}
val database = properties.getProperty("zeppelin.flink.hive.database",
"default")
- val hiveVersion = properties.getProperty("zeppelin.flink.hive.version",
"2.3.4")
+ val hiveVersion = properties.getProperty("zeppelin.flink.hive.version",
"2.3.7")
val hiveCatalog = new HiveCatalog("hive", database, hiveConfDir.toString,
hiveVersion)
this.btenv.registerCatalog("hive", hiveCatalog)
this.btenv.useCatalog("hive")
diff --git
a/flink/flink-scala-2.12/src/test/java/org/apache/zeppelin/flink/FlinkSqlInterpreterTest.java
b/flink/flink-scala-2.12/src/test/java/org/apache/zeppelin/flink/FlinkSqlInterpreterTest.java
index e51042dc14..29940d4671 100644
---
a/flink/flink-scala-2.12/src/test/java/org/apache/zeppelin/flink/FlinkSqlInterpreterTest.java
+++
b/flink/flink-scala-2.12/src/test/java/org/apache/zeppelin/flink/FlinkSqlInterpreterTest.java
@@ -90,7 +90,7 @@ public abstract class FlinkSqlInterpreterTest {
p.setProperty("zeppelin.flink.enableHive", "true");
p.setProperty("taskmanager.managed.memory.size", "32");
p.setProperty("taskmanager.memory.task.off-heap.size", "80mb");
- p.setProperty("zeppelin.flink.hive.version", "2.3.4");
+ p.setProperty("zeppelin.flink.hive.version", "2.3.7");
p.setProperty("zeppelin.pyflink.useIPython", "false");
p.setProperty("local.number-taskmanager", "4");
p.setProperty("zeppelin.python.gatewayserver_address", "127.0.0.1");
diff --git a/flink/flink1.15-shims/pom.xml b/flink/flink1.15-shims/pom.xml
index d6affc169d..8d549134c9 100644
--- a/flink/flink1.15-shims/pom.xml
+++ b/flink/flink1.15-shims/pom.xml
@@ -26,9 +26,7 @@
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.zeppelin</groupId>
<artifactId>flink1.15-shims</artifactId>
- <version>0.12.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Zeppelin: Flink1.15 Shims</name>
diff --git a/flink/flink1.16-shims/pom.xml b/flink/flink1.16-shims/pom.xml
index 83fe36ab2f..2bd14cfc5e 100644
--- a/flink/flink1.16-shims/pom.xml
+++ b/flink/flink1.16-shims/pom.xml
@@ -26,9 +26,7 @@
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.zeppelin</groupId>
<artifactId>flink1.16-shims</artifactId>
- <version>0.12.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Zeppelin: Flink1.16 Shims</name>
diff --git a/hbase/pom.xml b/hbase/pom.xml
index 46a2e63dd4..e7c623346c 100644
--- a/hbase/pom.xml
+++ b/hbase/pom.xml
@@ -33,56 +33,15 @@
<properties>
<!--library versions-->
<interpreter.name>hbase</interpreter.name>
- <hbase.hbase.version>2.4.12</hbase.hbase.version>
- <hbase.hadoop.version>${hadoop2.7.version}</hbase.hadoop.version>
<jruby.version>1.6.8</jruby.version>
- <protobuf.version>2.5.0</protobuf.version>
- <jline.version>2.12.1</jline.version>
-
</properties>
<dependencies>
-
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-exec</artifactId>
- <version>${commons.exec.version}</version>
- </dependency>
<dependency>
<groupId>org.jruby</groupId>
<artifactId>jruby-complete</artifactId>
<version>${jruby.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- <version>${hbase.hadoop.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <version>${hbase.hadoop.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>${hbase.hbase.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-annotations</artifactId>
- <version>${hbase.hbase.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- <version>${protobuf.version}</version>
- </dependency>
- <dependency>
- <groupId>jline</groupId>
- <artifactId>jline</artifactId>
- <version>${jline.version}</version>
- </dependency>
</dependencies>
<build>
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 4cc79044c0..35be95c71d 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -35,7 +35,7 @@
<!--library versions-->
<interpreter.name>jdbc</interpreter.name>
<postgresql.version>42.7.2</postgresql.version>
- <hadoop.version>${hadoop3.1.version}</hadoop.version>
+ <hadoop.version>${hadoop3.3.version}</hadoop.version>
<h2.version>2.2.220</h2.version>
<commons.dbcp2.version>2.0.1</commons.dbcp2.version>
<hive3.version>3.1.3</hive3.version>
@@ -85,65 +85,11 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
+ <artifactId>hadoop-client-runtime</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
-
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>jackrabbit-webdav</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jgit</groupId>
- <artifactId>org.eclipse.jgit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-jdbc</artifactId>
@@ -170,21 +116,25 @@
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-mapreduce</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ </exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- <version>4.4.1</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.5.13</version>
- </dependency>
-
<dependency>
<groupId>net.jodah</groupId>
<artifactId>concurrentunit</artifactId>
@@ -233,128 +183,6 @@
</dependency>
</dependencies>
</profile>
-
- <profile>
- <id>jdbc-hadoop2</id>
- <properties>
- <hadoop-common.version>${hadoop2.7.version}</hadoop-common.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop-common.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
-
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>jackrabbit-webdav</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jgit</groupId>
- <artifactId>org.eclipse.jgit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </profile>
-
- <profile>
- <id>jdbc-hadoop3</id>
- <properties>
- <hadoop-common.version>${hadoop3.0.version}</hadoop-common.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop-common.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
-
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>jackrabbit-webdav</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jgit</groupId>
- <artifactId>org.eclipse.jgit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </profile>
</profiles>
</project>
diff --git a/livy/pom.xml b/livy/pom.xml
index 8ad4917abf..e004b8424e 100644
--- a/livy/pom.xml
+++ b/livy/pom.xml
@@ -41,7 +41,7 @@
<!--test library versions-->
<livy.version>0.7.1-incubating</livy.version>
<spark.version>2.4.8</spark.version>
- <hadoop.version>${hadoop2.7.version}</hadoop.version>
+ <hadoop.version>${hadoop3.3.version}</hadoop.version>
</properties>
<dependencies>
@@ -80,28 +80,24 @@
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
+ <artifactId>hadoop-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- </exclusion>
- <exclusion>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
</exclusion>
<exclusion>
+ <artifactId>hadoop-hdfs</artifactId>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
</exclusion>
<exclusion>
+ <artifactId>hadoop-yarn-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
</exclusions>
</dependency>
@@ -177,125 +173,25 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <classifier>tests</classifier>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <classifier>tests</classifier>
+ <artifactId>hadoop-client-api</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
+ <artifactId>hadoop-client-runtime</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
+ <artifactId>hadoop-client-minicluster</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-tests</artifactId>
- <classifier>tests</classifier>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
</dependencies>
<build>
diff --git a/pom.xml b/pom.xml
index 5dd9007c53..f840d7915b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,8 +101,8 @@
<!-- These two lines could be changed like `maven.compiler.release` after
updating JDK11 -->
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
- <scala.version>${scala.2.11.version}</scala.version>
- <scala.binary.version>2.11</scala.binary.version>
+ <scala.version>${scala.2.12.version}</scala.version>
+ <scala.binary.version>2.12</scala.binary.version>
<scala.2.11.version>2.11.12</scala.2.11.version>
<scala.2.12.version>2.12.17</scala.2.12.version>
<scalatest.version>3.2.15</scalatest.version>
@@ -139,17 +139,12 @@
<maven.version>3.6.3</maven.version>
<dropwizard.version>4.1.14</dropwizard.version>
<micrometer.version>1.6.0</micrometer.version>
+ <findbugs.jsr305.version>3.0.2</findbugs.jsr305.version>
- <hadoop2.7.version>2.7.7</hadoop2.7.version>
- <hadoop3.0.version>3.0.3</hadoop3.0.version>
- <hadoop3.1.version>3.1.3</hadoop3.1.version>
<hadoop3.2.version>3.2.4</hadoop3.2.version>
<hadoop3.3.version>3.3.6</hadoop3.3.version>
- <hadoop.version>${hadoop2.7.version}</hadoop.version>
+ <hadoop.version>${hadoop3.3.version}</hadoop.version>
<hadoop.deps.scope>provided</hadoop.deps.scope>
- <hadoop-client-api.artifact>hadoop-client</hadoop-client-api.artifact>
-
<hadoop-client-runtime.artifact>hadoop-yarn-api</hadoop-client-runtime.artifact>
-
<hadoop-client-minicluster.artifact>hadoop-client</hadoop-client-minicluster.artifact>
<quartz.scheduler.version>2.3.2</quartz.scheduler.version>
<jettison.version>1.5.4</jettison.version>
@@ -339,14 +334,6 @@
<version>${commons.configuration2.version}</version>
</dependency>
- <!-- force the latest commons-lang version
- hadoop 2.7 has an older version as transitive dependency -->
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>2.6</version>
- </dependency>
-
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
@@ -371,6 +358,13 @@
<version>${commons.cli.version}</version>
</dependency>
+ <!-- Use the latest annotation version of findbugs in all modules -->
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ <version>${findbugs.jsr305.version}</version>
+ </dependency>
+
<!-- Apache Shiro -->
<dependency>
<groupId>org.apache.shiro</groupId>
@@ -383,770 +377,28 @@
<version>${shiro.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.shiro</groupId>
- <artifactId>shiro-config-core</artifactId>
- <version>${shiro.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.bouncycastle</groupId>
- <artifactId>bcpkix-jdk15on</artifactId>
- <version>${bouncycastle.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jettison</groupId>
- <artifactId>jettison</artifactId>
- <version>${jettison.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>${hadoop-client-api.artifact}</artifactId>
- <version>${hadoop.version}</version>
- <scope>${hadoop.deps.scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>jackrabbit-webdav</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jgit</groupId>
- <artifactId>org.eclipse.jgit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xml-apis</groupId>
- <artifactId>xml-apis</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.nimbusds</groupId>
- <artifactId>nimbus-jose-jwt</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-xml</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-configuration2</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-webapp</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-jaxb-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- <!-- using jcl-over-slf4j instead -->
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- <version>${hadoop.version}</version>
- <scope>${hadoop.deps.scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.contribs</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- <version>${hadoop.version}</version>
- <scope>${hadoop.deps.scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>jackrabbit-webdav</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jgit</groupId>
- <artifactId>org.eclipse.jgit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xml-apis</groupId>
- <artifactId>xml-apis</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- </exclusion>
- <!-- using jcl-over-slf4j instead -->
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <version>${hadoop.version}</version>
- <scope>${hadoop.deps.scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>jackrabbit-webdav</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jgit</groupId>
- <artifactId>org.eclipse.jgit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xml-apis</groupId>
- <artifactId>xml-apis</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- </exclusion>
- <!-- using jcl-over-slf4j instead -->
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
-
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-tests</artifactId>
- <version>${hadoop.version}</version>
- <classifier>tests</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>jackrabbit-webdav</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jgit</groupId>
- <artifactId>org.eclipse.jgit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xml-apis</groupId>
- <artifactId>xml-apis</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-jaxrs</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-xc</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.xml.bind</groupId>
- <artifactId>jaxb-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.zaxxer</groupId>
- <artifactId>HikariCP-java7</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-jaxb-annotations</artifactId>
- </exclusion>
- <!-- using jcl-over-slf4j instead -->
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- </exclusions>
+ <groupId>org.apache.shiro</groupId>
+ <artifactId>shiro-config-core</artifactId>
+ <version>${shiro.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <scope>${hadoop.deps.scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>jackrabbit-webdav</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jgit</groupId>
- <artifactId>org.eclipse.jgit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xml-apis</groupId>
- <artifactId>xml-apis</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils</artifactId>
- </exclusion>
- <exclusion>
- <groupId> commons-beanutils</groupId>
- <artifactId>commons-beanutils-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-configuration2</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-webapp</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.nimbusds</groupId>
- <artifactId>nimbus-jose-jwt</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- <!-- using jcl-over-slf4j instead -->
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.jamesmurty.utils</groupId>
- <artifactId>java-xmlbuilder</artifactId>
- </exclusion>
- </exclusions>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
+ <version>${bouncycastle.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jettison</groupId>
+ <artifactId>jettison</artifactId>
+ <version>${jettison.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
<version>${hadoop.version}</version>
- <classifier>tests</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>jackrabbit-webdav</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jgit</groupId>
- <artifactId>org.eclipse.jgit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xml-apis</groupId>
- <artifactId>xml-apis</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-configuration2</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-webapp</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.nimbusds</groupId>
- <artifactId>nimbus-jose-jwt</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- <!-- using jcl-over-slf4j instead -->
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- </exclusions>
+ <scope>${hadoop.deps.scope}</scope>
</dependency>
<!-- Test libraries -->
@@ -1208,196 +460,22 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>jackrabbit-webdav</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jgit</groupId>
- <artifactId>org.eclipse.jgit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xml-apis</groupId>
- <artifactId>xml-apis</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <!-- using jcl-over-slf4j instead -->
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- <classifier>tests</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>jackrabbit-webdav</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jgit</groupId>
- <artifactId>org.eclipse.jgit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xml-apis</groupId>
- <artifactId>xml-apis</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <!-- using jcl-over-slf4j instead -->
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>${hadoop-client-runtime.artifact}</artifactId>
+ <artifactId>hadoop-client-runtime</artifactId>
<version>${hadoop.version}</version>
<scope>${hadoop.deps.scope}</scope>
<exclusions>
<exclusion>
- <groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
+ <groupId>commons-logging</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>${hadoop-client-minicluster.artifact}</artifactId>
+ <artifactId>hadoop-client-minicluster</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
-
</dependencies>
</dependencyManagement>
diff --git a/rlang/pom.xml b/rlang/pom.xml
index 5a7099d13c..484380de79 100644
--- a/rlang/pom.xml
+++ b/rlang/pom.xml
@@ -105,12 +105,6 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.12</artifactId>
<version>${spark.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client-api</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
@@ -121,22 +115,12 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- <scope>compile</scope>
+ <artifactId>hadoop-client-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
+ <artifactId>hadoop-client-runtime</artifactId>
</dependency>
<dependency>
@@ -145,7 +129,6 @@
<version>1.4.9</version>
<scope>test</scope>
</dependency>
-
</dependencies>
<build>
diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml
index a977ab6978..3156a17053 100644
--- a/spark/interpreter/pom.xml
+++ b/spark/interpreter/pom.xml
@@ -166,36 +166,6 @@
<artifactId>spark-core_${spark.scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
diff --git a/spark/spark-scala-parent/pom.xml b/spark/spark-scala-parent/pom.xml
index adff9ad204..cc6fef8588 100644
--- a/spark/spark-scala-parent/pom.xml
+++ b/spark/spark-scala-parent/pom.xml
@@ -80,21 +80,6 @@
<scope>provided</scope>
</dependency>
- <!-- Use provided scope, hadoop dependencies are only for compilation
-->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <scope>provided</scope>
- </dependency>
-
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
diff --git a/spark/spark-shims/pom.xml b/spark/spark-shims/pom.xml
index 10a2231c2a..80112b7801 100644
--- a/spark/spark-shims/pom.xml
+++ b/spark/spark-shims/pom.xml
@@ -38,8 +38,7 @@
-->
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
+ <artifactId>hadoop-client-api</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
diff --git a/submarine/pom.xml b/submarine/pom.xml
index 831f728204..e162316d50 100644
--- a/submarine/pom.xml
+++ b/submarine/pom.xml
@@ -33,7 +33,7 @@
<properties>
<!--library versions-->
<interpreter.name>submarine</interpreter.name>
- <hadoop.version>${hadoop2.7.version}</hadoop.version>
+ <hadoop.version>${hadoop3.3.version}</hadoop.version>
<jinjava.version>2.5.4</jinjava.version>
<squirrel.version>0.3.8</squirrel.version>
<guava.version>24.1.1-jre</guava.version>
@@ -79,57 +79,8 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-client-runtime</artifactId>
<version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-xc</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-jaxrs</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>com.hubspot.jinjava</groupId>
diff --git a/zeppelin-interpreter-integration/pom.xml
b/zeppelin-interpreter-integration/pom.xml
index 85234ebad6..c79306eb3f 100644
--- a/zeppelin-interpreter-integration/pom.xml
+++ b/zeppelin-interpreter-integration/pom.xml
@@ -58,10 +58,6 @@
<artifactId>zeppelin-zengine</artifactId>
<version>${project.version}</version>
<exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
@@ -69,12 +65,6 @@
</exclusions>
</dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>20.0</version>
- </dependency>
-
<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-server</artifactId>
@@ -99,12 +89,6 @@
<version>${project.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
@@ -178,21 +162,19 @@
<properties>
<hadoop.version>${hadoop3.3.version}</hadoop.version>
-
<hadoop-client-runtime.artifact>hadoop-client-runtime</hadoop-client-runtime.artifact>
-
<hadoop-client-minicluster.artifact>hadoop-client-minicluster</hadoop-client-minicluster.artifact>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>${hadoop-client-runtime.artifact}</artifactId>
+ <artifactId>hadoop-client-runtime</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>${hadoop-client-minicluster.artifact}</artifactId>
+ <artifactId>hadoop-client-minicluster</artifactId>
<scope>test</scope>
</dependency>
diff --git
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/FlinkIntegrationTest.java
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/FlinkIntegrationTest.java
index 72dff00c9e..0d8167ab78 100644
---
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/FlinkIntegrationTest.java
+++
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/FlinkIntegrationTest.java
@@ -18,6 +18,7 @@
package org.apache.zeppelin.integration;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@@ -55,7 +56,6 @@ public abstract class FlinkIntegrationTest {
private static InterpreterSettingManager interpreterSettingManager;
private String flinkVersion;
- private String scalaVersion;
private String hadoopHome;
private String flinkHome;
@@ -63,15 +63,15 @@ public abstract class FlinkIntegrationTest {
LOGGER.info("Testing FlinkVersion: " + flinkVersion);
LOGGER.info("Testing ScalaVersion: " + scalaVersion);
this.flinkVersion = flinkVersion;
- this.scalaVersion = scalaVersion;
this.flinkHome = DownloadUtils.downloadFlink(flinkVersion, scalaVersion);
- this.hadoopHome = DownloadUtils.downloadHadoop("2.7.7");
+ this.hadoopHome = DownloadUtils.downloadHadoop("3.3.6");
}
@BeforeAll
public static void setUp() throws IOException {
Configuration conf = new Configuration();
conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
+ conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "target/hadoop-minicluster");
hadoopCluster = new MiniHadoopCluster(conf);
hadoopCluster.start();
@@ -141,7 +141,6 @@ public abstract class FlinkIntegrationTest {
interpreterSettingManager.close();
}
- // TODO(zjffdu) enable it when make yarn integration test work
@Test
public void testYarnMode() throws IOException, InterpreterException,
YarnException {
InterpreterSetting flinkInterpreterSetting =
interpreterSettingManager.getInterpreterSettingByName("flink");
diff --git
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest32.java
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest32.java
index dfd9be366d..27c511e64e 100644
---
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest32.java
+++
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest32.java
@@ -17,7 +17,6 @@
package org.apache.zeppelin.integration;
-import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Nested;
@@ -25,16 +24,6 @@ import java.io.IOException;
public class SparkIntegrationTest32 {
- @Nested
- @DisplayName("Hadoop2")
- public class Hadoop2 extends SparkIntegrationTest {
-
- @BeforeEach
- public void downloadSpark() throws IOException {
- prepareSpark("3.2.0", "2.7");
- }
- }
-
@Nested
@DisplayName("Hadoop3")
public class Hadoop3 extends SparkIntegrationTest {
diff --git
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest33.java
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest33.java
index 73846fcf64..9183257184 100644
---
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest33.java
+++
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest33.java
@@ -24,16 +24,6 @@ import java.io.IOException;
public class SparkIntegrationTest33 {
- @Nested
- @DisplayName("Hadoop2")
- public class Hadoop2 extends SparkIntegrationTest {
-
- @BeforeEach
- public void downloadSpark() throws IOException {
- prepareSpark("3.3.0", "2");
- }
- }
-
@Nested
@DisplayName("Hadoop3")
public class Hadoop3 extends SparkIntegrationTest {
diff --git
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest34.java
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest34.java
index 8af3ee6d5e..d66bdad053 100644
---
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest34.java
+++
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest34.java
@@ -22,17 +22,7 @@ import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Nested;
import java.io.IOException;
-public class SparkIntegrationTest34 extends SparkIntegrationTest {
-
- @Nested
- @DisplayName("Hadoop2")
- public class Hadoop2 extends SparkIntegrationTest {
-
- @BeforeEach
- public void downloadSpark() throws IOException {
- prepareSpark("3.4.0", "2");
- }
- }
+public class SparkIntegrationTest34 {
@Nested
@DisplayName("Hadoop3")
diff --git
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinFlinkClusterTest.java
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinFlinkClusterTest.java
index ac1952494d..c14e002650 100644
---
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinFlinkClusterTest.java
+++
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinFlinkClusterTest.java
@@ -45,11 +45,9 @@ import java.nio.file.Files;
public abstract class ZeppelinFlinkClusterTest extends AbstractTestRestApi {
private static final Logger LOGGER =
LoggerFactory.getLogger(ZeppelinFlinkClusterTest.class);
- private String flinkVersion;
private String flinkHome;
public void download(String flinkVersion, String scalaVersion) {
- this.flinkVersion = flinkVersion;
LOGGER.info("Testing FlinkVersion: " + flinkVersion);
LOGGER.info("Testing ScalaVersion: " + scalaVersion);
this.flinkHome = DownloadUtils.downloadFlink(flinkVersion, scalaVersion);
diff --git
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest32.java
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest32.java
index 18d5b70177..1f1b769245 100644
---
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest32.java
+++
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest32.java
@@ -23,16 +23,6 @@ import org.junit.jupiter.api.Nested;
public class ZeppelinSparkClusterTest32 {
- @Nested
- @DisplayName("Hadoop2")
- public class Hadoop2 extends ZeppelinSparkClusterTest {
-
- @BeforeEach
- public void downloadSpark() throws Exception {
- prepareSpark("3.2.0", "2.7");
- }
- }
-
@Nested
@DisplayName("Hadoop3")
public class Hadoop3 extends ZeppelinSparkClusterTest {
diff --git
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest33.java
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest33.java
index dbfacd9e40..43eb620f2d 100644
---
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest33.java
+++
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest33.java
@@ -23,16 +23,6 @@ import org.junit.jupiter.api.Nested;
public class ZeppelinSparkClusterTest33 {
- @Nested
- @DisplayName("Hadoop2")
- public class Hadoop2 extends ZeppelinSparkClusterTest {
-
- @BeforeEach
- public void downloadSpark() throws Exception {
- prepareSpark("3.3.0", "2");
- }
- }
-
@Nested
@DisplayName("Hadoop3")
public class Hadoop3 extends ZeppelinSparkClusterTest {
diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml
index 859edab89d..91c60860c5 100644
--- a/zeppelin-interpreter/pom.xml
+++ b/zeppelin-interpreter/pom.xml
@@ -253,74 +253,19 @@
<profiles>
<profile>
- <id>hadoop2</id>
-
+ <id>hadoop3</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
- <hadoop.version>${hadoop2.7.version}</hadoop.version>
- <hadoop-client-api.artifact>hadoop-client</hadoop-client-api.artifact>
-
<hadoop-client-runtime.artifact>hadoop-yarn-api</hadoop-client-runtime.artifact>
-
<hadoop-client-minicluster.artifact>hadoop-client</hadoop-client-minicluster.artifact>
+ <hadoop.version>${hadoop3.3.version}</hadoop.version>
</properties>
<dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-
- </profile>
- <profile>
- <id>hadoop3</id>
-
- <properties>
- <hadoop.version>${hadoop3.2.version}</hadoop.version>
-
<hadoop-client-api.artifact>hadoop-client-api</hadoop-client-api.artifact>
-
<hadoop-client-runtime.artifact>hadoop-client-runtime</hadoop-client-runtime.artifact>
-
<hadoop-client-minicluster.artifact>hadoop-client-minicluster</hadoop-client-minicluster.artifact>
- </properties>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>${hadoop-client-runtime.artifact}</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>${hadoop-client-minicluster.artifact}</artifactId>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </exclusion>
- </exclusions>
+ <artifactId>hadoop-client-runtime</artifactId>
</dependency>
</dependencies>
diff --git a/zeppelin-plugins/launcher/yarn/pom.xml
b/zeppelin-plugins/launcher/yarn/pom.xml
index bbb9925114..5f2dcfc363 100644
--- a/zeppelin-plugins/launcher/yarn/pom.xml
+++ b/zeppelin-plugins/launcher/yarn/pom.xml
@@ -73,44 +73,18 @@
<profiles>
<profile>
- <id>hadoop2</id>
-
+ <id>hadoop3</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
-
- <properties>
- <hadoop.version>${hadoop2.7.version}</hadoop.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- <version>${hadoop.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <scope>provided</scope>
- <version>${hadoop.version}</version>
- </dependency>
- </dependencies>
-
- </profile>
- <profile>
- <id>hadoop3</id>
-
<properties>
- <hadoop.version>${hadoop3.2.version}</hadoop.version>
-
<hadoop-client-runtime.artifact>hadoop-client-runtime</hadoop-client-runtime.artifact>
+ <hadoop.version>${hadoop3.3.version}</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
<scope>provided</scope>
<version>${hadoop.version}</version>
</dependency>
diff --git a/zeppelin-plugins/notebookrepo/filesystem/pom.xml
b/zeppelin-plugins/notebookrepo/filesystem/pom.xml
index c7a07db519..6e8858802b 100644
--- a/zeppelin-plugins/notebookrepo/filesystem/pom.xml
+++ b/zeppelin-plugins/notebookrepo/filesystem/pom.xml
@@ -49,42 +49,15 @@
<profiles>
<profile>
- <id>hadoop2</id>
+ <id>hadoop3</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
- <hadoop.version>${hadoop2.7.version}</hadoop.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- <version>${hadoop.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <scope>provided</scope>
- <version>${hadoop.version}</version>
- </dependency>
- </dependencies>
- </profile>
-
- <profile>
- <id>hadoop3</id>
- <properties>
- <hadoop.version>${hadoop3.2.version}</hadoop.version>
+ <hadoop.version>${hadoop3.3.version}</hadoop.version>
</properties>
<dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <scope>provided</scope>
- <version>${hadoop.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-runtime</artifactId>
@@ -100,85 +73,10 @@
</dependencies>
</profile>
- <profile>
- <id>hadoop2-azure</id>
- <properties>
- <hadoop.version>${hadoop2.7.version}</hadoop.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-azure</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.jcraf</groupId>
- <artifactId>jsch</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.microsoft.azure</groupId>
- <artifactId>azure-data-lake-store-sdk</artifactId>
- <version>${adl.sdk.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </profile>
-
- <profile>
- <id>hadoop2-aws</id>
- <properties>
- <hadoop.version>${hadoop2.7.version}</hadoop.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-aws</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </profile>
-
<profile>
<id>hadoop3-azure</id>
<properties>
- <hadoop.version>${hadoop3.0.version}</hadoop.version>
+ <hadoop.version>${hadoop3.3.version}</hadoop.version>
</properties>
<dependencies>
<dependency>
@@ -257,7 +155,7 @@
<profile>
<id>hadoop3-aws</id>
<properties>
- <hadoop.version>${hadoop3.0.version}</hadoop.version>
+ <hadoop.version>${hadoop3.3.version}</hadoop.version>
</properties>
<dependencies>
<dependency>
diff --git a/zeppelin-plugins/notebookrepo/s3/pom.xml
b/zeppelin-plugins/notebookrepo/s3/pom.xml
index d6f8c82e97..1df29eee33 100644
--- a/zeppelin-plugins/notebookrepo/s3/pom.xml
+++ b/zeppelin-plugins/notebookrepo/s3/pom.xml
@@ -66,30 +66,6 @@
</exclusion>
</exclusions>
</dependency>
- <!-- Fix enforcer plugin for hadoop2
- Can be removed when hadoop2 is removed.
- hadoop2 ships with a very old version.
- Keep the version in sync with s3proxy test dependency
- -->
- <dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- <version>5.0.1</version>
- </dependency>
- <!-- Fix enforcer plugin
- org.apache.hadoop:hadoop-yarn-client 2.7 and org.gaul:s3proxy
are shipped with javax.xml.bind:jaxb-api
- -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- <version>${hadoop2.7.version}</version>
- <exclusions>
- <exclusion>
- <groupId>javax.xml.bind</groupId>
- <artifactId>jaxb-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
</dependencies>
</dependencyManagement>
<dependencies>
diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml
index 2652f19e9f..41c0ae198c 100644
--- a/zeppelin-server/pom.xml
+++ b/zeppelin-server/pom.xml
@@ -417,7 +417,7 @@
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
- <configuration combine.children="append">
+ <configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
<argLine>-Xmx3g -Xms1g -Dfile.encoding=UTF-8</argLine>
@@ -475,29 +475,13 @@
</properties>
</profile>
- <profile>
- <id>hadoop2</id>
-
- <properties>
- <hadoop.version>${hadoop2.7.version}</hadoop.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <scope>${hadoop.deps.scope}</scope>
- </dependency>
- </dependencies>
-
- </profile>
<profile>
<id>hadoop3</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
- <hadoop.version>${hadoop3.2.version}</hadoop.version>
+ <hadoop.version>${hadoop3.3.version}</hadoop.version>
<hadoop-client-api.artifact>hadoop-client-api</hadoop-client-api.artifact>
<hadoop-client-runtime.artifact>hadoop-client-runtime</hadoop-client-runtime.artifact>
</properties>
diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml
index 6bf1a7414d..cabe6442e1 100644
--- a/zeppelin-zengine/pom.xml
+++ b/zeppelin-zengine/pom.xml
@@ -38,10 +38,22 @@
<org.reflections.version>0.9.8</org.reflections.version>
<xml.apis.version>1.4.01</xml.apis.version>
<commons.vfs2.version>2.6.0</commons.vfs2.version>
+ <!-- Should match which the version from commons-vfs2 -->
+ <jackson.annocations.version>2.9.8</jackson.annocations.version>
<eclipse.jgit.version>4.5.4.201711221230-r</eclipse.jgit.version>
<eirslett.version>1.6</eirslett.version>
</properties>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${jackson.annocations.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
@@ -198,10 +210,7 @@
<artifactId>commons-vfs2</artifactId>
<version>${commons.vfs2.version}</version>
<exclusions>
- <exclusion>
- <groupId>org.codehaus.plexus</groupId>
- <artifactId>plexus-utils</artifactId>
- </exclusion>
+ <!-- use hadoop-client-runtime to mount hadoop filesystems -->
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-client</artifactId>
@@ -308,62 +317,18 @@
<profiles>
<profile>
- <id>hadoop2</id>
-
+ <id>hadoop3</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
-
<properties>
- <hadoop.version>${hadoop2.7.version}</hadoop.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- </dependency>
- </dependencies>
-
- </profile>
- <profile>
- <id>hadoop3</id>
-
- <properties>
- <hadoop.version>${hadoop3.2.version}</hadoop.version>
-
<hadoop-client-api.artifact>hadoop-client-api</hadoop-client-api.artifact>
-
<hadoop-client-runtime.artifact>hadoop-client-runtime</hadoop-client-runtime.artifact>
-
<hadoop-client-minicluster.artifact>hadoop-client-minicluster</hadoop-client-minicluster.artifact>
-
+ <hadoop.version>${hadoop3.3.version}</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>${hadoop-client-runtime.artifact}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>${hadoop-client-minicluster.artifact}</artifactId>
- <scope>test</scope>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </exclusion>
- </exclusions>
+ <artifactId>hadoop-client-runtime</artifactId>
</dependency>
</dependencies>
</profile>
diff --git
a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/integration/DownloadUtils.java
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/integration/DownloadUtils.java
index 8cbbdacd27..6310c1d60a 100644
---
a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/integration/DownloadUtils.java
+++
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/integration/DownloadUtils.java
@@ -78,10 +78,13 @@ public class DownloadUtils {
+ flinkVersion + "/flink-hadoop-compatibility_" +
scalaVersion + "-" + flinkVersion + ".jar",
"-P", targetFlinkHomeFolder + "/lib"});
runShellCommand(new String[]{"wget",
-
"https://repo1.maven.org/maven2/org/apache/hive/hive-exec/2.3.4/hive-exec-2.3.4.jar";,
+
"https://repo1.maven.org/maven2/org/apache/hive/hive-exec/2.3.7/hive-exec-2.3.7.jar";,
"-P", targetFlinkHomeFolder + "/lib"});
runShellCommand(new String[]{"wget",
-
"https://repo1.maven.org/maven2/org/apache/flink/flink-shaded-hadoop2-uber/2.7.5-1.8.1/flink-shaded-hadoop2-uber-2.7.5-1.8.1.jar";,
+
"https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-api/3.3.6/hadoop-client-api-3.3.6.jar";,
+ "-P", targetFlinkHomeFolder + "/lib"});
+ runShellCommand(new String[]{"wget",
+
"https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-runtime/3.3.6/hadoop-client-runtime-3.3.6.jar";,
"-P", targetFlinkHomeFolder + "/lib"});
runShellCommand(new String[]{"wget",
"https://repo1.maven.org/maven2/org/apache/flink/flink-table-api-scala_"; +
scalaVersion + "/"
