This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new 938ed0c ZEPPELIN-4005. SparkRInterpreter is broken for spark 2.1.3
and 2.2.2
938ed0c is described below
commit 938ed0c36f03a77b9cb271c428032ef821241c00
Author: Jeff Zhang <zjf...@apache.org>
AuthorDate: Fri Feb 15 17:25:55 2019 +0800
ZEPPELIN-4005. SparkRInterpreter is broken for spark 2.1.3 and 2.2.2
### What is this PR for?
The security patch about the security communication between JVM and R
process is backported to spark 2.13 and 2.2.2. And Zeppelin didn't catch that.
This PR fix it and also update the test case.
### What type of PR is it?
[Bug Fix]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://jira.apache.org/jira/browse/ZEPPELIN-4005
### How should this be tested?
* CI pass
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zjf...@apache.org>
Closes #3307 from zjffdu/ZEPPELIN-4005 and squashes the following commits:
fcae651a9 [Jeff Zhang] ZEPPELIN-4005. SparkRInterpreter is broken for spark
2.1.3 and 2.2.2
---
.../org/apache/zeppelin/spark/SparkVersion.java | 21 ++++++++++++++++-----
.../org/apache/zeppelin/spark/SparkVersionTest.java | 8 ++++++++
.../integration/SparkIntegrationTest21.java | 2 +-
.../integration/SparkIntegrationTest22.java | 2 +-
.../integration/ZeppelinSparkClusterTest21.java | 2 +-
.../integration/ZeppelinSparkClusterTest22.java | 2 +-
6 files changed, 28 insertions(+), 9 deletions(-)
diff --git
a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
index 6ee0015..42b279c 100644
---
a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
+++
b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
@@ -37,6 +37,9 @@ public class SparkVersion {
public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_3_0_0;
private int version;
+ private int majorVersion;
+ private int minorVersion;
+ private int patchVersion;
private String versionString;
SparkVersion(String versionString) {
@@ -51,11 +54,11 @@ public class SparkVersion {
}
String versions[] = numberPart.split("\\.");
- int major = Integer.parseInt(versions[0]);
- int minor = Integer.parseInt(versions[1]);
- int patch = Integer.parseInt(versions[2]);
+ this.majorVersion = Integer.parseInt(versions[0]);
+ this.minorVersion = Integer.parseInt(versions[1]);
+ this.patchVersion = Integer.parseInt(versions[2]);
// version is always 5 digits. (e.g. 2.0.0 -> 20000, 1.6.2 -> 10602)
- version = Integer.parseInt(String.format("%d%02d%02d", major, minor,
patch));
+ version = Integer.parseInt(String.format("%d%02d%02d", majorVersion,
minorVersion, patchVersion));
} catch (Exception e) {
logger.error("Can not recognize Spark version " + versionString +
". Assume it's a future release", e);
@@ -86,7 +89,10 @@ public class SparkVersion {
}
public boolean isSecretSocketSupported() {
- return this.newerThanEquals(SPARK_2_3_1);
+ return this.newerThanEquals(SparkVersion.SPARK_2_4_0) ||
+ this.newerThanEqualsPatchVersion(SPARK_2_3_1) ||
+
this.newerThanEqualsPatchVersion(SparkVersion.fromVersionString("2.2.2")) ||
+
this.newerThanEqualsPatchVersion(SparkVersion.fromVersionString("2.1.3"));
}
public boolean equals(Object versionToCompare) {
@@ -101,6 +107,11 @@ public class SparkVersion {
return version >= versionToCompare.version;
}
+ public boolean newerThanEqualsPatchVersion(SparkVersion versionToCompare) {
+ return version / 100 == versionToCompare.version / 100 &&
+ version % 100 >= versionToCompare.version % 100;
+ }
+
public boolean olderThan(SparkVersion versionToCompare) {
return version < versionToCompare.version;
}
diff --git
a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java
b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java
index 1f50d92..bc43e5d 100644
---
a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java
+++
b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java
@@ -55,6 +55,14 @@ public class SparkVersionTest {
assertTrue(SparkVersion.SPARK_2_0_0.olderThanEquals(SparkVersion.SPARK_2_0_0));
assertFalse(SparkVersion.SPARK_2_3_0.olderThan(SparkVersion.SPARK_2_0_0));
+ // test newerThanEqualsPatchVersion
+ assertTrue(SparkVersion.fromVersionString("2.3.1")
+
.newerThanEqualsPatchVersion(SparkVersion.fromVersionString("2.3.0")));
+ assertFalse(SparkVersion.fromVersionString("2.3.1")
+
.newerThanEqualsPatchVersion(SparkVersion.fromVersionString("2.3.2")));
+ assertFalse(SparkVersion.fromVersionString("2.3.1")
+
.newerThanEqualsPatchVersion(SparkVersion.fromVersionString("2.2.0")));
+
// conversion
assertEquals(20300, SparkVersion.SPARK_2_3_0.toNumber());
assertEquals("2.3.0", SparkVersion.SPARK_2_3_0.toString());
diff --git
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest21.java
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest21.java
index e60fa2b..37305cd 100644
---
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest21.java
+++
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest21.java
@@ -33,7 +33,7 @@ public class SparkIntegrationTest21 extends
SparkIntegrationTest{
@Parameterized.Parameters
public static List<Object[]> data() {
return Arrays.asList(new Object[][]{
- {"2.1.2"}
+ {"2.1.3"}
});
}
diff --git
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest22.java
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest22.java
index 35d245e..a400118 100644
---
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest22.java
+++
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest22.java
@@ -33,7 +33,7 @@ public class SparkIntegrationTest22 extends
SparkIntegrationTest{
@Parameterized.Parameters
public static List<Object[]> data() {
return Arrays.asList(new Object[][]{
- {"2.2.1"}
+ {"2.2.2"}
});
}
diff --git
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest21.java
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest21.java
index 4ac4676..fd98364 100644
---
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest21.java
+++
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest21.java
@@ -34,7 +34,7 @@ public class ZeppelinSparkClusterTest21 extends
ZeppelinSparkClusterTest {
@Parameterized.Parameters
public static List<Object[]> data() {
return Arrays.asList(new Object[][]{
- {"2.1.2"}
+ {"2.1.3"}
});
}
}
diff --git
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest22.java
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest22.java
index cc351dd..9b51e17 100644
---
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest22.java
+++
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest22.java
@@ -34,7 +34,7 @@ public class ZeppelinSparkClusterTest22 extends
ZeppelinSparkClusterTest {
@Parameterized.Parameters
public static List<Object[]> data() {
return Arrays.asList(new Object[][]{
- {"2.2.1"}
+ {"2.2.2"}
});
}
}
