This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin-on-parquet-v2 by this
push:
new 1b4e4c2 KYLIN-4694 Fix 'NoClassDefFoundError:
Lcom/esotericsoftware/kryo/io/Output' when query with sparder on yarn
1b4e4c2 is described below
commit 1b4e4c245e620f1c2ccfc1267c5f7bd9a382aa83
Author: Zhichao Zhang <441586...@qq.com>
AuthorDate: Thu Aug 13 23:20:12 2020 +0800
KYLIN-4694 Fix 'NoClassDefFoundError: Lcom/esotericsoftware/kryo/io/Output'
when query with sparder on yarn
Fix 'NoClassDefFoundError: Lcom/esotericsoftware/kryo/io/Output' when query
with sparder on yarn.
Root cause:
Class 'com/esotericsoftware/kryo/io/Output' is loaded by SparkClassLoader,
but there are some classes which are loaded by SparkClassLoader's parent
classloader using this class.
---
build/deploy/context.xml | 2 +-
.../spark/classloader/DebugTomcatClassLoader.java | 147 ----------------
.../spark/classloader/KylinItClassLoader.java | 174 -------------------
.../spark/classloader/KylinItSparkClassLoader.java | 189 ---------------------
.../kylin/spark/classloader/SparkClassLoader.java | 22 +--
.../kylin/spark/classloader/TomcatClassLoader.java | 84 +++------
6 files changed, 26 insertions(+), 592 deletions(-)
diff --git a/build/deploy/context.xml b/build/deploy/context.xml
index 46349ab..897e7bb 100644
--- a/build/deploy/context.xml
+++ b/build/deploy/context.xml
@@ -33,6 +33,6 @@
<Valve className="org.apache.catalina.valves.CometConnectionManagerValve"
/>
-->
- <Loader
loaderClass="org.apache.kylin.spark.classloader.DebugTomcatClassLoader"/>
+ <Loader
loaderClass="org.apache.kylin.spark.classloader.TomcatClassLoader"/>
</Context>
diff --git
a/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/DebugTomcatClassLoader.java
b/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/DebugTomcatClassLoader.java
deleted file mode 100644
index 8e6de5a..0000000
---
a/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/DebugTomcatClassLoader.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.spark.classloader;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.HashSet;
-import java.util.Set;
-import org.apache.catalina.loader.ParallelWebappClassLoader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DebugTomcatClassLoader extends ParallelWebappClassLoader {
- private static final String[] PARENT_CL_PRECEDENT_CLASSES = new String[] {
- // Java standard library:
- "com.sun.", "launcher.", "javax.", "org.ietf", "java", "org.omg",
"org.w3c", "org.xml", "sunw.",
- // logging
- "org.slf4j", "org.apache.commons.logging", "org.apache.log4j",
"org.apache.catalina", "org.apache.tomcat"};
- private static final String[] THIS_CL_PRECEDENT_CLASSES = new String[]
{"io.kyligence", "org.apache.kylin",
- "org.apache.calcite"};
- private static final String[] CODE_GEN_CLASS = new String[]
{"org.apache.spark.sql.catalyst.expressions.Object",
- "Baz"};
-
- private static final Set<String> wontFindClasses = new HashSet<>();
-
- static {
- wontFindClasses.add("Class");
- wontFindClasses.add("Object");
- wontFindClasses.add("org");
- wontFindClasses.add("java.lang.org");
- wontFindClasses.add("java.lang$org");
- wontFindClasses.add("java$lang$org");
- wontFindClasses.add("org.apache");
- wontFindClasses.add("org.apache.calcite");
- wontFindClasses.add("org.apache.calcite.runtime");
- wontFindClasses.add("org.apache.calcite.linq4j");
- wontFindClasses.add("Long");
- wontFindClasses.add("String");
- }
-
- private static Logger logger =
LoggerFactory.getLogger(DebugTomcatClassLoader.class);
- private SparkClassLoader sparkClassLoader;
-
- /**
- * Creates a DynamicClassLoader that can load classes dynamically
- * from jar files under a specific folder.
- *
- * @param parent the parent ClassLoader to set.
- */
- public DebugTomcatClassLoader(ClassLoader parent) throws IOException {
- super(parent);
- sparkClassLoader = new SparkClassLoader(this);
- ClassLoaderUtils.setSparkClassLoader(sparkClassLoader);
- ClassLoaderUtils.setOriginClassLoader(this);
- init();
- }
-
- public void init() {
-
- String classPath = System.getProperty("java.class.path");
- if (classPath == null) {
- throw new RuntimeException("");
- }
-
- String[] jars = classPath.split(":");
- for (String jar : jars) {
- try {
- URL url = new File(jar).toURI().toURL();
- addURL(url);
- } catch (MalformedURLException e) {
- e.printStackTrace();
- }
- }
- }
-
- @Override
- public Class<?> loadClass(String name, boolean resolve) throws
ClassNotFoundException {
- if (isWontFind(name)) {
- throw new ClassNotFoundException();
- }
- if (isCodeGen(name)) {
- throw new ClassNotFoundException();
- }
-
- if (name.startsWith("org.apache.kylin.spark.classloader")) {
- return parent.loadClass(name);
- }
- if (sparkClassLoader.classNeedPreempt(name)) {
- return sparkClassLoader.loadClass(name);
- }
- if (isParentCLPrecedent(name)) {
- logger.debug("Skipping exempt class " + name + " - delegating
directly to parent");
- return parent.loadClass(name);
- }
- return super.loadClass(name, resolve);
- }
-
- @Override
- public InputStream getResourceAsStream(String name) {
- if (sparkClassLoader.fileNeedPreempt(name)) {
- return sparkClassLoader.getResourceAsStream(name);
- }
- return super.getResourceAsStream(name);
-
- }
-
- private boolean isParentCLPrecedent(String name) {
- for (String exemptPrefix : PARENT_CL_PRECEDENT_CLASSES) {
- if (name.startsWith(exemptPrefix)) {
- return true;
- }
- }
- return false;
- }
-
- private boolean isWontFind(String name) {
- return wontFindClasses.contains(name);
- }
-
- private boolean isCodeGen(String name) {
- for (String exemptPrefix : CODE_GEN_CLASS) {
- if (name.startsWith(exemptPrefix)) {
- return true;
- }
- }
- return false;
- }
-}
diff --git
a/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/KylinItClassLoader.java
b/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/KylinItClassLoader.java
deleted file mode 100644
index c156ba9..0000000
---
a/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/KylinItClassLoader.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.spark.classloader;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.kylin.spark.classloader.ClassLoaderUtils.findFile;
-
-public class KylinItClassLoader extends URLClassLoader {
- private static final String[] PARENT_CL_PRECEDENT_CLASS = new String[] {
- // Java standard library:
- "com.sun.", "launcher.", "javax.", "org.ietf", "java", "org.omg",
"org.w3c", "org.xml", "sunw.",
- // logging
- "org.slf4j", "org.apache.commons.logging", "org.apache.log4j",
"sun", "org.apache.catalina",
- "org.apache.tomcat", };
- private static final String[] THIS_CL_PRECEDENT_CLASS = new String[]
{"io.kyligence", "org.apache.kylin",
- "org.apache.calcite"};
- private static final String[] CODE_GEN_CLASS = new String[]
{"org.apache.spark.sql.catalyst.expressions.Object"};
- public static KylinItClassLoader defaultClassLoad = null;
- private static Logger logger =
LoggerFactory.getLogger(KylinItClassLoader.class);
- public KylinItSparkClassLoader sparkClassLoader;
- ClassLoader parent;
-
- /**
- * Creates a DynamicClassLoader that can load classes dynamically
- * from jar files under a specific folder.
- *
- * @param parent the parent ClassLoader to set.
- */
- public KylinItClassLoader(ClassLoader parent) throws IOException {
- super(((URLClassLoader) getSystemClassLoader()).getURLs());
- this.parent = parent;
- sparkClassLoader = new KylinItSparkClassLoader(this);
- ClassLoaderUtils.setSparkClassLoader(sparkClassLoader);
- ClassLoaderUtils.setOriginClassLoader(this);
- defaultClassLoad = this;
- init();
- }
-
- public void init() {
-
- String classPath = System.getProperty("java.class.path");
- if (classPath == null) {
- throw new RuntimeException("");
- }
-
- String[] jars = classPath.split(":");
- for (String jar : jars) {
- if (jar.contains("spark-")) {
- continue;
- }
- try {
- URL url = new File(jar).toURI().toURL();
- addURL(url);
- } catch (MalformedURLException e) {
- e.printStackTrace();
- }
- }
- String sparkHome = System.getenv("SPARK_HOME");
- try {
- File sparkJar = findFile(sparkHome + "/jars", "spark-yarn_.*.jar");
- addURL(sparkJar.toURI().toURL());
- addURL(new
File("../examples/test_case_data/sandbox").toURI().toURL());
- } catch (MalformedURLException e) {
- e.printStackTrace();
- }
-
- }
-
- @Override
- public Class<?> loadClass(String name, boolean resolve) throws
ClassNotFoundException {
- if (isCodeGen(name)) {
- throw new ClassNotFoundException();
- }
- if (name.startsWith("org.apache.kylin.spark.classloader")) {
- return parent.loadClass(name);
- }
- if (isThisCLPrecedent(name)) {
- synchronized (getClassLoadingLock(name)) {
- // Check whether the class has already been loaded:
- Class<?> clasz = findLoadedClass(name);
- if (clasz != null) {
- logger.debug("Class " + name + " already loaded");
- } else {
- try {
- // Try to find this class using the URLs passed to
this ClassLoader
- logger.debug("Finding class: " + name);
- clasz = super.findClass(name);
- } catch (ClassNotFoundException e) {
- // Class not found using this ClassLoader, so delegate
to parent
- logger.debug("Class " + name + " not found -
delegating to parent");
- try {
- clasz = parent.loadClass(name);
- } catch (ClassNotFoundException e2) {
- // Class not found in this ClassLoader or in the
parent ClassLoader
- // Log some debug output before re-throwing
ClassNotFoundException
- logger.debug("Class " + name + " not found in
parent loader");
- throw e2;
- }
- }
- }
- return clasz;
- }
- }
- if (isParentCLPrecedent(name)) {
- logger.debug("Skipping exempt class " + name + " - delegating
directly to parent");
- return parent.loadClass(name);
- }
- if (sparkClassLoader.classNeedPreempt(name)) {
- return sparkClassLoader.loadClass(name);
- }
- return super.loadClass(name, resolve);
- }
-
- @Override
- public InputStream getResourceAsStream(String name) {
- if (sparkClassLoader.fileNeedPreempt(name)) {
- return sparkClassLoader.getResourceAsStream(name);
- }
- return super.getResourceAsStream(name);
-
- }
-
- private boolean isParentCLPrecedent(String name) {
- for (String exemptPrefix : PARENT_CL_PRECEDENT_CLASS) {
- if (name.startsWith(exemptPrefix)) {
- return true;
- }
- }
- return false;
- }
-
- private boolean isThisCLPrecedent(String name) {
- for (String exemptPrefix : THIS_CL_PRECEDENT_CLASS) {
- if (name.startsWith(exemptPrefix)) {
- return true;
- }
- }
- return false;
- }
-
- private boolean isCodeGen(String name) {
- for (String exemptPrefix : CODE_GEN_CLASS) {
- if (name.startsWith(exemptPrefix)) {
- return true;
- }
- }
- return false;
- }
-
-}
diff --git
a/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/KylinItSparkClassLoader.java
b/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/KylinItSparkClassLoader.java
deleted file mode 100644
index 5b4963b..0000000
---
a/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/KylinItSparkClassLoader.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.spark.classloader;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.HashSet;
-import java.util.Set;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.kylin.spark.classloader.ClassLoaderUtils.findFile;
-
-
-public class KylinItSparkClassLoader extends URLClassLoader {
- private static final String[] SPARK_CL_PREEMPT_CLASSES = new String[]
{"org.apache.spark", "scala.",
- "org.spark_project"
- // "javax.ws.rs.core.Application",
- // "javax.ws.rs.core.UriBuilder",
"org.glassfish.jersey", "javax.ws.rs.ext"
- //user javax.ws.rs.api 2.01 not jersey-core-1.9.jar
- };
- private static final String[] SPARK_CL_PREEMPT_FILES = new String[]
{"spark-version-info.properties",
- "HiveClientImpl", "org/apache/spark"};
-
- private static final String[] THIS_CL_PRECEDENT_CLASSES = new String[]
{"javax.ws.rs", "org.apache.hadoop.hive"};
-
- private static final String[] PARENT_CL_PRECEDENT_CLASSES = new String[] {
- // // Java standard library:
- "com.sun.", "launcher.", "java.", "javax.", "org.ietf", "org.omg",
"org.w3c", "org.xml", "sunw.", "sun.",
- // logging
- "org.apache.commons.logging", "org.apache.log4j", "com.hadoop",
"org.slf4j",
- // Hadoop/ZK:
- "org.apache.hadoop", "org.apache.zookeeper", "io.kyligence",
"org.apache.kylin", "com.intellij",
- "org.apache.calcite", "org.roaringbitmap", "org.apache.parquet"};
- private static final Set<String> classNotFoundCache = new HashSet<>();
- private static Logger logger =
LoggerFactory.getLogger(KylinItSparkClassLoader.class);
-
- /**
- * Creates a DynamicClassLoader that can load classes dynamically
- * from jar files under a specific folder.
- * CubeControllerTest
- *
- * @param parent the parent ClassLoader to set.
- */
- protected KylinItSparkClassLoader(ClassLoader parent) throws IOException {
- super(new URL[] {}, parent);
- init();
- }
-
- public void init() throws MalformedURLException {
- String sparkHome = System.getenv("SPARK_HOME");
- if (sparkHome == null) {
- sparkHome = System.getProperty("SPARK_HOME");
- if (sparkHome == null) {
- throw new RuntimeException(
- "Spark home not found; set it explicitly or use the
SPARK_HOME environment variable.");
- }
- }
- File file = new File(sparkHome + "/jars");
- File[] jars = file.listFiles();
- for (File jar : jars) {
- addURL(jar.toURI().toURL());
- }
- File sparkFile = findFile("../../build/conf",
"spark-executor-log4j.properties");
-
- try {
- // sparder and query module has org.apache.spark class ,if not add,
- // that will be load by system classloader
- // (find class api will be find the parent classloader first,
- // so ,parent classloader can not load it ,spark class will not
found)
- // why SparkClassLoader is unnecessary?
- // DebugTomcatClassLoader and TomcatClassLoader find class api
will be find itself first
- // so, parent classloader can load it , spark class will be found
- addURL(new File("../sparder/target/classes").toURI().toURL());
- addURL(new
File("../storage-parquet/target/classes").toURI().toURL());
- addURL(new File("../query/target/classes").toURI().toURL());
- addURL(new File("../query/target/test-classes").toURI().toURL());
- addURL(new File("../udf/target/classes").toURI().toURL());
- System.setProperty("kylin.query.engine.sparder-additional-files",
sparkFile.getCanonicalPath());
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public Class<?> loadClass(String name, boolean resolve) throws
ClassNotFoundException {
- if (needToUseGlobal(name)) {
- logger.debug("Skipping exempt class " + name + " - delegating
directly to parent");
- try {
- return getParent().loadClass(name);
- } catch (ClassNotFoundException e) {
- return super.findClass(name);
- }
- }
-
- synchronized (getClassLoadingLock(name)) {
- // Check whether the class has already been loaded:
- Class<?> clasz = findLoadedClass(name);
- if (clasz != null) {
- logger.debug("Class " + name + " already loaded");
- } else {
- try {
- // Try to find this class using the URLs passed to this
ClassLoader
- logger.debug("Finding class: " + name);
- clasz = super.findClass(name);
- if (clasz == null) {
- logger.debug("cannot find class" + name);
- }
- } catch (ClassNotFoundException e) {
- classNotFoundCache.add(name);
- // Class not found using this ClassLoader, so delegate to
parent
- logger.debug("Class " + name + " not found - delegating to
parent");
- try {
- clasz = getParent().loadClass(name);
- } catch (ClassNotFoundException e2) {
- // Class not found in this ClassLoader or in the
parent ClassLoader
- // Log some debug output before re-throwing
ClassNotFoundException
- logger.debug("Class " + name + " not found in parent
loader");
- throw e2;
- }
- }
- }
- return clasz;
- }
- }
-
- private boolean isThisCLPrecedent(String name) {
- for (String exemptPrefix : THIS_CL_PRECEDENT_CLASSES) {
- if (name.startsWith(exemptPrefix)) {
- return true;
- }
- }
- return false;
- }
-
- private boolean isParentCLPrecedent(String name) {
- for (String exemptPrefix : PARENT_CL_PRECEDENT_CLASSES) {
- if (name.startsWith(exemptPrefix)) {
- return true;
- }
- }
- return false;
- }
-
- private boolean needToUseGlobal(String name) {
- return !isThisCLPrecedent(name) && !classNeedPreempt(name) &&
isParentCLPrecedent(name);
- }
-
- boolean classNeedPreempt(String name) {
- if (classNotFoundCache.contains(name)) {
- return false;
- }
- for (String exemptPrefix : SPARK_CL_PREEMPT_CLASSES) {
- if (name.startsWith(exemptPrefix)) {
- return true;
- }
- }
- return false;
- }
-
- boolean fileNeedPreempt(String name) {
-
- for (String exemptPrefix : SPARK_CL_PREEMPT_FILES) {
- if (name.contains(exemptPrefix)) {
- return true;
- }
- }
- return false;
- }
-}
diff --git
a/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/SparkClassLoader.java
b/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/SparkClassLoader.java
index b105a26..d74fb26 100644
---
a/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/SparkClassLoader.java
+++
b/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/SparkClassLoader.java
@@ -36,7 +36,7 @@ import java.util.Set;
public class SparkClassLoader extends URLClassLoader {
//preempt these classes from parent
private static String[] SPARK_CL_PREEMPT_CLASSES = new String[]
{"org.apache.spark", "scala.",
- "org.spark_project"};
+ "org.spark_project", "com.esotericsoftware.kryo"};
//preempt these files from parent
private static String[] SPARK_CL_PREEMPT_FILES = new String[]
{"spark-version-info.properties", "HiveClientImpl",
@@ -47,12 +47,12 @@ public class SparkClassLoader extends URLClassLoader {
//when loading class (indirectly used by SPARK_CL_PREEMPT_CLASSES), some
of them should use parent's first
private static String[] PARENT_CL_PRECEDENT_CLASSES = new String[] {
- // // Java standard library:
+ // Java standard library:
"com.sun.", "launcher.", "java.", "javax.", "org.ietf", "org.omg",
"org.w3c", "org.xml", "sunw.", "sun.",
// logging
"org.apache.commons.logging", "org.apache.log4j", "org.slf4j",
"org.apache.hadoop",
// Hadoop/ZK:
- "io.kyligence", "org.apache.kylin", "com.intellij",
"org.apache.calcite"};
+ "org.apache.kylin", "com.intellij", "org.apache.calcite"};
private static final Set<String> classNotFoundCache = new HashSet<>();
private static Logger logger =
LoggerFactory.getLogger(SparkClassLoader.class);
@@ -121,22 +121,6 @@ public class SparkClassLoader extends URLClassLoader {
for (File jar : jars) {
addURL(jar.toURI().toURL());
}
- /*if (System.getenv("KYLIN_HOME") != null) {
- // for prod
- String kylin_home = System.getenv("KYLIN_HOME");
- File sparkJar = findFile(kylin_home + "/lib",
"kylin-udf-.*-SNAPSHOT.jar");
- if (sparkJar != null) {
- logger.info("Add kylin UDF jar to spark classloader : " +
sparkJar.getName());
- addURL(sparkJar.toURI().toURL());
- } else {
- throw new RuntimeException(
- "Can not find kylin UDF jar, please set KYLIN_HOME and
make sure the kylin-udf-*.jar exists in $KYLIN_HOME/lib");
- }
- } else if (Files.exists(Paths.get("../udf/target/classes"))) {
- // for debugtomcat
- logger.info("Add kylin UDF classes to spark classloader");
- addURL(new File("../udf/target/classes").toURI().toURL());
- }*/
}
diff --git
a/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/TomcatClassLoader.java
b/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/TomcatClassLoader.java
index 45007c5..f403dbb 100644
---
a/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/TomcatClassLoader.java
+++
b/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/TomcatClassLoader.java
@@ -22,48 +22,27 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
+import java.net.URL;
import java.util.HashSet;
import java.util.Set;
import org.apache.catalina.loader.ParallelWebappClassLoader;
-import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.kylin.spark.classloader.ClassLoaderUtils.findFile;
-
public class TomcatClassLoader extends ParallelWebappClassLoader {
- private static String[] PARENT_CL_PRECEDENT_CLASSES = new String[] {
+ private static final String[] PARENT_CL_PRECEDENT_CLASSES = new String[] {
// Java standard library:
"com.sun.", "launcher.", "javax.", "org.ietf", "java", "org.omg",
"org.w3c", "org.xml", "sunw.",
// logging
"org.slf4j", "org.apache.commons.logging", "org.apache.log4j",
"org.apache.catalina", "org.apache.tomcat"};
-
- private static String[] THIS_CL_PRECEDENT_CLASSES = new String[]
{"io.kyligence", "org.apache.kylin",
+ private static final String[] THIS_CL_PRECEDENT_CLASSES = new String[]
{"org.apache.kylin",
"org.apache.calcite"};
-
- private static String[] CODEGEN_CLASSES = new String[]
{"org.apache.spark.sql.catalyst.expressions.Object",
+ private static final String[] CODE_GEN_CLASS = new String[]
{"org.apache.spark.sql.catalyst.expressions.Object",
"Baz"};
private static final Set<String> wontFindClasses = new HashSet<>();
static {
- String tomcatClassLoaderParentClPrecedentClasses = System
- .getenv("TOMCATCLASSLOADER_PARENT_CL_PRECEDENT_CLASSES");
- if (!StringUtils.isEmpty(tomcatClassLoaderParentClPrecedentClasses)) {
- PARENT_CL_PRECEDENT_CLASSES =
StringUtils.split(tomcatClassLoaderParentClPrecedentClasses, ",");
- }
-
- String tomcatClassLoaderThisClPrecedentClasses = System
- .getenv("TOMCATCLASSLOADER_THIS_CL_PRECEDENT_CLASSES");
- if (!StringUtils.isEmpty(tomcatClassLoaderThisClPrecedentClasses)) {
- THIS_CL_PRECEDENT_CLASSES =
StringUtils.split(tomcatClassLoaderThisClPrecedentClasses, ",");
- }
-
- String tomcatClassLoaderCodegenClasses =
System.getenv("TOMCATCLASSLOADER_CODEGEN_CLASSES");
- if (!StringUtils.isEmpty(tomcatClassLoaderCodegenClasses)) {
- CODEGEN_CLASSES =
StringUtils.split(tomcatClassLoaderCodegenClasses, ",");
- }
-
wontFindClasses.add("Class");
wontFindClasses.add("Object");
wontFindClasses.add("org");
@@ -78,9 +57,8 @@ public class TomcatClassLoader extends
ParallelWebappClassLoader {
wontFindClasses.add("String");
}
- public static TomcatClassLoader defaultClassLoad = null;
private static Logger logger =
LoggerFactory.getLogger(TomcatClassLoader.class);
- public SparkClassLoader sparkClassLoader;
+ private SparkClassLoader sparkClassLoader;
/**
* Creates a DynamicClassLoader that can load classes dynamically
@@ -93,53 +71,44 @@ public class TomcatClassLoader extends
ParallelWebappClassLoader {
sparkClassLoader = new SparkClassLoader(this);
ClassLoaderUtils.setSparkClassLoader(sparkClassLoader);
ClassLoaderUtils.setOriginClassLoader(this);
- defaultClassLoad = this;
init();
}
public void init() {
- String sparkHome = System.getenv("SPARK_HOME");
- if (sparkHome == null || sparkHome.isEmpty()) {
- throw new RuntimeException("Error found spark home.");
- }
- try {
- // SparkContext use spi to match deploy mode
- // otherwise SparkContext init fail ,can not find yarn deploy mode
- File yarnJar = findFile(sparkHome + "/jars", "spark-yarn.*.jar");
- addURL(yarnJar.toURI().toURL());
- // jersey in spark will attempt find @Path class file in current
classloader.
- // Not possible to delegate to spark loader
- // otherwise spark web ui executors tab can not render
- File coreJar = findFile(sparkHome + "/jars", "spark-core.*.jar");
- addURL(coreJar.toURI().toURL());
- } catch (MalformedURLException e) {
- e.printStackTrace();
+
+ String classPath = System.getProperty("java.class.path");
+ if (classPath == null) {
+ throw new RuntimeException("");
}
+ String[] jars = classPath.split(":");
+ for (String jar : jars) {
+ try {
+ URL url = new File(jar).toURI().toURL();
+ addURL(url);
+ } catch (MalformedURLException e) {
+ e.printStackTrace();
+ }
+ }
}
@Override
public Class<?> loadClass(String name, boolean resolve) throws
ClassNotFoundException {
- // when calcite compile class, some stupid class name will be
proposed, not worth to actually lookup
if (isWontFind(name)) {
throw new ClassNotFoundException();
}
- // spark codegen classload parent is
Thread.currentThread().getContextClassLoader()
- // and calcite baz classloader is EnumerableInterpretable.class's
classloader
if (isCodeGen(name)) {
throw new ClassNotFoundException();
}
- // class loaders should conform to global's
+
if (name.startsWith("org.apache.kylin.spark.classloader")) {
return parent.loadClass(name);
}
- // if spark CL needs preempt
if (sparkClassLoader.classNeedPreempt(name)) {
return sparkClassLoader.loadClass(name);
}
- // tomcat classpath include KYLIN_HOME/lib , ensure this classload can
load kylin class
- if (isParentCLPrecedent(name) && !isThisCLPrecedent(name)) {
- logger.debug("delegate " + name + " directly to parent");
+ if (isParentCLPrecedent(name)) {
+ logger.debug("Skipping exempt class " + name + " - delegating
directly to parent");
return parent.loadClass(name);
}
return super.loadClass(name, resolve);
@@ -163,21 +132,12 @@ public class TomcatClassLoader extends
ParallelWebappClassLoader {
return false;
}
- private boolean isThisCLPrecedent(String name) {
- for (String exemptPrefix : THIS_CL_PRECEDENT_CLASSES) {
- if (name.startsWith(exemptPrefix)) {
- return true;
- }
- }
- return false;
- }
-
private boolean isWontFind(String name) {
return wontFindClasses.contains(name);
}
private boolean isCodeGen(String name) {
- for (String exemptPrefix : CODEGEN_CLASSES) {
+ for (String exemptPrefix : CODE_GEN_CLASS) {
if (name.startsWith(exemptPrefix)) {
return true;
}
