This is an automated email from the ASF dual-hosted git repository.
sarutak pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 83363394b51f [SPARK-57081][TESTS] Fix percent-encoding issue in
SparkTestUtils classpath handling
83363394b51f is described below
commit 83363394b51f7c95805652edf653328efef4724d
Author: Kousuke Saruta <[email protected]>
AuthorDate: Thu May 28 21:20:53 2026 +0900
[SPARK-57081][TESTS] Fix percent-encoding issue in SparkTestUtils classpath
handling
### What changes were proposed in this pull request?
This PR replaces `URL.getFile()` with `new File(url.toURI).getPath` (or
`new File(url.toURI)`) in `SparkTestUtils.scala` to correctly handle classpath
URLs containing percent-encoded characters (e.g., spaces encoded as `%20`).
Four call sites are fixed:
- `createCompiledClass`
- `createJarWithJavaSources`
- `createJarWithScalaSources`
- `expandManifestClasspath`
Additionally, a new test suite `SparkTestUtilsSuite` is added to cover both
this fix and the `expandManifestClasspath` logic introduced in #55564.
### Why are the changes needed?
`URL.getFile()` returns a percent-encoded string. When a classpath URL
contains spaces or special characters (e.g., `C:\Program Files\...` or CI
environments with spaces in paths), the encoded string (e.g.,
`/path%20with%20spaces/lib.jar`) is passed directly to the compiler or `new
File(...)`, which fails to resolve the actual file. Using `url.toURI` properly
decodes the path.
Without this change, tests which use `SparkTestUtils` fails if the
repository is in the path containing white spaces.
```
$ build/sbt 'core/testOnly
org.apache.spark.executor.ClassLoaderIsolationSuite
...
[info] - SPARK-51537 Executor isolation avoids reloading plugin jars ***
FAILED *** (1 second, 674 milliseconds)
[info] java.lang.AssertionError: assertion failed: Compiled file not
found: /private/space containing path/spark/core/TestExecutorPlugin.class
...
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Confirmed `ClassLoaderIsolationSuite` passed even if the repository is in a
space containing path.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude (via Kiro CLI, auto model selection)
Closes #56125 from sarutak/fix-percent-encoding-issue.
Authored-by: Kousuke Saruta <[email protected]>
Signed-off-by: Kousuke Saruta <[email protected]>
(cherry picked from commit 7961cefdc861f23bb35015b2af350cb497ce4e16)
Signed-off-by: Kousuke Saruta <[email protected]>
---
.../org/apache/spark/util/SparkTestUtils.scala | 11 ++++----
.../apache/spark/util/SparkTestUtilsSuite.scala | 30 ++++++++++++++++++++++
2 files changed, 36 insertions(+), 5 deletions(-)
diff --git
a/common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala
b/common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala
index fa5f99a1aae2..0af0e0f6de45 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala
@@ -53,8 +53,8 @@ private[spark] trait SparkTestUtils {
Seq(
"-classpath",
classpathUrls
- .map {
- _.getFile
+ .map { u =>
+ new File(u.toURI).getPath
}
.mkString(File.pathSeparator))
} else {
@@ -123,7 +123,8 @@ private[spark] trait SparkTestUtils {
val options = Seq("-d", classDir.getAbsolutePath) ++ (
if (classpathUrls.nonEmpty) {
- Seq("-classpath",
classpathUrls.map(_.getFile).mkString(File.pathSeparator))
+ Seq("-classpath",
+ classpathUrls.map(u => new
File(u.toURI).getPath).mkString(File.pathSeparator))
} else Seq.empty
)
@@ -177,7 +178,7 @@ private[spark] trait SparkTestUtils {
// on Windows to work around CMD's command-line length limit and by some
build/CI
// tools. Expand any such JARs before invoking scalac so the classpath is
complete.
val expandedClasspath = classpathUrls.flatMap(expandManifestClasspath)
- val cpStr = expandedClasspath.map(_.getFile).mkString(File.pathSeparator)
+ val cpStr = expandedClasspath.map(u => new
File(u.toURI).getPath).mkString(File.pathSeparator)
val args = Array("-classpath", cpStr, "-d", classDir.getAbsolutePath) ++
sourceFiles.map(_.getAbsolutePath)
@@ -216,7 +217,7 @@ private[spark] trait SparkTestUtils {
* original URL unchanged.
*/
private[spark] def expandManifestClasspath(url: URL): Seq[URL] = {
- val file = new File(url.getFile)
+ val file = new File(url.toURI)
if (!file.exists() || !file.getName.endsWith(".jar")) return Seq(url)
try {
val jarFile = new JarFile(file)
diff --git
a/common/utils/src/test/scala/org/apache/spark/util/SparkTestUtilsSuite.scala
b/common/utils/src/test/scala/org/apache/spark/util/SparkTestUtilsSuite.scala
new file mode 100644
index 000000000000..10a599739f6c
--- /dev/null
+++
b/common/utils/src/test/scala/org/apache/spark/util/SparkTestUtilsSuite.scala
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
+
+class SparkTestUtilsSuite extends AnyFunSuite with SparkTestUtils { //
scalastyle:ignore funsuite
+
+ test("SPARK-57081: createCompiledClass with spaces in classpath") {
+ val dir = SparkFileUtils.createTempDir(namePrefix = "path with spaces")
+ val sourceFile = new JavaSourceFromString("Hello", "public class Hello {}")
+ val result = createCompiledClass("Hello", dir, sourceFile,
Seq(dir.toURI.toURL))
+ assert(result.exists(), s"Compiled class file should exist at
${result.getPath}")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
