Repository: spark Updated Branches: refs/heads/master 5014d6e25 -> 14a32a647
[SPARK-22330][CORE] Linear containsKey operation for serialized maps â¦alization. ## What changes were proposed in this pull request? Use non-linear containsKey operation for serialized maps, lookup into underlying map. ## How was this patch tested? unit tests Author: Alexander Istomin <[email protected]> Closes #19553 from Whoosh/SPARK-22330. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/14a32a64 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/14a32a64 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/14a32a64 Branch: refs/heads/master Commit: 14a32a647a019ed45793784069fbf077b9c45e60 Parents: 5014d6e Author: Alexander Istomin <[email protected]> Authored: Tue Nov 7 00:47:16 2017 +0100 Committer: Wenchen Fan <[email protected]> Committed: Tue Nov 7 00:47:16 2017 +0100 ---------------------------------------------------------------------- .../org/apache/spark/api/java/JavaUtils.scala | 9 +++- .../apache/spark/api/java/JavaUtilsSuite.scala | 49 ++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/14a32a64/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala b/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala index d650623..fd96052 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala @@ -43,10 +43,17 @@ private[spark] object JavaUtils { override def size: Int = underlying.size + // Delegate to implementation because AbstractMap implementation iterates over whole key set + override def containsKey(key: AnyRef): Boolean = try { + underlying.contains(key.asInstanceOf[A]) + } catch { + case _: ClassCastException => false + } + override def get(key: AnyRef): B = try { underlying.getOrElse(key.asInstanceOf[A], null.asInstanceOf[B]) } catch { - case ex: ClassCastException => null.asInstanceOf[B] + case _: ClassCastException => null.asInstanceOf[B] } override def entrySet: ju.Set[ju.Map.Entry[A, B]] = new ju.AbstractSet[ju.Map.Entry[A, B]] { http://git-wip-us.apache.org/repos/asf/spark/blob/14a32a64/core/src/test/scala/org/apache/spark/api/java/JavaUtilsSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/api/java/JavaUtilsSuite.scala b/core/src/test/scala/org/apache/spark/api/java/JavaUtilsSuite.scala new file mode 100644 index 0000000..8e6e3e0 --- /dev/null +++ b/core/src/test/scala/org/apache/spark/api/java/JavaUtilsSuite.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.api.java + +import java.io.Serializable + +import org.mockito.Mockito._ + +import org.apache.spark.SparkFunSuite + + +class JavaUtilsSuite extends SparkFunSuite { + + test("containsKey implementation without iteratively entrySet call") { + val src = new scala.collection.mutable.HashMap[Double, String] + val key: Double = 42.5 + val key2 = "key" + + src.put(key, "42") + + val map: java.util.Map[Double, String] = spy(JavaUtils.mapAsSerializableJavaMap(src)) + + assert(map.containsKey(key)) + + // ClassCast checking, shouldn't throw exception + assert(!map.containsKey(key2)) + assert(map.get(key2) == null) + + assert(map.get(key).eq("42")) + assert(map.isInstanceOf[Serializable]) + + verify(map, never()).entrySet() + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
