[SPARK-10966] [SQL] Codegen framework cleanup This PR is mostly cosmetic and cleans up some warts in codegen (nearly all of which were inherited from the original quasiquote version). - Add lines numbers to errors (in stacktraces when debug logging is on, and always for compile fails) - Use a variable for input row instead of hardcoding "i" everywhere - rename `primitive` -> `value` (since its often actually an object)
Author: Michael Armbrust <[email protected]> Closes #9006 from marmbrus/codegen-cleanup. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f5d154bc Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f5d154bc Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f5d154bc Branch: refs/heads/master Commit: f5d154bc731aedfc2eecdb4ed6af8cac820511c9 Parents: 9672602 Author: Michael Armbrust <[email protected]> Authored: Wed Oct 7 10:17:29 2015 -0700 Committer: Michael Armbrust <[email protected]> Committed: Wed Oct 7 10:17:29 2015 -0700 ---------------------------------------------------------------------- .../catalyst/expressions/BoundAttribute.scala | 6 +- .../spark/sql/catalyst/expressions/Cast.scala | 2 +- .../sql/catalyst/expressions/Expression.scala | 18 +++--- .../catalyst/expressions/InputFileName.scala | 2 +- .../expressions/MonotonicallyIncreasingID.scala | 2 +- .../sql/catalyst/expressions/SortOrder.scala | 6 +- .../catalyst/expressions/SparkPartitionID.scala | 2 +- .../sql/catalyst/expressions/arithmetic.scala | 62 +++++++++--------- .../expressions/codegen/CodeGenerator.scala | 47 +++++++++----- .../expressions/codegen/CodegenFallback.scala | 6 +- .../codegen/GenerateMutableProjection.scala | 8 +-- .../expressions/codegen/GenerateOrdering.scala | 10 +-- .../expressions/codegen/GeneratePredicate.scala | 4 +- .../codegen/GenerateProjection.scala | 4 +- .../codegen/GenerateSafeProjection.scala | 14 ++-- .../codegen/GenerateUnsafeProjection.scala | 20 +++--- .../expressions/collectionOperations.scala | 4 +- .../expressions/complexTypeCreator.scala | 16 ++--- .../expressions/complexTypeExtractors.scala | 8 +-- .../expressions/conditionalExpressions.scala | 36 +++++------ .../expressions/datetimeExpressions.scala | 62 +++++++++--------- .../expressions/decimalExpressions.scala | 6 +- .../sql/catalyst/expressions/literals.scala | 14 ++-- .../catalyst/expressions/mathExpressions.scala | 68 ++++++++++---------- .../spark/sql/catalyst/expressions/misc.scala | 10 +-- .../catalyst/expressions/nullExpressions.scala | 24 +++---- .../sql/catalyst/expressions/predicates.scala | 34 +++++----- .../expressions/randomExpressions.scala | 4 +- .../expressions/regexpExpressions.scala | 24 +++---- .../spark/sql/catalyst/expressions/sets.scala | 10 +-- .../expressions/stringExpressions.scala | 64 +++++++++--------- 31 files changed, 306 insertions(+), 291 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala index 473b9b7..ff1f28d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala @@ -68,10 +68,10 @@ case class BoundReference(ordinal: Int, dataType: DataType, nullable: Boolean) override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { val javaType = ctx.javaType(dataType) - val value = ctx.getValue("i", dataType, ordinal.toString) + val value = ctx.getValue(ctx.INPUT_ROW, dataType, ordinal.toString) s""" - boolean ${ev.isNull} = i.isNullAt($ordinal); - $javaType ${ev.primitive} = ${ev.isNull} ? ${ctx.defaultValue(dataType)} : ($value); + boolean ${ev.isNull} = ${ctx.INPUT_ROW}.isNullAt($ordinal); + $javaType ${ev.value} = ${ev.isNull} ? ${ctx.defaultValue(dataType)} : ($value); """ } } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index f0bce38..99d7444 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -438,7 +438,7 @@ case class Cast(child: Expression, dataType: DataType) val eval = child.gen(ctx) val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx) eval.code + - castCode(ctx, eval.primitive, eval.isNull, ev.primitive, ev.isNull, dataType, nullSafeCast) + castCode(ctx, eval.value, eval.isNull, ev.value, ev.isNull, dataType, nullSafeCast) } // three function arguments are: child.primitive, result.primitive and result.isNull http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala index 0b98f55..96284b9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala @@ -276,7 +276,7 @@ abstract class UnaryExpression extends Expression { ev: GeneratedExpressionCode, f: String => String): String = { nullSafeCodeGen(ctx, ev, eval => { - s"${ev.primitive} = ${f(eval)};" + s"${ev.value} = ${f(eval)};" }) } @@ -292,10 +292,10 @@ abstract class UnaryExpression extends Expression { ev: GeneratedExpressionCode, f: String => String): String = { val eval = child.gen(ctx) - val resultCode = f(eval.primitive) + val resultCode = f(eval.value) eval.code + s""" boolean ${ev.isNull} = ${eval.isNull}; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; if (!${ev.isNull}) { $resultCode } @@ -357,7 +357,7 @@ abstract class BinaryExpression extends Expression { ev: GeneratedExpressionCode, f: (String, String) => String): String = { nullSafeCodeGen(ctx, ev, (eval1, eval2) => { - s"${ev.primitive} = ${f(eval1, eval2)};" + s"${ev.value} = ${f(eval1, eval2)};" }) } @@ -375,11 +375,11 @@ abstract class BinaryExpression extends Expression { f: (String, String) => String): String = { val eval1 = left.gen(ctx) val eval2 = right.gen(ctx) - val resultCode = f(eval1.primitive, eval2.primitive) + val resultCode = f(eval1.value, eval2.value) s""" ${eval1.code} boolean ${ev.isNull} = ${eval1.isNull}; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; if (!${ev.isNull}) { ${eval2.code} if (!${eval2.isNull}) { @@ -482,7 +482,7 @@ abstract class TernaryExpression extends Expression { ev: GeneratedExpressionCode, f: (String, String, String) => String): String = { nullSafeCodeGen(ctx, ev, (eval1, eval2, eval3) => { - s"${ev.primitive} = ${f(eval1, eval2, eval3)};" + s"${ev.value} = ${f(eval1, eval2, eval3)};" }) } @@ -499,11 +499,11 @@ abstract class TernaryExpression extends Expression { ev: GeneratedExpressionCode, f: (String, String, String) => String): String = { val evals = children.map(_.gen(ctx)) - val resultCode = f(evals(0).primitive, evals(1).primitive, evals(2).primitive) + val resultCode = f(evals(0).value, evals(1).value, evals(2).value) s""" ${evals(0).code} boolean ${ev.isNull} = true; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; if (!${evals(0).isNull}) { ${evals(1).code} if (!${evals(1).isNull}) { http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala index 1e74f71..d809877 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala @@ -42,7 +42,7 @@ case class InputFileName() extends LeafExpression with Nondeterministic { override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { ev.isNull = "false" - s"final ${ctx.javaType(dataType)} ${ev.primitive} = " + + s"final ${ctx.javaType(dataType)} ${ev.value} = " + "org.apache.spark.rdd.SqlNewHadoopRDD.getInputFileName();" } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala index 291b7a5..2d7679f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala @@ -66,7 +66,7 @@ private[sql] case class MonotonicallyIncreasingID() extends LeafExpression with ev.isNull = "false" s""" - final ${ctx.javaType(dataType)} ${ev.primitive} = $partitionMaskTerm + $countTerm; + final ${ctx.javaType(dataType)} ${ev.value} = $partitionMaskTerm + $countTerm; $countTerm++; """ } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala index 98e0290..290c128 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala @@ -63,7 +63,7 @@ case class SortPrefix(child: SortOrder) extends UnaryExpression { override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { val childCode = child.child.gen(ctx) - val input = childCode.primitive + val input = childCode.value val BinaryPrefixCmp = classOf[BinaryPrefixComparator].getName val DoublePrefixCmp = classOf[DoublePrefixComparator].getName @@ -97,10 +97,10 @@ case class SortPrefix(child: SortOrder) extends UnaryExpression { childCode.code + s""" - |long ${ev.primitive} = ${nullValue}L; + |long ${ev.value} = ${nullValue}L; |boolean ${ev.isNull} = false; |if (!${childCode.isNull}) { - | ${ev.primitive} = $prefixCode; + | ${ev.value} = $prefixCode; |} """.stripMargin } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala index 4b1772a..8bff173 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala @@ -47,6 +47,6 @@ private[sql] case class SparkPartitionID() extends LeafExpression with Nondeterm ctx.addMutableState(ctx.JAVA_INT, idTerm, s"$idTerm = org.apache.spark.TaskContext.getPartitionId();") ev.isNull = "false" - s"final ${ctx.javaType(dataType)} ${ev.primitive} = $idTerm;" + s"final ${ctx.javaType(dataType)} ${ev.value} = $idTerm;" } } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index 98464ed..61a17fd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -42,7 +42,7 @@ case class UnaryMinus(child: Expression) extends UnaryExpression with ExpectsInp // for example, we could not write --9223372036854775808L in code s""" ${ctx.javaType(dt)} $originValue = (${ctx.javaType(dt)})($eval); - ${ev.primitive} = (${ctx.javaType(dt)})(-($originValue)); + ${ev.value} = (${ctx.javaType(dt)})(-($originValue)); """}) case dt: CalendarIntervalType => defineCodeGen(ctx, ev, c => s"$c.negate()") } @@ -223,20 +223,20 @@ case class Divide(left: Expression, right: Expression) extends BinaryArithmetic val eval1 = left.gen(ctx) val eval2 = right.gen(ctx) val isZero = if (dataType.isInstanceOf[DecimalType]) { - s"${eval2.primitive}.isZero()" + s"${eval2.value}.isZero()" } else { - s"${eval2.primitive} == 0" + s"${eval2.value} == 0" } val javaType = ctx.javaType(dataType) val divide = if (dataType.isInstanceOf[DecimalType]) { - s"${eval1.primitive}.$decimalMethod(${eval2.primitive})" + s"${eval1.value}.$decimalMethod(${eval2.value})" } else { - s"($javaType)(${eval1.primitive} $symbol ${eval2.primitive})" + s"($javaType)(${eval1.value} $symbol ${eval2.value})" } s""" ${eval2.code} boolean ${ev.isNull} = false; - $javaType ${ev.primitive} = ${ctx.defaultValue(javaType)}; + $javaType ${ev.value} = ${ctx.defaultValue(javaType)}; if (${eval2.isNull} || $isZero) { ${ev.isNull} = true; } else { @@ -244,7 +244,7 @@ case class Divide(left: Expression, right: Expression) extends BinaryArithmetic if (${eval1.isNull}) { ${ev.isNull} = true; } else { - ${ev.primitive} = $divide; + ${ev.value} = $divide; } } """ @@ -285,20 +285,20 @@ case class Remainder(left: Expression, right: Expression) extends BinaryArithmet val eval1 = left.gen(ctx) val eval2 = right.gen(ctx) val isZero = if (dataType.isInstanceOf[DecimalType]) { - s"${eval2.primitive}.isZero()" + s"${eval2.value}.isZero()" } else { - s"${eval2.primitive} == 0" + s"${eval2.value} == 0" } val javaType = ctx.javaType(dataType) val remainder = if (dataType.isInstanceOf[DecimalType]) { - s"${eval1.primitive}.$decimalMethod(${eval2.primitive})" + s"${eval1.value}.$decimalMethod(${eval2.value})" } else { - s"($javaType)(${eval1.primitive} $symbol ${eval2.primitive})" + s"($javaType)(${eval1.value} $symbol ${eval2.value})" } s""" ${eval2.code} boolean ${ev.isNull} = false; - $javaType ${ev.primitive} = ${ctx.defaultValue(javaType)}; + $javaType ${ev.value} = ${ctx.defaultValue(javaType)}; if (${eval2.isNull} || $isZero) { ${ev.isNull} = true; } else { @@ -306,7 +306,7 @@ case class Remainder(left: Expression, right: Expression) extends BinaryArithmet if (${eval1.isNull}) { ${ev.isNull} = true; } else { - ${ev.primitive} = $remainder; + ${ev.value} = $remainder; } } """ @@ -341,24 +341,24 @@ case class MaxOf(left: Expression, right: Expression) extends BinaryArithmetic { override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { val eval1 = left.gen(ctx) val eval2 = right.gen(ctx) - val compCode = ctx.genComp(dataType, eval1.primitive, eval2.primitive) + val compCode = ctx.genComp(dataType, eval1.value, eval2.value) eval1.code + eval2.code + s""" boolean ${ev.isNull} = false; - ${ctx.javaType(left.dataType)} ${ev.primitive} = + ${ctx.javaType(left.dataType)} ${ev.value} = ${ctx.defaultValue(left.dataType)}; if (${eval1.isNull}) { ${ev.isNull} = ${eval2.isNull}; - ${ev.primitive} = ${eval2.primitive}; + ${ev.value} = ${eval2.value}; } else if (${eval2.isNull}) { ${ev.isNull} = ${eval1.isNull}; - ${ev.primitive} = ${eval1.primitive}; + ${ev.value} = ${eval1.value}; } else { if ($compCode > 0) { - ${ev.primitive} = ${eval1.primitive}; + ${ev.value} = ${eval1.value}; } else { - ${ev.primitive} = ${eval2.primitive}; + ${ev.value} = ${eval2.value}; } } """ @@ -395,24 +395,24 @@ case class MinOf(left: Expression, right: Expression) extends BinaryArithmetic { override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { val eval1 = left.gen(ctx) val eval2 = right.gen(ctx) - val compCode = ctx.genComp(dataType, eval1.primitive, eval2.primitive) + val compCode = ctx.genComp(dataType, eval1.value, eval2.value) eval1.code + eval2.code + s""" boolean ${ev.isNull} = false; - ${ctx.javaType(left.dataType)} ${ev.primitive} = + ${ctx.javaType(left.dataType)} ${ev.value} = ${ctx.defaultValue(left.dataType)}; if (${eval1.isNull}) { ${ev.isNull} = ${eval2.isNull}; - ${ev.primitive} = ${eval2.primitive}; + ${ev.value} = ${eval2.value}; } else if (${eval2.isNull}) { ${ev.isNull} = ${eval1.isNull}; - ${ev.primitive} = ${eval1.primitive}; + ${ev.value} = ${eval1.value}; } else { if ($compCode < 0) { - ${ev.primitive} = ${eval1.primitive}; + ${ev.value} = ${eval1.value}; } else { - ${ev.primitive} = ${eval2.primitive}; + ${ev.value} = ${eval2.value}; } } """ @@ -451,9 +451,9 @@ case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic { s""" ${ctx.javaType(dataType)} r = $eval1.remainder($eval2); if (r.compare(new org.apache.spark.sql.types.Decimal().set(0)) < 0) { - ${ev.primitive} = (r.$decimalAdd($eval2)).remainder($eval2); + ${ev.value} = (r.$decimalAdd($eval2)).remainder($eval2); } else { - ${ev.primitive} = r; + ${ev.value} = r; } """ // byte and short are casted into int when add, minus, times or divide @@ -461,18 +461,18 @@ case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic { s""" ${ctx.javaType(dataType)} r = (${ctx.javaType(dataType)})($eval1 % $eval2); if (r < 0) { - ${ev.primitive} = (${ctx.javaType(dataType)})((r + $eval2) % $eval2); + ${ev.value} = (${ctx.javaType(dataType)})((r + $eval2) % $eval2); } else { - ${ev.primitive} = r; + ${ev.value} = r; } """ case _ => s""" ${ctx.javaType(dataType)} r = $eval1 % $eval2; if (r < 0) { - ${ev.primitive} = (r + $eval2) % $eval2; + ${ev.value} = (r + $eval2) % $eval2; } else { - ${ev.primitive} = r; + ${ev.value} = r; } """ } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala index 9a28781..2dd6804 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala @@ -42,10 +42,10 @@ class LongHashSet extends org.apache.spark.util.collection.OpenHashSet[Long] * @param code The sequence of statements required to evaluate the expression. * @param isNull A term that holds a boolean value representing whether the expression evaluated * to null. - * @param primitive A term for a possible primitive value of the result of the evaluation. Not - * valid if `isNull` is set to `true`. + * @param value A term for a (possibly primitive) value of the result of the evaluation. Not + * valid if `isNull` is set to `true`. */ -case class GeneratedExpressionCode(var code: String, var isNull: String, var primitive: String) +case class GeneratedExpressionCode(var code: String, var isNull: String, var value: String) /** * A context for codegen, which is used to bookkeeping the expressions those are not supported @@ -99,6 +99,9 @@ class CodeGenContext { final val JAVA_FLOAT = "float" final val JAVA_DOUBLE = "double" + /** The variable name of the input row in generated code. */ + final val INPUT_ROW = "i" + private val curId = new java.util.concurrent.atomic.AtomicInteger() /** @@ -112,21 +115,21 @@ class CodeGenContext { } /** - * Returns the code to access a value in `SpecializedGetters` for a given DataType. + * Returns the specialized code to access a value from `inputRow` at `ordinal`. */ - def getValue(getter: String, dataType: DataType, ordinal: String): String = { + def getValue(input: String, dataType: DataType, ordinal: String): String = { val jt = javaType(dataType) dataType match { - case _ if isPrimitiveType(jt) => s"$getter.get${primitiveTypeName(jt)}($ordinal)" - case t: DecimalType => s"$getter.getDecimal($ordinal, ${t.precision}, ${t.scale})" - case StringType => s"$getter.getUTF8String($ordinal)" - case BinaryType => s"$getter.getBinary($ordinal)" - case CalendarIntervalType => s"$getter.getInterval($ordinal)" - case t: StructType => s"$getter.getStruct($ordinal, ${t.size})" - case _: ArrayType => s"$getter.getArray($ordinal)" - case _: MapType => s"$getter.getMap($ordinal)" + case _ if isPrimitiveType(jt) => s"$input.get${primitiveTypeName(jt)}($ordinal)" + case t: DecimalType => s"$input.getDecimal($ordinal, ${t.precision}, ${t.scale})" + case StringType => s"$input.getUTF8String($ordinal)" + case BinaryType => s"$input.getBinary($ordinal)" + case CalendarIntervalType => s"$input.getInterval($ordinal)" + case t: StructType => s"$input.getStruct($ordinal, ${t.size})" + case _: ArrayType => s"$input.getArray($ordinal)" + case _: MapType => s"$input.getMap($ordinal)" case NullType => "null" - case _ => s"($jt)$getter.get($ordinal, null)" + case _ => s"($jt)$input.get($ordinal, null)" } } @@ -384,11 +387,23 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin classOf[UnsafeMapData].getName )) evaluator.setExtendedClass(classOf[GeneratedClass]) + + def formatted = CodeFormatter.format(code) + def withLineNums = formatted.split("\n").zipWithIndex.map { + case (l, n) => f"${n + 1}%03d $l" + }.mkString("\n") + + logDebug({ + // Only add extra debugging info to byte code when we are going to print the source code. + evaluator.setDebuggingInformation(false, true, false) + withLineNums + }) + try { - evaluator.cook(code) + evaluator.cook("generated.java", code) } catch { case e: Exception => - val msg = s"failed to compile: $e\n" + CodeFormatter.format(code) + val msg = s"failed to compile: $e\n$withLineNums" logError(msg, e) throw new Exception(msg, e) } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodegenFallback.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodegenFallback.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodegenFallback.scala index 3492d2c..d51a8de 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodegenFallback.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodegenFallback.scala @@ -34,11 +34,11 @@ trait CodegenFallback extends Expression { val objectTerm = ctx.freshName("obj") s""" /* expression: ${this} */ - Object $objectTerm = expressions[${ctx.references.size - 1}].eval(i); + Object $objectTerm = expressions[${ctx.references.size - 1}].eval(${ctx.INPUT_ROW}); boolean ${ev.isNull} = $objectTerm == null; - ${ctx.javaType(this.dataType)} ${ev.primitive} = ${ctx.defaultValue(this.dataType)}; + ${ctx.javaType(this.dataType)} ${ev.value} = ${ctx.defaultValue(this.dataType)}; if (!${ev.isNull}) { - ${ev.primitive} = (${ctx.boxedType(this.dataType)}) $objectTerm; + ${ev.value} = (${ctx.boxedType(this.dataType)}) $objectTerm; } """ } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala index 793023b..d82d191 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala @@ -49,7 +49,7 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], () => Mu if (${evaluationCode.isNull}) { ${ctx.setColumn("mutableRow", e.dataType, i, null)}; } else { - ${ctx.setColumn("mutableRow", e.dataType, i, evaluationCode.primitive)}; + ${ctx.setColumn("mutableRow", e.dataType, i, evaluationCode.value)}; } """ } else { @@ -58,12 +58,12 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], () => Mu if (${evaluationCode.isNull}) { mutableRow.setNullAt($i); } else { - ${ctx.setColumn("mutableRow", e.dataType, i, evaluationCode.primitive)}; + ${ctx.setColumn("mutableRow", e.dataType, i, evaluationCode.value)}; } """ } } - val allProjections = ctx.splitExpressions("i", projectionCodes) + val allProjections = ctx.splitExpressions(ctx.INPUT_ROW, projectionCodes) val code = s""" public Object generate($exprType[] expr) { @@ -94,7 +94,7 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], () => Mu } public Object apply(Object _i) { - InternalRow i = (InternalRow) _i; + InternalRow ${ctx.INPUT_ROW} = (InternalRow) _i; $allProjections return mutableRow; } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala index 42be394..c2b4202 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala @@ -74,21 +74,21 @@ object GenerateOrdering extends CodeGenerator[Seq[SortOrder], Ordering[InternalR val isNullB = ctx.freshName("isNullB") val primitiveB = ctx.freshName("primitiveB") s""" - i = a; + ${ctx.INPUT_ROW} = a; boolean $isNullA; ${ctx.javaType(order.child.dataType)} $primitiveA; { ${eval.code} $isNullA = ${eval.isNull}; - $primitiveA = ${eval.primitive}; + $primitiveA = ${eval.value}; } - i = b; + ${ctx.INPUT_ROW} = b; boolean $isNullB; ${ctx.javaType(order.child.dataType)} $primitiveB; { ${eval.code} $isNullB = ${eval.isNull}; - $primitiveB = ${eval.primitive}; + $primitiveB = ${eval.value}; } if ($isNullA && $isNullB) { // Nothing @@ -128,7 +128,7 @@ object GenerateOrdering extends CodeGenerator[Seq[SortOrder], Ordering[InternalR @Override public int compare(InternalRow a, InternalRow b) { - InternalRow i = null; // Holds current row being evaluated. + InternalRow ${ctx.INPUT_ROW} = null; // Holds current row being evaluated. $comparisons return 0; } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala index c7e718a..ae6ffe6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala @@ -56,9 +56,9 @@ object GeneratePredicate extends CodeGenerator[Expression, (InternalRow) => Bool } @Override - public boolean eval(InternalRow i) { + public boolean eval(InternalRow ${ctx.INPUT_ROW}) { ${eval.code} - return !${eval.isNull} && ${eval.primitive}; + return !${eval.isNull} && ${eval.value}; } }""" http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateProjection.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateProjection.scala index 75524b5..dbcc9dc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateProjection.scala @@ -59,7 +59,7 @@ object GenerateProjection extends CodeGenerator[Seq[Expression], Projection] { ${eval.code} nullBits[$i] = ${eval.isNull}; if (!${eval.isNull}) { - c$i = ${eval.primitive}; + c$i = ${eval.value}; } } """ @@ -180,7 +180,7 @@ object GenerateProjection extends CodeGenerator[Seq[Expression], Projection] { $columns - public SpecificRow(InternalRow i) { + public SpecificRow(InternalRow ${ctx.INPUT_ROW}) { $initColumns } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala index 7ad352d..ea09e02 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala @@ -51,7 +51,7 @@ object GenerateSafeProjection extends CodeGenerator[Seq[Expression], Projection] s""" if (!$tmp.isNullAt($i)) { ${converter.code} - $values[$i] = ${converter.primitive}; + $values[$i] = ${converter.value}; } """ } @@ -85,7 +85,7 @@ object GenerateSafeProjection extends CodeGenerator[Seq[Expression], Projection] for (int $index = 0; $index < $numElements; $index++) { if (!$tmp.isNullAt($index)) { ${elementConverter.code} - $values[$index] = ${elementConverter.primitive}; + $values[$index] = ${elementConverter.value}; } } final ArrayData $output = new $arrayClass($values); @@ -109,7 +109,7 @@ object GenerateSafeProjection extends CodeGenerator[Seq[Expression], Projection] final MapData $tmp = $input; ${keyConverter.code} ${valueConverter.code} - final MapData $output = new $mapClass(${keyConverter.primitive}, ${valueConverter.primitive}); + final MapData $output = new $mapClass(${keyConverter.value}, ${valueConverter.value}); """ GeneratedExpressionCode(code, "false", output) @@ -133,18 +133,18 @@ object GenerateSafeProjection extends CodeGenerator[Seq[Expression], Projection] case (NoOp, _) => "" case (e, i) => val evaluationCode = e.gen(ctx) - val converter = convertToSafe(ctx, evaluationCode.primitive, e.dataType) + val converter = convertToSafe(ctx, evaluationCode.value, e.dataType) evaluationCode.code + s""" if (${evaluationCode.isNull}) { mutableRow.setNullAt($i); } else { ${converter.code} - ${ctx.setColumn("mutableRow", e.dataType, i, converter.primitive)}; + ${ctx.setColumn("mutableRow", e.dataType, i, converter.value)}; } """ } - val allExpressions = ctx.splitExpressions("i", expressionCodes) + val allExpressions = ctx.splitExpressions(ctx.INPUT_ROW, expressionCodes) val code = s""" public Object generate($exprType[] expr) { return new SpecificSafeProjection(expr); @@ -164,7 +164,7 @@ object GenerateSafeProjection extends CodeGenerator[Seq[Expression], Projection] } public Object apply(Object _i) { - InternalRow i = (InternalRow) _i; + InternalRow ${ctx.INPUT_ROW} = (InternalRow) _i; $allExpressions return mutableRow; } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala index 8e58cb9..3e0e817 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala @@ -93,7 +93,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro // Remember the current cursor so that we can calculate how many bytes are // written later. final int $tmpCursor = $bufferHolder.cursor; - ${writeStructToBuffer(ctx, input.primitive, t.map(_.dataType), bufferHolder)} + ${writeStructToBuffer(ctx, input.value, t.map(_.dataType), bufferHolder)} $rowWriter.setOffsetAndSize($index, $tmpCursor, $bufferHolder.cursor - $tmpCursor); """ @@ -102,7 +102,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro // Remember the current cursor so that we can calculate how many bytes are // written later. final int $tmpCursor = $bufferHolder.cursor; - ${writeArrayToBuffer(ctx, input.primitive, et, bufferHolder)} + ${writeArrayToBuffer(ctx, input.value, et, bufferHolder)} $rowWriter.setOffsetAndSize($index, $tmpCursor, $bufferHolder.cursor - $tmpCursor); $rowWriter.alignToWords($bufferHolder.cursor - $tmpCursor); """ @@ -112,7 +112,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro // Remember the current cursor so that we can calculate how many bytes are // written later. final int $tmpCursor = $bufferHolder.cursor; - ${writeMapToBuffer(ctx, input.primitive, kt, vt, bufferHolder)} + ${writeMapToBuffer(ctx, input.value, kt, vt, bufferHolder)} $rowWriter.setOffsetAndSize($index, $tmpCursor, $bufferHolder.cursor - $tmpCursor); $rowWriter.alignToWords($bufferHolder.cursor - $tmpCursor); """ @@ -122,19 +122,19 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro s""" final long $fieldOffset = $rowWriter.getFieldOffset($index); Platform.putLong($bufferHolder.buffer, $fieldOffset, 0L); - ${writePrimitiveType(ctx, input.primitive, dt, s"$bufferHolder.buffer", fieldOffset)} + ${writePrimitiveType(ctx, input.value, dt, s"$bufferHolder.buffer", fieldOffset)} """ case t: DecimalType if t.precision <= Decimal.MAX_LONG_DIGITS => - s"$rowWriter.writeCompactDecimal($index, ${input.primitive}, " + + s"$rowWriter.writeCompactDecimal($index, ${input.value}, " + s"${t.precision}, ${t.scale});" case t: DecimalType => - s"$rowWriter.write($index, ${input.primitive}, ${t.precision}, ${t.scale});" + s"$rowWriter.write($index, ${input.value}, ${t.precision}, ${t.scale});" case NullType => "" - case _ => s"$rowWriter.write($index, ${input.primitive});" + case _ => s"$rowWriter.write($index, ${input.value});" } s""" @@ -324,7 +324,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro val code = s""" $bufferHolder.reset(); - ${writeExpressionsToBuffer(ctx, "i", exprEvals, exprTypes, bufferHolder)} + ${writeExpressionsToBuffer(ctx, ctx.INPUT_ROW, exprEvals, exprTypes, bufferHolder)} $result.pointTo($bufferHolder.buffer, ${expressions.length}, $bufferHolder.totalSize()); """ GeneratedExpressionCode(code, "false", result) @@ -363,9 +363,9 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro return apply((InternalRow) row); } - public UnsafeRow apply(InternalRow i) { + public UnsafeRow apply(InternalRow ${ctx.INPUT_ROW}) { ${eval.code} - return ${eval.primitive}; + return ${eval.value}; } } """ http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index 7b8c5b7..75c66bc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -35,7 +35,7 @@ case class Size(child: Expression) extends UnaryExpression with ExpectsInputType } override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { - nullSafeCodeGen(ctx, ev, c => s"${ev.primitive} = ($c).numElements();") + nullSafeCodeGen(ctx, ev, c => s"${ev.value} = ($c).numElements();") } } @@ -173,7 +173,7 @@ case class ArrayContains(left: Expression, right: Expression) ${ev.isNull} = true; } else if (${ctx.genEqual(right.dataType, value, getValue)}) { ${ev.isNull} = false; - ${ev.primitive} = true; + ${ev.value} = true; break; } } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala index 82eab5f..a5f02e2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala @@ -59,11 +59,11 @@ case class CreateArray(children: Seq[Expression]) extends Expression { if (${eval.isNull}) { $values[$i] = null; } else { - $values[$i] = ${eval.primitive}; + $values[$i] = ${eval.value}; } """ }.mkString("\n") + - s"final ArrayData ${ev.primitive} = new $arrayClass($values);" + s"final ArrayData ${ev.value} = new $arrayClass($values);" } override def prettyName: String = "array" @@ -107,11 +107,11 @@ case class CreateStruct(children: Seq[Expression]) extends Expression { if (${eval.isNull}) { $values[$i] = null; } else { - $values[$i] = ${eval.primitive}; + $values[$i] = ${eval.value}; } """ }.mkString("\n") + - s"final InternalRow ${ev.primitive} = new $rowClass($values);" + s"final InternalRow ${ev.value} = new $rowClass($values);" } override def prettyName: String = "struct" @@ -176,11 +176,11 @@ case class CreateNamedStruct(children: Seq[Expression]) extends Expression { if (${eval.isNull}) { $values[$i] = null; } else { - $values[$i] = ${eval.primitive}; + $values[$i] = ${eval.value}; } """ }.mkString("\n") + - s"final InternalRow ${ev.primitive} = new $rowClass($values);" + s"final InternalRow ${ev.value} = new $rowClass($values);" } override def prettyName: String = "named_struct" @@ -218,7 +218,7 @@ case class CreateStructUnsafe(children: Seq[Expression]) extends Expression { override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { val eval = GenerateUnsafeProjection.createCode(ctx, children) ev.isNull = eval.isNull - ev.primitive = eval.primitive + ev.value = eval.value eval.code } @@ -258,7 +258,7 @@ case class CreateNamedStructUnsafe(children: Seq[Expression]) extends Expression override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { val eval = GenerateUnsafeProjection.createCode(ctx, valExprs) ev.isNull = eval.isNull - ev.primitive = eval.primitive + ev.value = eval.value eval.code } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala index 9927da2..a2b5a6a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala @@ -113,7 +113,7 @@ case class GetStructField(child: Expression, field: StructField, ordinal: Int) if ($eval.isNullAt($ordinal)) { ${ev.isNull} = true; } else { - ${ev.primitive} = ${ctx.getValue(eval, dataType, ordinal.toString)}; + ${ev.value} = ${ctx.getValue(eval, dataType, ordinal.toString)}; } """ }) @@ -175,7 +175,7 @@ case class GetArrayStructFields( } } } - ${ev.primitive} = new $arrayClass(values); + ${ev.value} = new $arrayClass(values); """ }) } @@ -219,7 +219,7 @@ case class GetArrayItem(child: Expression, ordinal: Expression) if (index >= $eval1.numElements() || index < 0) { ${ev.isNull} = true; } else { - ${ev.primitive} = ${ctx.getValue(eval1, dataType, "index")}; + ${ev.value} = ${ctx.getValue(eval1, dataType, "index")}; } """ }) @@ -295,7 +295,7 @@ case class GetMapValue(child: Expression, key: Expression) } if ($found) { - ${ev.primitive} = ${ctx.getValue(eval1 + ".valueArray()", dataType, index)}; + ${ev.value} = ${ctx.getValue(eval1 + ".valueArray()", dataType, index)}; } else { ${ev.isNull} = true; } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala index d51f3d3..d532629 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala @@ -60,15 +60,15 @@ case class If(predicate: Expression, trueValue: Expression, falseValue: Expressi s""" ${condEval.code} boolean ${ev.isNull} = false; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; - if (!${condEval.isNull} && ${condEval.primitive}) { + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; + if (!${condEval.isNull} && ${condEval.value}) { ${trueEval.code} ${ev.isNull} = ${trueEval.isNull}; - ${ev.primitive} = ${trueEval.primitive}; + ${ev.value} = ${trueEval.value}; } else { ${falseEval.code} ${ev.isNull} = ${falseEval.isNull}; - ${ev.primitive} = ${falseEval.primitive}; + ${ev.value} = ${falseEval.value}; } """ } @@ -166,11 +166,11 @@ case class CaseWhen(branches: Seq[Expression]) extends CaseWhenLike { s""" if (!$got) { ${cond.code} - if (!${cond.isNull} && ${cond.primitive}) { + if (!${cond.isNull} && ${cond.value}) { $got = true; ${res.code} ${ev.isNull} = ${res.isNull}; - ${ev.primitive} = ${res.primitive}; + ${ev.value} = ${res.value}; } } """ @@ -182,7 +182,7 @@ case class CaseWhen(branches: Seq[Expression]) extends CaseWhenLike { if (!$got) { ${res.code} ${ev.isNull} = ${res.isNull}; - ${ev.primitive} = ${res.primitive}; + ${ev.value} = ${res.value}; } """ } else { @@ -192,7 +192,7 @@ case class CaseWhen(branches: Seq[Expression]) extends CaseWhenLike { s""" boolean $got = false; boolean ${ev.isNull} = true; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; $cases $other """ @@ -267,11 +267,11 @@ case class CaseKeyWhen(key: Expression, branches: Seq[Expression]) extends CaseW s""" if (!$got) { ${cond.code} - if (!${cond.isNull} && ${ctx.genEqual(key.dataType, keyEval.primitive, cond.primitive)}) { + if (!${cond.isNull} && ${ctx.genEqual(key.dataType, keyEval.value, cond.value)}) { $got = true; ${res.code} ${ev.isNull} = ${res.isNull}; - ${ev.primitive} = ${res.primitive}; + ${ev.value} = ${res.value}; } } """ @@ -283,7 +283,7 @@ case class CaseKeyWhen(key: Expression, branches: Seq[Expression]) extends CaseW if (!$got) { ${res.code} ${ev.isNull} = ${res.isNull}; - ${ev.primitive} = ${res.primitive}; + ${ev.value} = ${res.value}; } """ } else { @@ -293,7 +293,7 @@ case class CaseKeyWhen(key: Expression, branches: Seq[Expression]) extends CaseW s""" boolean $got = false; boolean ${ev.isNull} = true; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; ${keyEval.code} if (!${keyEval.isNull}) { $cases @@ -351,15 +351,15 @@ case class Least(children: Seq[Expression]) extends Expression { def updateEval(i: Int): String = s""" if (!${evalChildren(i).isNull} && (${ev.isNull} || - ${ctx.genComp(dataType, evalChildren(i).primitive, ev.primitive)} < 0)) { + ${ctx.genComp(dataType, evalChildren(i).value, ev.value)} < 0)) { ${ev.isNull} = false; - ${ev.primitive} = ${evalChildren(i).primitive}; + ${ev.value} = ${evalChildren(i).value}; } """ s""" ${evalChildren.map(_.code).mkString("\n")} boolean ${ev.isNull} = true; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; ${children.indices.map(updateEval).mkString("\n")} """ } @@ -406,15 +406,15 @@ case class Greatest(children: Seq[Expression]) extends Expression { def updateEval(i: Int): String = s""" if (!${evalChildren(i).isNull} && (${ev.isNull} || - ${ctx.genComp(dataType, evalChildren(i).primitive, ev.primitive)} > 0)) { + ${ctx.genComp(dataType, evalChildren(i).value, ev.value)} > 0)) { ${ev.isNull} = false; - ${ev.primitive} = ${evalChildren(i).primitive}; + ${ev.value} = ${evalChildren(i).value}; } """ s""" ${evalChildren.map(_.code).mkString("\n")} boolean ${ev.isNull} = true; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; ${children.indices.map(updateEval).mkString("\n")} """ } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 32dc9b7..13cc6bb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -82,7 +82,7 @@ case class DateAdd(startDate: Expression, days: Expression) override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { nullSafeCodeGen(ctx, ev, (sd, d) => { - s"""${ev.primitive} = $sd + $d;""" + s"""${ev.value} = $sd + $d;""" }) } } @@ -105,7 +105,7 @@ case class DateSub(startDate: Expression, days: Expression) override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { nullSafeCodeGen(ctx, ev, (sd, d) => { - s"""${ev.primitive} = $sd - $d;""" + s"""${ev.value} = $sd - $d;""" }) } } @@ -269,7 +269,7 @@ case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCa """) s""" $c.setTimeInMillis($time * 1000L * 3600L * 24L); - ${ev.primitive} = $c.get($cal.WEEK_OF_YEAR); + ${ev.value} = $c.get($cal.WEEK_OF_YEAR); """ }) } @@ -368,19 +368,19 @@ case class UnixTimestamp(timeExp: Expression, format: Expression) if (fString == null) { s""" boolean ${ev.isNull} = true; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; """ } else { val eval1 = left.gen(ctx) s""" ${eval1.code} boolean ${ev.isNull} = ${eval1.isNull}; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; if (!${ev.isNull}) { try { $sdf $formatter = new $sdf("$fString"); - ${ev.primitive} = - $formatter.parse(${eval1.primitive}.toString()).getTime() / 1000L; + ${ev.value} = + $formatter.parse(${eval1.value}.toString()).getTime() / 1000L; } catch (java.lang.Throwable e) { ${ev.isNull} = true; } @@ -392,7 +392,7 @@ case class UnixTimestamp(timeExp: Expression, format: Expression) nullSafeCodeGen(ctx, ev, (string, format) => { s""" try { - ${ev.primitive} = + ${ev.value} = (new $sdf($format.toString())).parse($string.toString()).getTime() / 1000L; } catch (java.lang.Throwable e) { ${ev.isNull} = true; @@ -404,9 +404,9 @@ case class UnixTimestamp(timeExp: Expression, format: Expression) s""" ${eval1.code} boolean ${ev.isNull} = ${eval1.isNull}; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; if (!${ev.isNull}) { - ${ev.primitive} = ${eval1.primitive} / 1000000L; + ${ev.value} = ${eval1.value} / 1000000L; } """ case DateType => @@ -415,9 +415,9 @@ case class UnixTimestamp(timeExp: Expression, format: Expression) s""" ${eval1.code} boolean ${ev.isNull} = ${eval1.isNull}; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; if (!${ev.isNull}) { - ${ev.primitive} = $dtu.daysToMillis(${eval1.primitive}) / 1000L; + ${ev.value} = $dtu.daysToMillis(${eval1.value}) / 1000L; } """ } @@ -477,18 +477,18 @@ case class FromUnixTime(sec: Expression, format: Expression) if (constFormat == null) { s""" boolean ${ev.isNull} = true; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; """ } else { val t = left.gen(ctx) s""" ${t.code} boolean ${ev.isNull} = ${t.isNull}; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; if (!${ev.isNull}) { try { - ${ev.primitive} = UTF8String.fromString(new $sdf("${constFormat.toString}").format( - new java.util.Date(${t.primitive} * 1000L))); + ${ev.value} = UTF8String.fromString(new $sdf("${constFormat.toString}").format( + new java.util.Date(${t.value} * 1000L))); } catch (java.lang.Throwable e) { ${ev.isNull} = true; } @@ -499,7 +499,7 @@ case class FromUnixTime(sec: Expression, format: Expression) nullSafeCodeGen(ctx, ev, (seconds, f) => { s""" try { - ${ev.primitive} = UTF8String.fromString((new $sdf($f.toString())).format( + ${ev.value} = UTF8String.fromString((new $sdf($f.toString())).format( new java.util.Date($seconds * 1000L))); } catch (java.lang.Throwable e) { ${ev.isNull} = true; @@ -571,7 +571,7 @@ case class NextDay(startDate: Expression, dayOfWeek: Expression) } else { val dayOfWeekValue = DateTimeUtils.getDayOfWeekFromString(input) s""" - |${ev.primitive} = $dateTimeUtilClass.getNextDateForDayOfWeek($sd, $dayOfWeekValue); + |${ev.value} = $dateTimeUtilClass.getNextDateForDayOfWeek($sd, $dayOfWeekValue); """.stripMargin } } else { @@ -580,7 +580,7 @@ case class NextDay(startDate: Expression, dayOfWeek: Expression) |if ($dayOfWeekTerm == -1) { | ${ev.isNull} = true; |} else { - | ${ev.primitive} = $dateTimeUtilClass.getNextDateForDayOfWeek($sd, $dayOfWeekTerm); + | ${ev.value} = $dateTimeUtilClass.getNextDateForDayOfWeek($sd, $dayOfWeekTerm); |} """.stripMargin } @@ -640,7 +640,7 @@ case class FromUTCTimestamp(left: Expression, right: Expression) if (tz == null) { s""" |boolean ${ev.isNull} = true; - |long ${ev.primitive} = 0; + |long ${ev.value} = 0; """.stripMargin } else { val tzTerm = ctx.freshName("tz") @@ -650,10 +650,10 @@ case class FromUTCTimestamp(left: Expression, right: Expression) s""" |${eval.code} |boolean ${ev.isNull} = ${eval.isNull}; - |long ${ev.primitive} = 0; + |long ${ev.value} = 0; |if (!${ev.isNull}) { - | ${ev.primitive} = ${eval.primitive} + - | ${tzTerm}.getOffset(${eval.primitive} / 1000) * 1000L; + | ${ev.value} = ${eval.value} + + | ${tzTerm}.getOffset(${eval.value} / 1000) * 1000L; |} """.stripMargin } @@ -765,7 +765,7 @@ case class ToUTCTimestamp(left: Expression, right: Expression) if (tz == null) { s""" |boolean ${ev.isNull} = true; - |long ${ev.primitive} = 0; + |long ${ev.value} = 0; """.stripMargin } else { val tzTerm = ctx.freshName("tz") @@ -775,10 +775,10 @@ case class ToUTCTimestamp(left: Expression, right: Expression) s""" |${eval.code} |boolean ${ev.isNull} = ${eval.isNull}; - |long ${ev.primitive} = 0; + |long ${ev.value} = 0; |if (!${ev.isNull}) { - | ${ev.primitive} = ${eval.primitive} - - | ${tzTerm}.getOffset(${eval.primitive} / 1000) * 1000L; + | ${ev.value} = ${eval.value} - + | ${tzTerm}.getOffset(${eval.value} / 1000) * 1000L; |} """.stripMargin } @@ -849,16 +849,16 @@ case class TruncDate(date: Expression, format: Expression) if (truncLevel == -1) { s""" boolean ${ev.isNull} = true; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; """ } else { val d = date.gen(ctx) s""" ${d.code} boolean ${ev.isNull} = ${d.isNull}; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; if (!${ev.isNull}) { - ${ev.primitive} = $dtu.truncDate(${d.primitive}, $truncLevel); + ${ev.value} = $dtu.truncDate(${d.value}, $truncLevel); } """ } @@ -870,7 +870,7 @@ case class TruncDate(date: Expression, format: Expression) if ($form == -1) { ${ev.isNull} = true; } else { - ${ev.primitive} = $dtu.truncDate($dateVal, $form); + ${ev.value} = $dtu.truncDate($dateVal, $form); } """ }) http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala index b7be12f..78f6631 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala @@ -55,8 +55,8 @@ case class MakeDecimal(child: Expression, precision: Int, scale: Int) extends Un override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { nullSafeCodeGen(ctx, ev, eval => { s""" - ${ev.primitive} = (new Decimal()).setOrNull($eval, $precision, $scale); - ${ev.isNull} = ${ev.primitive} == null; + ${ev.value} = (new Decimal()).setOrNull($eval, $precision, $scale); + ${ev.isNull} = ${ev.value} == null; """ }) } @@ -97,7 +97,7 @@ case class CheckOverflow(child: Expression, dataType: DecimalType) extends Unary s""" | Decimal $tmp = $eval.clone(); | if ($tmp.changePrecision(${dataType.precision}, ${dataType.scale})) { - | ${ev.primitive} = $tmp; + | ${ev.value} = $tmp; | } else { | ${ev.isNull} = true; | } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala index 8c0c5d5..51be819 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala @@ -97,12 +97,12 @@ case class Literal protected (value: Any, dataType: DataType) // change the isNull and primitive to consts, to inline them if (value == null) { ev.isNull = "true" - s"final ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)};" + s"final ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)};" } else { dataType match { case BooleanType => ev.isNull = "false" - ev.primitive = value.toString + ev.value = value.toString "" case FloatType => val v = value.asInstanceOf[Float] @@ -110,7 +110,7 @@ case class Literal protected (value: Any, dataType: DataType) super.genCode(ctx, ev) } else { ev.isNull = "false" - ev.primitive = s"${value}f" + ev.value = s"${value}f" "" } case DoubleType => @@ -119,20 +119,20 @@ case class Literal protected (value: Any, dataType: DataType) super.genCode(ctx, ev) } else { ev.isNull = "false" - ev.primitive = s"${value}D" + ev.value = s"${value}D" "" } case ByteType | ShortType => ev.isNull = "false" - ev.primitive = s"(${ctx.javaType(dataType)})$value" + ev.value = s"(${ctx.javaType(dataType)})$value" "" case IntegerType | DateType => ev.isNull = "false" - ev.primitive = value.toString + ev.value = value.toString "" case TimestampType | LongType => ev.isNull = "false" - ev.primitive = s"${value}L" + ev.value = s"${value}L" "" // eval() version may be faster for non-primitive types case other => http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index 39de0e8..a8164e9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -89,7 +89,7 @@ abstract class UnaryLogExpression(f: Double => Double, name: String) if ($c <= $yAsymptote) { ${ev.isNull} = true; } else { - ${ev.primitive} = java.lang.Math.${funcName}($c); + ${ev.value} = java.lang.Math.${funcName}($c); } """ ) @@ -191,8 +191,8 @@ case class Conv(numExpr: Expression, fromBaseExpr: Expression, toBaseExpr: Expre val numconv = NumberConverter.getClass.getName.stripSuffix("$") nullSafeCodeGen(ctx, ev, (num, from, to) => s""" - ${ev.primitive} = $numconv.convert($num.getBytes(), $from, $to); - if (${ev.primitive} == null) { + ${ev.value} = $numconv.convert($num.getBytes(), $from, $to); + if (${ev.value} == null) { ${ev.isNull} = true; } """ @@ -270,7 +270,7 @@ case class Factorial(child: Expression) extends UnaryExpression with ImplicitCas if ($eval > 20 || $eval < 0) { ${ev.isNull} = true; } else { - ${ev.primitive} = + ${ev.value} = org.apache.spark.sql.catalyst.expressions.Factorial.factorial($eval); } """ @@ -288,7 +288,7 @@ case class Log2(child: Expression) if ($c <= $yAsymptote) { ${ev.isNull} = true; } else { - ${ev.primitive} = java.lang.Math.log($c) / java.lang.Math.log(2); + ${ev.value} = java.lang.Math.log($c) / java.lang.Math.log(2); } """ ) @@ -432,7 +432,7 @@ case class Hex(child: Expression) extends UnaryExpression with ImplicitCastInput override protected def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { nullSafeCodeGen(ctx, ev, (c) => { val hex = Hex.getClass.getName.stripSuffix("$") - s"${ev.primitive} = " + (child.dataType match { + s"${ev.value} = " + (child.dataType match { case StringType => s"""$hex.hex($c.getBytes());""" case _ => s"""$hex.hex($c);""" }) @@ -458,8 +458,8 @@ case class Unhex(child: Expression) extends UnaryExpression with ImplicitCastInp nullSafeCodeGen(ctx, ev, (c) => { val hex = Hex.getClass.getName.stripSuffix("$") s""" - ${ev.primitive} = $hex.unhex($c.getBytes()); - ${ev.isNull} = ${ev.primitive} == null; + ${ev.value} = $hex.unhex($c.getBytes()); + ${ev.isNull} = ${ev.value} == null; """ }) } @@ -605,7 +605,7 @@ case class Logarithm(left: Expression, right: Expression) if ($c2 <= 0.0) { ${ev.isNull} = true; } else { - ${ev.primitive} = java.lang.Math.log($c2); + ${ev.value} = java.lang.Math.log($c2); } """) } else { @@ -614,7 +614,7 @@ case class Logarithm(left: Expression, right: Expression) if ($c1 <= 0.0 || $c2 <= 0.0) { ${ev.isNull} = true; } else { - ${ev.primitive} = java.lang.Math.log($c2) / java.lang.Math.log($c1); + ${ev.value} = java.lang.Math.log($c2) / java.lang.Math.log($c1); } """) } @@ -727,74 +727,74 @@ case class Round(child: Expression, scale: Expression) val evaluationCode = child.dataType match { case _: DecimalType => s""" - if (${ce.primitive}.changePrecision(${ce.primitive}.precision(), ${_scale})) { - ${ev.primitive} = ${ce.primitive}; + if (${ce.value}.changePrecision(${ce.value}.precision(), ${_scale})) { + ${ev.value} = ${ce.value}; } else { ${ev.isNull} = true; }""" case ByteType => if (_scale < 0) { s""" - ${ev.primitive} = new java.math.BigDecimal(${ce.primitive}). + ${ev.value} = new java.math.BigDecimal(${ce.value}). setScale(${_scale}, java.math.BigDecimal.ROUND_HALF_UP).byteValue();""" } else { - s"${ev.primitive} = ${ce.primitive};" + s"${ev.value} = ${ce.value};" } case ShortType => if (_scale < 0) { s""" - ${ev.primitive} = new java.math.BigDecimal(${ce.primitive}). + ${ev.value} = new java.math.BigDecimal(${ce.value}). setScale(${_scale}, java.math.BigDecimal.ROUND_HALF_UP).shortValue();""" } else { - s"${ev.primitive} = ${ce.primitive};" + s"${ev.value} = ${ce.value};" } case IntegerType => if (_scale < 0) { s""" - ${ev.primitive} = new java.math.BigDecimal(${ce.primitive}). + ${ev.value} = new java.math.BigDecimal(${ce.value}). setScale(${_scale}, java.math.BigDecimal.ROUND_HALF_UP).intValue();""" } else { - s"${ev.primitive} = ${ce.primitive};" + s"${ev.value} = ${ce.value};" } case LongType => if (_scale < 0) { s""" - ${ev.primitive} = new java.math.BigDecimal(${ce.primitive}). + ${ev.value} = new java.math.BigDecimal(${ce.value}). setScale(${_scale}, java.math.BigDecimal.ROUND_HALF_UP).longValue();""" } else { - s"${ev.primitive} = ${ce.primitive};" + s"${ev.value} = ${ce.value};" } case FloatType => // if child eval to NaN or Infinity, just return it. if (_scale == 0) { s""" - if (Float.isNaN(${ce.primitive}) || Float.isInfinite(${ce.primitive})){ - ${ev.primitive} = ${ce.primitive}; + if (Float.isNaN(${ce.value}) || Float.isInfinite(${ce.value})){ + ${ev.value} = ${ce.value}; } else { - ${ev.primitive} = Math.round(${ce.primitive}); + ${ev.value} = Math.round(${ce.value}); }""" } else { s""" - if (Float.isNaN(${ce.primitive}) || Float.isInfinite(${ce.primitive})){ - ${ev.primitive} = ${ce.primitive}; + if (Float.isNaN(${ce.value}) || Float.isInfinite(${ce.value})){ + ${ev.value} = ${ce.value}; } else { - ${ev.primitive} = java.math.BigDecimal.valueOf(${ce.primitive}). + ${ev.value} = java.math.BigDecimal.valueOf(${ce.value}). setScale(${_scale}, java.math.BigDecimal.ROUND_HALF_UP).floatValue(); }""" } case DoubleType => // if child eval to NaN or Infinity, just return it. if (_scale == 0) { s""" - if (Double.isNaN(${ce.primitive}) || Double.isInfinite(${ce.primitive})){ - ${ev.primitive} = ${ce.primitive}; + if (Double.isNaN(${ce.value}) || Double.isInfinite(${ce.value})){ + ${ev.value} = ${ce.value}; } else { - ${ev.primitive} = Math.round(${ce.primitive}); + ${ev.value} = Math.round(${ce.value}); }""" } else { s""" - if (Double.isNaN(${ce.primitive}) || Double.isInfinite(${ce.primitive})){ - ${ev.primitive} = ${ce.primitive}; + if (Double.isNaN(${ce.value}) || Double.isInfinite(${ce.value})){ + ${ev.value} = ${ce.value}; } else { - ${ev.primitive} = java.math.BigDecimal.valueOf(${ce.primitive}). + ${ev.value} = java.math.BigDecimal.valueOf(${ce.value}). setScale(${_scale}, java.math.BigDecimal.ROUND_HALF_UP).doubleValue(); }""" } @@ -803,13 +803,13 @@ case class Round(child: Expression, scale: Expression) if (scaleV == null) { // if scale is null, no need to eval its child at all s""" boolean ${ev.isNull} = true; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; """ } else { s""" ${ce.code} boolean ${ev.isNull} = ${ce.isNull}; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; if (!${ev.isNull}) { $evaluationCode } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala index 8d8d66d..0f6d02f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala @@ -92,18 +92,18 @@ case class Sha2(left: Expression, right: Expression) try { java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-224"); md.update($eval1); - ${ev.primitive} = UTF8String.fromBytes(md.digest()); + ${ev.value} = UTF8String.fromBytes(md.digest()); } catch (java.security.NoSuchAlgorithmException e) { ${ev.isNull} = true; } } else if ($eval2 == 256 || $eval2 == 0) { - ${ev.primitive} = + ${ev.value} = UTF8String.fromString($digestUtils.sha256Hex($eval1)); } else if ($eval2 == 384) { - ${ev.primitive} = + ${ev.value} = UTF8String.fromString($digestUtils.sha384Hex($eval1)); } else if ($eval2 == 512) { - ${ev.primitive} = + ${ev.value} = UTF8String.fromString($digestUtils.sha512Hex($eval1)); } else { ${ev.isNull} = true; @@ -155,7 +155,7 @@ case class Crc32(child: Expression) extends UnaryExpression with ImplicitCastInp s""" $CRC32 checksum = new $CRC32(); checksum.update($value, 0, $value.length); - ${ev.primitive} = checksum.getValue(); + ${ev.value} = checksum.getValue(); """ }) } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala index 287718f..94deafb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala @@ -64,7 +64,7 @@ case class Coalesce(children: Seq[Expression]) extends Expression { override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { s""" boolean ${ev.isNull} = true; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; """ + children.map { e => val eval = e.gen(ctx) @@ -73,7 +73,7 @@ case class Coalesce(children: Seq[Expression]) extends Expression { ${eval.code} if (!${eval.isNull}) { ${ev.isNull} = false; - ${ev.primitive} = ${eval.primitive}; + ${ev.value} = ${eval.value}; } } """ @@ -111,8 +111,8 @@ case class IsNaN(child: Expression) extends UnaryExpression s""" ${eval.code} boolean ${ev.isNull} = false; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; - ${ev.primitive} = !${eval.isNull} && Double.isNaN(${eval.primitive}); + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; + ${ev.value} = !${eval.isNull} && Double.isNaN(${eval.value}); """ } } @@ -152,18 +152,18 @@ case class NaNvl(left: Expression, right: Expression) s""" ${leftGen.code} boolean ${ev.isNull} = false; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; if (${leftGen.isNull}) { ${ev.isNull} = true; } else { - if (!Double.isNaN(${leftGen.primitive})) { - ${ev.primitive} = ${leftGen.primitive}; + if (!Double.isNaN(${leftGen.value})) { + ${ev.value} = ${leftGen.value}; } else { ${rightGen.code} if (${rightGen.isNull}) { ${ev.isNull} = true; } else { - ${ev.primitive} = ${rightGen.primitive}; + ${ev.value} = ${rightGen.value}; } } } @@ -186,7 +186,7 @@ case class IsNull(child: Expression) extends UnaryExpression with Predicate { override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { val eval = child.gen(ctx) ev.isNull = "false" - ev.primitive = eval.isNull + ev.value = eval.isNull eval.code } } @@ -205,7 +205,7 @@ case class IsNotNull(child: Expression) extends UnaryExpression with Predicate { override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { val eval = child.gen(ctx) ev.isNull = "false" - ev.primitive = s"(!(${eval.isNull}))" + ev.value = s"(!(${eval.isNull}))" eval.code } } @@ -249,7 +249,7 @@ case class AtLeastNNonNulls(n: Int, children: Seq[Expression]) extends Predicate s""" if ($nonnull < $n) { ${eval.code} - if (!${eval.isNull} && !Double.isNaN(${eval.primitive})) { + if (!${eval.isNull} && !Double.isNaN(${eval.value})) { $nonnull += 1; } } @@ -269,7 +269,7 @@ case class AtLeastNNonNulls(n: Int, children: Seq[Expression]) extends Predicate int $nonnull = 0; $code boolean ${ev.isNull} = false; - boolean ${ev.primitive} = $nonnull >= $n; + boolean ${ev.value} = $nonnull >= $n; """ } } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala index daefc01..6855747 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala @@ -148,19 +148,19 @@ case class In(value: Expression, list: Seq[Expression]) extends Predicate val listGen = list.map(_.gen(ctx)) val listCode = listGen.map(x => s""" - if (!${ev.primitive}) { + if (!${ev.value}) { ${x.code} if (${x.isNull}) { ${ev.isNull} = true; - } else if (${ctx.genEqual(value.dataType, valueGen.primitive, x.primitive)}) { + } else if (${ctx.genEqual(value.dataType, valueGen.value, x.value)}) { ${ev.isNull} = false; - ${ev.primitive} = true; + ${ev.value} = true; } } """).mkString("\n") s""" ${valueGen.code} - boolean ${ev.primitive} = false; + boolean ${ev.value} = false; boolean ${ev.isNull} = ${valueGen.isNull}; if (!${ev.isNull}) { $listCode @@ -208,10 +208,10 @@ case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with s""" ${childGen.code} boolean ${ev.isNull} = ${childGen.isNull}; - boolean ${ev.primitive} = false; + boolean ${ev.value} = false; if (!${ev.isNull}) { - ${ev.primitive} = $hsetTerm.contains(${childGen.primitive}); - if (!${ev.primitive} && $hasNullTerm) { + ${ev.value} = $hsetTerm.contains(${childGen.value}); + if (!${ev.value} && $hasNullTerm) { ${ev.isNull} = true; } } @@ -251,14 +251,14 @@ case class And(left: Expression, right: Expression) extends BinaryOperator with s""" ${eval1.code} boolean ${ev.isNull} = false; - boolean ${ev.primitive} = false; + boolean ${ev.value} = false; - if (!${eval1.isNull} && !${eval1.primitive}) { + if (!${eval1.isNull} && !${eval1.value}) { } else { ${eval2.code} - if (!${eval2.isNull} && !${eval2.primitive}) { + if (!${eval2.isNull} && !${eval2.value}) { } else if (!${eval1.isNull} && !${eval2.isNull}) { - ${ev.primitive} = true; + ${ev.value} = true; } else { ${ev.isNull} = true; } @@ -300,14 +300,14 @@ case class Or(left: Expression, right: Expression) extends BinaryOperator with P s""" ${eval1.code} boolean ${ev.isNull} = false; - boolean ${ev.primitive} = true; + boolean ${ev.value} = true; - if (!${eval1.isNull} && ${eval1.primitive}) { + if (!${eval1.isNull} && ${eval1.value}) { } else { ${eval2.code} - if (!${eval2.isNull} && ${eval2.primitive}) { + if (!${eval2.isNull} && ${eval2.value}) { } else if (!${eval1.isNull} && !${eval2.isNull}) { - ${ev.primitive} = false; + ${ev.value} = false; } else { ${ev.isNull} = true; } @@ -403,10 +403,10 @@ case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComp override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { val eval1 = left.gen(ctx) val eval2 = right.gen(ctx) - val equalCode = ctx.genEqual(left.dataType, eval1.primitive, eval2.primitive) + val equalCode = ctx.genEqual(left.dataType, eval1.value, eval2.value) ev.isNull = "false" eval1.code + eval2.code + s""" - boolean ${ev.primitive} = (${eval1.isNull} && ${eval2.isNull}) || + boolean ${ev.value} = (${eval1.isNull} && ${eval2.isNull}) || (!${eval1.isNull} && $equalCode); """ } http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala index 62d3d20..8bde8cb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala @@ -69,7 +69,7 @@ case class Rand(seed: Long) extends RDG { s"$rngTerm = new $className(${seed}L + org.apache.spark.TaskContext.getPartitionId());") ev.isNull = "false" s""" - final ${ctx.javaType(dataType)} ${ev.primitive} = $rngTerm.nextDouble(); + final ${ctx.javaType(dataType)} ${ev.value} = $rngTerm.nextDouble(); """ } } @@ -92,7 +92,7 @@ case class Randn(seed: Long) extends RDG { s"$rngTerm = new $className(${seed}L + org.apache.spark.TaskContext.getPartitionId());") ev.isNull = "false" s""" - final ${ctx.javaType(dataType)} ${ev.primitive} = $rngTerm.nextGaussian(); + final ${ctx.javaType(dataType)} ${ev.value} = $rngTerm.nextGaussian(); """ } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
