Repository: spark Updated Branches: refs/heads/master 9672602c7 -> f5d154bc7
http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index 6dff28a..64f1594 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -92,15 +92,15 @@ case class Like(left: Expression, right: Expression) s""" ${eval.code} boolean ${ev.isNull} = ${eval.isNull}; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; if (!${ev.isNull}) { - ${ev.primitive} = $pattern.matcher(${eval.primitive}.toString()).matches(); + ${ev.value} = $pattern.matcher(${eval.value}.toString()).matches(); } """ } else { s""" boolean ${ev.isNull} = true; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; """ } } else { @@ -108,7 +108,7 @@ case class Like(left: Expression, right: Expression) s""" String rightStr = ${eval2}.toString(); ${patternClass} $pattern = ${patternClass}.compile($escapeFunc(rightStr)); - ${ev.primitive} = $pattern.matcher(${eval1}.toString()).matches(); + ${ev.value} = $pattern.matcher(${eval1}.toString()).matches(); """ }) } @@ -140,15 +140,15 @@ case class RLike(left: Expression, right: Expression) s""" ${eval.code} boolean ${ev.isNull} = ${eval.isNull}; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; if (!${ev.isNull}) { - ${ev.primitive} = $pattern.matcher(${eval.primitive}.toString()).find(0); + ${ev.value} = $pattern.matcher(${eval.value}.toString()).find(0); } """ } else { s""" boolean ${ev.isNull} = true; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; """ } } else { @@ -156,7 +156,7 @@ case class RLike(left: Expression, right: Expression) s""" String rightStr = ${eval2}.toString(); ${patternClass} $pattern = ${patternClass}.compile(rightStr); - ${ev.primitive} = $pattern.matcher(${eval1}.toString()).find(0); + ${ev.value} = $pattern.matcher(${eval1}.toString()).find(0); """ }) } @@ -184,7 +184,7 @@ case class StringSplit(str: Expression, pattern: Expression) val arrayClass = classOf[GenericArrayData].getName nullSafeCodeGen(ctx, ev, (str, pattern) => // Array in java is covariant, so we don't need to cast UTF8String[] to Object[]. - s"""${ev.primitive} = new $arrayClass($str.split($pattern, -1));""") + s"""${ev.value} = new $arrayClass($str.split($pattern, -1));""") } override def prettyName: String = "split" @@ -275,7 +275,7 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio m.appendReplacement(${termResult}, ${termLastReplacement}); } m.appendTail(${termResult}); - ${ev.primitive} = UTF8String.fromString(${termResult}.toString()); + ${ev.value} = UTF8String.fromString(${termResult}.toString()); ${ev.isNull} = false; """ }) @@ -335,10 +335,10 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio ${termPattern}.matcher($subject.toString()); if (m.find()) { java.util.regex.MatchResult mr = m.toMatchResult(); - ${ev.primitive} = UTF8String.fromString(mr.group($idx)); + ${ev.value} = UTF8String.fromString(mr.group($idx)); ${ev.isNull} = false; } else { - ${ev.primitive} = UTF8String.EMPTY_UTF8; + ${ev.value} = UTF8String.EMPTY_UTF8; ${ev.isNull} = false; }""" }) http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/sets.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/sets.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/sets.scala index 5b0fe8d..d124d29 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/sets.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/sets.scala @@ -67,7 +67,7 @@ case class NewSet(elementType: DataType) extends LeafExpression with CodegenFall case IntegerType | LongType => ev.isNull = "false" s""" - ${ctx.javaType(dataType)} ${ev.primitive} = new ${ctx.javaType(dataType)}(); + ${ctx.javaType(dataType)} ${ev.value} = new ${ctx.javaType(dataType)}(); """ case _ => super.genCode(ctx, ev) } @@ -116,10 +116,10 @@ case class AddItemToSet(item: Expression, set: Expression) val htype = ctx.javaType(dataType) ev.isNull = "false" - ev.primitive = setEval.primitive + ev.value = setEval.value itemEval.code + setEval.code + s""" if (!${itemEval.isNull} && !${setEval.isNull}) { - (($htype)${setEval.primitive}).add(${itemEval.primitive}); + (($htype)${setEval.value}).add(${itemEval.value}); } """ case _ => super.genCode(ctx, ev) @@ -167,10 +167,10 @@ case class CombineSets(left: Expression, right: Expression) val htype = ctx.javaType(dataType) ev.isNull = leftEval.isNull - ev.primitive = leftEval.primitive + ev.value = leftEval.value leftEval.code + rightEval.code + s""" if (!${leftEval.isNull} && !${rightEval.isNull}) { - ${leftEval.primitive}.union((${htype})${rightEval.primitive}); + ${leftEval.value}.union((${htype})${rightEval.value}); } """ case _ => super.genCode(ctx, ev) http://git-wip-us.apache.org/repos/asf/spark/blob/f5d154bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 4ab27c0..abc5c94 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -50,12 +50,12 @@ case class Concat(children: Seq[Expression]) extends Expression with ImplicitCas override protected def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { val evals = children.map(_.gen(ctx)) val inputs = evals.map { eval => - s"${eval.isNull} ? null : ${eval.primitive}" + s"${eval.isNull} ? null : ${eval.value}" }.mkString(", ") evals.map(_.code).mkString("\n") + s""" boolean ${ev.isNull} = false; - UTF8String ${ev.primitive} = UTF8String.concat($inputs); - if (${ev.primitive} == null) { + UTF8String ${ev.value} = UTF8String.concat($inputs); + if (${ev.value} == null) { ${ev.isNull} = true; } """ @@ -104,12 +104,12 @@ case class ConcatWs(children: Seq[Expression]) val evals = children.map(_.gen(ctx)) val inputs = evals.map { eval => - s"${eval.isNull} ? (UTF8String) null : ${eval.primitive}" + s"${eval.isNull} ? (UTF8String) null : ${eval.value}" }.mkString(", ") evals.map(_.code).mkString("\n") + s""" - UTF8String ${ev.primitive} = UTF8String.concatWs($inputs); - boolean ${ev.isNull} = ${ev.primitive} == null; + UTF8String ${ev.value} = UTF8String.concatWs($inputs); + boolean ${ev.isNull} = ${ev.value} == null; """ } else { val array = ctx.freshName("array") @@ -121,19 +121,19 @@ case class ConcatWs(children: Seq[Expression]) child.dataType match { case StringType => ("", // we count all the StringType arguments num at once below. - s"$array[$idxInVararg ++] = ${eval.isNull} ? (UTF8String) null : ${eval.primitive};") + s"$array[$idxInVararg ++] = ${eval.isNull} ? (UTF8String) null : ${eval.value};") case _: ArrayType => val size = ctx.freshName("n") (s""" if (!${eval.isNull}) { - $varargNum += ${eval.primitive}.numElements(); + $varargNum += ${eval.value}.numElements(); } """, s""" if (!${eval.isNull}) { - final int $size = ${eval.primitive}.numElements(); + final int $size = ${eval.value}.numElements(); for (int j = 0; j < $size; j ++) { - $array[$idxInVararg ++] = ${ctx.getValue(eval.primitive, StringType, "j")}; + $array[$idxInVararg ++] = ${ctx.getValue(eval.value, StringType, "j")}; } } """) @@ -147,8 +147,8 @@ case class ConcatWs(children: Seq[Expression]) ${varargCount.mkString("\n")} UTF8String[] $array = new UTF8String[$varargNum]; ${varargBuild.mkString("\n")} - UTF8String ${ev.primitive} = UTF8String.concatWs(${evals.head.primitive}, $array); - boolean ${ev.isNull} = ${ev.primitive} == null; + UTF8String ${ev.value} = UTF8String.concatWs(${evals.head.value}, $array); + boolean ${ev.isNull} = ${ev.value} == null; """ } } @@ -308,7 +308,7 @@ case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replac ${termDict} = org.apache.spark.sql.catalyst.expressions.StringTranslate .buildDict(${termLastMatching}, ${termLastReplace}); } - ${ev.primitive} = ${src}.translate(${termDict}); + ${ev.value} = ${src}.translate(${termDict}); """ }) } @@ -334,7 +334,7 @@ case class FindInSet(left: Expression, right: Expression) extends BinaryExpressi override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { nullSafeCodeGen(ctx, ev, (word, set) => - s"${ev.primitive} = $set.findInSet($word);" + s"${ev.value} = $set.findInSet($word);" ) } @@ -481,7 +481,7 @@ case class StringLocate(substr: Expression, str: Expression, start: Expression) val strGen = str.gen(ctx) val startGen = start.gen(ctx) s""" - int ${ev.primitive} = 0; + int ${ev.value} = 0; boolean ${ev.isNull} = false; ${startGen.code} if (!${startGen.isNull}) { @@ -489,8 +489,8 @@ case class StringLocate(substr: Expression, str: Expression, start: Expression) if (!${substrGen.isNull}) { ${strGen.code} if (!${strGen.isNull}) { - ${ev.primitive} = ${strGen.primitive}.indexOf(${substrGen.primitive}, - ${startGen.primitive}) + 1; + ${ev.value} = ${strGen.value}.indexOf(${substrGen.value}, + ${startGen.value}) + 1; } else { ${ev.isNull} = true; } @@ -586,9 +586,9 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC if (ctx.boxedType(v._1) != ctx.javaType(v._1)) { // Java primitives get boxed in order to allow null values. s"(${v._2.isNull}) ? (${ctx.boxedType(v._1)}) null : " + - s"new ${ctx.boxedType(v._1)}(${v._2.primitive})" + s"new ${ctx.boxedType(v._1)}(${v._2.value})" } else { - s"(${v._2.isNull}) ? null : ${v._2.primitive}" + s"(${v._2.isNull}) ? null : ${v._2.value}" } s + "," + nullSafeString }) @@ -600,13 +600,13 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC s""" ${pattern.code} boolean ${ev.isNull} = ${pattern.isNull}; - ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)}; + ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; if (!${ev.isNull}) { ${argListCode.mkString} $stringBuffer $sb = new $stringBuffer(); $formatter $form = new $formatter($sb, ${classOf[Locale].getName}.US); - $form.format(${pattern.primitive}.toString() $argListString); - ${ev.primitive} = UTF8String.fromString($sb.toString()); + $form.format(${pattern.value}.toString() $argListString); + ${ev.value} = UTF8String.fromString($sb.toString()); } """ } @@ -682,7 +682,7 @@ case class StringSpace(child: Expression) override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { nullSafeCodeGen(ctx, ev, (length) => - s"""${ev.primitive} = UTF8String.blankString(($length < 0) ? 0 : $length);""") + s"""${ev.value} = UTF8String.blankString(($length < 0) ? 0 : $length);""") } override def prettyName: String = "space" @@ -760,7 +760,7 @@ case class Levenshtein(left: Expression, right: Expression) extends BinaryExpres override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { nullSafeCodeGen(ctx, ev, (left, right) => - s"${ev.primitive} = $left.levenshteinDistance($right);") + s"${ev.value} = $left.levenshteinDistance($right);") } } @@ -803,9 +803,9 @@ case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInp s""" byte[] $bytes = $child.getBytes(); if ($bytes.length > 0) { - ${ev.primitive} = (int) $bytes[0]; + ${ev.value} = (int) $bytes[0]; } else { - ${ev.primitive} = 0; + ${ev.value} = 0; } """}) } @@ -827,7 +827,7 @@ case class Base64(child: Expression) extends UnaryExpression with ImplicitCastIn override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { nullSafeCodeGen(ctx, ev, (child) => { - s"""${ev.primitive} = UTF8String.fromBytes( + s"""${ev.value} = UTF8String.fromBytes( org.apache.commons.codec.binary.Base64.encodeBase64($child)); """}) } @@ -848,7 +848,7 @@ case class UnBase64(child: Expression) extends UnaryExpression with ImplicitCast override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = { nullSafeCodeGen(ctx, ev, (child) => { s""" - ${ev.primitive} = org.apache.commons.codec.binary.Base64.decodeBase64($child.toString()); + ${ev.value} = org.apache.commons.codec.binary.Base64.decodeBase64($child.toString()); """}) } } @@ -875,7 +875,7 @@ case class Decode(bin: Expression, charset: Expression) nullSafeCodeGen(ctx, ev, (bytes, charset) => s""" try { - ${ev.primitive} = UTF8String.fromString(new String($bytes, $charset.toString())); + ${ev.value} = UTF8String.fromString(new String($bytes, $charset.toString())); } catch (java.io.UnsupportedEncodingException e) { org.apache.spark.unsafe.Platform.throwException(e); } @@ -905,7 +905,7 @@ case class Encode(value: Expression, charset: Expression) nullSafeCodeGen(ctx, ev, (string, charset) => s""" try { - ${ev.primitive} = $string.toString().getBytes($charset.toString()); + ${ev.value} = $string.toString().getBytes($charset.toString()); } catch (java.io.UnsupportedEncodingException e) { org.apache.spark.unsafe.Platform.throwException(e); }""") @@ -1014,9 +1014,9 @@ case class FormatNumber(x: Expression, d: Expression) $lastDValue = $d; $numberFormat.applyPattern($dFormat.toPattern()); } - ${ev.primitive} = UTF8String.fromString($numberFormat.format(${typeHelper(num)})); + ${ev.value} = UTF8String.fromString($numberFormat.format(${typeHelper(num)})); } else { - ${ev.primitive} = null; + ${ev.value} = null; ${ev.isNull} = true; } """ --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
