This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 80384da925e7b284eea461d21754ebc2d7695c66
Author: XiaoxiangYu <hit_la...@126.com>
AuthorDate: Sat Sep 21 22:25:26 2019 +0800
KYLIN-4167 Fix minor bug
---
build/conf/kylin-server-log4j.properties | 12 +-
.../kylin/storage/gtrecord/CubeTupleConverter.java | 2 +-
.../optrule/visitor/TimezoneRewriteVisitor.java | 152 ---------------------
.../relnode/visitor/TupleFilterVisitorTest.java | 9 +-
.../coordinator/coordinate/BuildJobSubmitter.java | 7 +-
.../coordinate/StreamingCoordinator.java | 3 +-
6 files changed, 18 insertions(+), 167 deletions(-)
diff --git a/build/conf/kylin-server-log4j.properties
b/build/conf/kylin-server-log4j.properties
index 0b3cef9..e4d204f 100644
--- a/build/conf/kylin-server-log4j.properties
+++ b/build/conf/kylin-server-log4j.properties
@@ -29,14 +29,4 @@ log4j.appender.file.MaxBackupIndex=10
log4j.rootLogger=INFO,file
log4j.logger.org.apache.kylin=DEBUG
log4j.logger.org.springframework=WARN
-log4j.logger.org.springframework.security=INFO
-
-# Realtime OLAP config
-log4j.appender.realtime=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.realtime.layout=org.apache.log4j.PatternLayout
-log4j.appender.realtime.File=${catalina.home}/../logs/kylin_coordinator.log
-log4j.appender.realtime.layout.ConversionPattern=%d{ISO8601} %-5p [%t]
%c{2}:%L : %m%n
-log4j.appender.realtime.Append=true
-log4j.appender.realtime.MaxFileSize=268435456
-log4j.appender.realtime.MaxBackupIndex=10
-log4j.logger.org.apache.kylin.stream.*=DEBUG, realtime
\ No newline at end of file
+log4j.logger.org.springframework.security=INFO
\ No newline at end of file
diff --git
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
index 2e04daa..94734b3 100644
---
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
+++
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
@@ -163,7 +163,7 @@ public class CubeTupleConverter implements ITupleConverter {
int ti = tupleIdx[i];
if (ti >= 0) {
// add offset to return result according to timezone
- if (autoJustByTimezone && timestampColumn.contains(i)) {
+ if (autoJustByTimezone && timestampColumn.contains(ti)) {
try {
String v = toString(gtValues[i]);
if (v != null) {
diff --git
a/query/src/main/java/org/apache/kylin/query/optrule/visitor/TimezoneRewriteVisitor.java
b/query/src/main/java/org/apache/kylin/query/optrule/visitor/TimezoneRewriteVisitor.java
deleted file mode 100644
index 731ee00..0000000
---
a/query/src/main/java/org/apache/kylin/query/optrule/visitor/TimezoneRewriteVisitor.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.query.optrule.visitor;
-
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexCorrelVariable;
-import org.apache.calcite.rex.RexDynamicParam;
-import org.apache.calcite.rex.RexFieldAccess;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexLiteral;
-import org.apache.calcite.rex.RexLocalRef;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.rex.RexOver;
-import org.apache.calcite.rex.RexPatternFieldRef;
-import org.apache.calcite.rex.RexRangeRef;
-import org.apache.calcite.rex.RexSubQuery;
-import org.apache.calcite.rex.RexTableInputRef;
-import org.apache.calcite.rex.RexVisitorImpl;
-import org.apache.calcite.sql.SqlKind;
-import org.apache.calcite.sql.type.SqlTypeFamily;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.time.LocalDateTime;
-import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-
-public class TimezoneRewriteVisitor extends RexVisitorImpl<RexNode> {
- public static final Logger logger =
LoggerFactory.getLogger(TimezoneRewriteVisitor.class);
-
- public TimezoneRewriteVisitor(boolean deep) {
- super(deep);
- }
-
- @Override
- public RexNode visitCall(RexCall call) {
- List<RexNode> subList = new ArrayList<>();
- SqlTypeName type = call.getType().getSqlTypeName();
- if (call.getKind() == SqlKind.CAST) {
- if (type.getFamily() == SqlTypeFamily.DATE || type.getFamily() ==
SqlTypeFamily.DATETIME
- || type.getFamily() == SqlTypeFamily.TIMESTAMP) {
- for (RexNode node : call.getOperands()) {
- if (node instanceof RexLiteral) {
- RexLiteral literal = (RexLiteral) node;
- String toBeModify = literal.getValue2().toString();
- logger.info(toBeModify);
- if (toBeModify.length() != 19) {
- subList.add(node);
- continue;
- }
- // minus offset by timezone in RelNode level
- // this will affect code gen
- int minusHours = 0;
- String afetrModify = LocalDateTime
- .parse(toBeModify,
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss",
Locale.ROOT)).minusHours(minusHours)
-
.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss", Locale.ROOT));
- logger.info("{} -> {}", toBeModify, afetrModify);
- RexLiteral newliteral =
RexLiteral.fromJdbcString(literal.getType(), literal.getTypeName(),
- afetrModify);
- subList.add(newliteral);
- } else {
- subList.add(node);
- }
- }
- }
- return call.clone(call.type, subList);
- }
-
- for (RexNode operand : call.operands) {
- RexNode node = operand.accept(this);
- subList.add(node);
- }
- return call.clone(call.type, subList);
- }
-
- @Override
- public RexNode visitLiteral(RexLiteral literal) {
- return literal;
- }
-
- @Override
- public RexNode visitInputRef(RexInputRef inputRef) {
- return inputRef;
- }
-
- @Override
- public RexNode visitLocalRef(RexLocalRef localRef) {
- return localRef;
- }
-
- @Override
- public RexNode visitOver(RexOver over) {
- return over;
- }
-
- @Override
- public RexNode visitCorrelVariable(RexCorrelVariable correlVariable) {
- return correlVariable;
- }
-
- @Override
- public RexNode visitDynamicParam(RexDynamicParam dynamicParam) {
- return dynamicParam;
- }
-
- @Override
- public RexNode visitRangeRef(RexRangeRef rangeRef) {
- return rangeRef;
- }
-
- @Override
- public RexNode visitFieldAccess(RexFieldAccess fieldAccess) {
- return fieldAccess;
- }
-
- @Override
- public RexNode visitSubQuery(RexSubQuery subQuery) {
- return subQuery;
- }
-
- @Override
- public RexNode visitTableInputRef(RexTableInputRef ref) {
- return ref;
- }
-
- @Override
- public RexNode visitPatternFieldRef(RexPatternFieldRef fieldRef) {
- return fieldRef;
- }
-
- public static void main(String[] args) {
-
- }
-}
diff --git
a/query/src/test/java/org/apache/kylin/query/relnode/visitor/TupleFilterVisitorTest.java
b/query/src/test/java/org/apache/kylin/query/relnode/visitor/TupleFilterVisitorTest.java
index 352dfab..99c4645 100644
---
a/query/src/test/java/org/apache/kylin/query/relnode/visitor/TupleFilterVisitorTest.java
+++
b/query/src/test/java/org/apache/kylin/query/relnode/visitor/TupleFilterVisitorTest.java
@@ -19,6 +19,7 @@
package org.apache.kylin.query.relnode.visitor;
import com.google.common.collect.Lists;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
@@ -26,9 +27,15 @@ import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
-public class TupleFilterVisitorTest {
+public class TupleFilterVisitorTest extends LocalFileMetadataTestCase {
+
+ @BeforeClass
+ public static void setupResource() {
+ staticCreateTestMetadata();
+ }
@Test
public void testMergeToInClause1() {
diff --git
a/stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/BuildJobSubmitter.java
b/stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/BuildJobSubmitter.java
index cf3771b..9e32f46 100644
---
a/stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/BuildJobSubmitter.java
+++
b/stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/BuildJobSubmitter.java
@@ -104,7 +104,12 @@ public class BuildJobSubmitter implements Runnable {
buildInfos = previousValue;
}
}
- buildInfos.add(segmentBuildJob);
+ boolean addSucceed = buildInfos.add(segmentBuildJob);
+ if (!addSucceed) {
+ logger.debug("Add {} failed because we have a duplicated one.",
segmentBuildJob);
+ buildInfos.remove(segmentBuildJob);
+ buildInfos.add(segmentBuildJob);
+ }
}
void addToCheckList(String cubeName) {
diff --git
a/stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/StreamingCoordinator.java
b/stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/StreamingCoordinator.java
index 875a4d7..7c3c575 100644
---
a/stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/StreamingCoordinator.java
+++
b/stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/StreamingCoordinator.java
@@ -237,7 +237,8 @@ public class StreamingCoordinator implements
CoordinatorClient {
@Override
public void segmentRemoteStoreComplete(Node receiver, String cubeName,
Pair<Long, Long> segment) {
checkLead();
- logger.info("Segment remote store complete signal received for
cube:{}, segment:{}", cubeName, segment);
+ logger.info("Segment remote store complete signal received for
cube:{}, segment:{}, by {}.", cubeName, segment,
+ receiver);
buildJobSubmitter.addToCheckList(cubeName);
}
