minor change on error messages Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c587b2ed Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c587b2ed Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c587b2ed
Branch: refs/heads/KYLIN-1971 Commit: c587b2ed3a968262a794e893f6af8fc109b02730 Parents: 235b123 Author: shaofengshi <shaofeng...@apache.org> Authored: Fri Nov 4 13:33:19 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Fri Nov 4 13:33:19 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/engine/mr/common/AbstractHadoopJob.java | 8 ++++---- .../apache/kylin/engine/mr/common/HadoopStatusGetter.java | 2 +- .../main/java/org/apache/kylin/source/kafka/KafkaSource.java | 8 ++++++++ 3 files changed, 13 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/c587b2ed/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index 77791ce..21bb10e 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -204,7 +204,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { StringUtil.appendWithSeparator(kylinDependency, filteredHive); } else { - logger.info("No hive dependency jars set in the environment, will find them from jvm:"); + logger.info("No hive dependency jars set in the environment, will find them from classpath:"); try { String hiveExecJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver")); @@ -227,17 +227,17 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { // for kafka dependencies if (kylinKafkaDependency != null) { kylinKafkaDependency = kylinKafkaDependency.replace(":", ","); - logger.info("Kafka Dependencies Before Filtered: " + kylinKafkaDependency); + logger.info("Kafka Dependencies: " + kylinKafkaDependency); StringUtil.appendWithSeparator(kylinDependency, kylinKafkaDependency); } else { - logger.info("No Kafka dependency jars set in the environment, will find them from jvm:"); + logger.info("No Kafka dependency jar set in the environment, will find them from classpath:"); try { String kafkaClientJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.kafka.clients.consumer.KafkaConsumer")); StringUtil.appendWithSeparator(kylinDependency, kafkaClientJarPath); logger.info("kafka jar file: " + kafkaClientJarPath); } catch (ClassNotFoundException e) { - logger.error("Cannot found kafka dependency jars: " + e); + logger.warn("Not found kafka client jar from classpath, it is optional for normal build: " + e); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/c587b2ed/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java index 619de90..7dcb73e 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java @@ -55,7 +55,7 @@ public class HadoopStatusGetter { private final String mrJobId; private final String yarnUrl; - protected static final Logger logger = LoggerFactory.getLogger(HadoopStatusChecker.class); + protected static final Logger logger = LoggerFactory.getLogger(HadoopStatusGetter.class); public HadoopStatusGetter(String yarnUrl, String mrJobId) { this.yarnUrl = yarnUrl; http://git-wip-us.apache.org/repos/asf/kylin/blob/c587b2ed/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java ---------------------------------------------------------------------- diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java index bb676e6..7a5d94f 100644 --- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java +++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java @@ -107,6 +107,14 @@ public class KafkaSource implements ISource { totalEndOffset += v; } + if (totalStartOffset > totalEndOffset) { + throw new IllegalArgumentException("Illegal offset: start: " + totalStartOffset + ", end: " + totalEndOffset); + } + + if (totalStartOffset == totalEndOffset) { + throw new IllegalArgumentException("No new message comes, startOffset = endOffset:" + totalStartOffset); + } + result.setStartOffset(totalStartOffset); result.setEndOffset(totalEndOffset);
