[kylin] branch 2.6.x updated: KYLIN-3845 solve the problem that kylin build error in stream build task. (#663)

[nic]

This is an automated email from the ASF dual-hosted git repository.

nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/2.6.x by this push:
     new 46eb871  KYLIN-3845 solve the problem that kylin build error in stream 
build task. (#663)
46eb871 is described below

commit 46eb8714fbf1f82b9975c896d46383a8900a96b6
Author: zhaojintaozhao <49121258+zhaojintaoz...@users.noreply.github.com>
AuthorDate: Tue Jun 4 13:40:45 2019 +0800

    KYLIN-3845 solve the problem that kylin build error in stream build task. 
(#663)
    
    * KYLIN-3845 solve the problem that kylin build error if Kafka data source 
lacks selected dimensions or metrics in a kylin stream build task.
---
 .../main/java/org/apache/kylin/cube/util/KeyValueBuilder.java  |  4 ++++
 .../main/java/org/apache/kylin/dict/DictionaryGenerator.java   |  8 ++++----
 .../kylin/engine/mr/steps/FactDistinctColumnsMapper.java       | 10 +++++++++-
 3 files changed, 17 insertions(+), 5 deletions(-)

diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java 
b/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java
index 0636a5c..fab1448 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java
@@ -61,6 +61,10 @@ public class KeyValueBuilder implements Serializable {
     }
 
     private String getCell(int i, String[] flatRow) {
+        if (i >= flatRow.length) {
+            return null;
+        }
+
         if (isNull(flatRow[i]))
             return null;
         else
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 7c33b4a..48d2b08 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -92,7 +92,9 @@ public class DictionaryGenerator {
 
         // build
         Dictionary<String> dict = builder.build();
-
+        logger.debug("Dictionary cardinality: " + dict.getSize());
+        logger.debug("Dictionary builder class: " + 
builder.getClass().getName());
+        logger.debug("Dictionary class: " + dict.getClass().getName());
         // log a few samples
         StringBuilder buf = new StringBuilder();
         for (String s : samples) {
@@ -102,9 +104,7 @@ public class DictionaryGenerator {
             
buf.append(s.toString()).append("=>").append(dict.getIdFromValue(s));
         }
         logger.debug("Dictionary value samples: " + buf.toString());
-        logger.debug("Dictionary cardinality: " + dict.getSize());
-        logger.debug("Dictionary builder class: " + 
builder.getClass().getName());
-        logger.debug("Dictionary class: " + dict.getClass().getName());
+
         return dict;
     }
 
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 7bffce7..2c3bc8d 100755
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -174,7 +174,15 @@ public class FactDistinctColumnsMapper<KEYIN> extends 
FactDistinctColumnsMapperB
         for (String[] row : rowCollection) {
             
context.getCounter(RawDataCounter.BYTES).increment(countSizeInBytes(row));
             for (int i = 0; i < allCols.size(); i++) {
-                String fieldValue = row[columnIndex[i]];
+                int colIndex = columnIndex[i];
+                int rowSize = row.length;
+                String fieldValue = " ";
+                if (colIndex <= rowSize - 1) {
+                    fieldValue = row[colIndex];
+                } else {
+                    logger.debug(
+                            "colIndex:" + colIndex + " is more than rowSize: " 
+ rowSize + " -1, so set empty value.");
+                }
                 if (fieldValue == null)
                     continue;