[incubator-pinot] 01/01: Use docBase from leaf context for multi-segment search

[jlli]

This is an automated email from the ASF dual-hosted git repository.

jlli pushed a commit to branch hotfixtextindex
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git

commit f455a6d9dcb131ac0b025849d80cd99f20b9a66c
Author: Siddharth Teotia <steo...@steotia-mn1.linkedin.biz>
AuthorDate: Mon Oct 26 09:39:37 2020 -0700

    Use docBase from leaf context for multi-segment search
---
 .travis/.travis_install.sh                              | 17 ++++++++++++++---
 .travis/.travis_nightly_build.sh                        |  2 +-
 .../index/readers/text/LuceneDocIdCollector.java        |  6 +++++-
 3 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/.travis/.travis_install.sh b/.travis/.travis_install.sh
index 5b676cc..0f9808f 100755
--- a/.travis/.travis_install.sh
+++ b/.travis/.travis_install.sh
@@ -49,17 +49,28 @@ java -version
 if [ $noThirdEyeChange -ne 0 ]; then
   echo "Full Pinot build"
   echo "No ThirdEye changes"
+
+  echo "Fixing git setup for $TRAVIS_BRANCH"
+  git checkout ${TRAVIS_BRANCH}
+  git branch -u origin/${TRAVIS_BRANCH}
+  git config branch.${TRAVIS_BRANCH}.remote origin
+  git config branch.${TRAVIS_BRANCH}.merge refs/heads/${TRAVIS_BRANCH}
+
   if [ "$TRAVIS_JDK_VERSION" != 'oraclejdk8' ]; then
+    df -h
+    ls -al /tmp/
+    du -sh /tmp/
+    du -sh /tmp/*
     # JDK 11 prints more logs exceeding Travis limits.
-    mvn clean install -B -DskipTests=true -Pbin-dist -Dmaven.javadoc.skip=true 
${DEPLOY_BUILD_OPTS} ${KAFKA_BUILD_OPTS} > /tmp/mvn_build_log
+    mvn clean install -B -DscmBranch="${TRAVIS_BRANCH}" -DskipTests=true 
-Pbin-dist -Dmaven.javadoc.skip=true ${KAFKA_BUILD_OPTS}
     if [ $? -eq 0 ]; then
       exit 0
     else
-      tail -1000 /tmp/mvn_build_log
+#      cat /tmp/mvn_build_log
       exit 1
     fi
   else
-    mvn clean install -B -DskipTests=true -Pbin-dist -Dmaven.javadoc.skip=true 
${DEPLOY_BUILD_OPTS} ${KAFKA_BUILD_OPTS} || exit $?
+    mvn clean install -B -DscmBranch="${TRAVIS_BRANCH}" -DskipTests=true 
-Pbin-dist -Dmaven.javadoc.skip=true ${KAFKA_BUILD_OPTS} || exit $?
   fi
 fi
 
diff --git a/.travis/.travis_nightly_build.sh b/.travis/.travis_nightly_build.sh
index 602a4bf..27f9447 100755
--- a/.travis/.travis_nightly_build.sh
+++ b/.travis/.travis_nightly_build.sh
@@ -27,5 +27,5 @@ if [ -n "${DEPLOY_BUILD_OPTS}" ]; then
   mvn versions:commit -q -B
 
   # Deploy to bintray
-  mvn deploy -s .travis/.ci.settings.xml -DskipTests -q 
-DretryFailedDeploymentCount=5 
-DaltDeploymentRepository=bintray-linkedin-maven::default::'https://api.bintray.com/maven/linkedin/maven/pinot/;publish=1;override=1'
+  mvn deploy -s .travis/.ci.settings.xml -DscmBranch="${TRAVIS_BRANCH}" 
-DskipTests -q -DretryFailedDeploymentCount=5 
-DaltDeploymentRepository=bintray-linkedin-maven::default::'https://api.bintray.com/maven/linkedin/maven/pinot/;publish=1;override=1'
 fi
diff --git 
a/pinot-core/src/main/java/org/apache/pinot/core/segment/index/readers/text/LuceneDocIdCollector.java
 
b/pinot-core/src/main/java/org/apache/pinot/core/segment/index/readers/text/LuceneDocIdCollector.java
index b24667b..723dd00 100644
--- 
a/pinot-core/src/main/java/org/apache/pinot/core/segment/index/readers/text/LuceneDocIdCollector.java
+++ 
b/pinot-core/src/main/java/org/apache/pinot/core/segment/index/readers/text/LuceneDocIdCollector.java
@@ -67,7 +67,11 @@ public class LuceneDocIdCollector implements Collector {
 
       @Override
       public void collect(int doc) throws IOException {
-        _docIds.add(_docIdTranslator.getPinotDocId(doc));
+        // even though we merge lucene sub-indexes, there could still be cases
+        // that can lead to multiple sub-indexes. For search on multiple
+        // sub-indexes, we need to use compute the absolute lucene docID across
+        // sub-indexes because that's how the lookup table in docIdTranslator 
is built
+        _docIds.add(_docIdTranslator.getPinotDocId(context.docBase + doc));
       }
     };
   }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org