[doris-thirdparty] branch clucene updated: [fix](analyzer) Fix memory leak in Standard Analyzer reuseTokenStream (#60)

[adonisling]

This is an automated email from the ASF dual-hosted git repository.

adonisling pushed a commit to branch clucene
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git


The following commit(s) were added to refs/heads/clucene by this push:
     new f0eb372  [fix](analyzer) Fix memory leak in Standard Analyzer 
reuseTokenStream (#60)
f0eb372 is described below

commit f0eb3726b65038e43b83e6f577782eb9c4043b80
Author: airborne12 <airborn...@gmail.com>
AuthorDate: Thu Apr 27 14:15:02 2023 +0800

    [fix](analyzer) Fix memory leak in Standard Analyzer reuseTokenStream (#60)
---
 src/core/CLucene/analysis/AnalysisHeader.h              | 4 ++--
 src/core/CLucene/analysis/standard/StandardAnalyzer.cpp | 5 ++++-
 src/core/CLucene/analysis/standard/StandardAnalyzer.h   | 8 ++++++++
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/core/CLucene/analysis/AnalysisHeader.h 
b/src/core/CLucene/analysis/AnalysisHeader.h
index e3d572a..de19835 100644
--- a/src/core/CLucene/analysis/AnalysisHeader.h
+++ b/src/core/CLucene/analysis/AnalysisHeader.h
@@ -267,12 +267,12 @@ protected:
     /** Used by Analyzers that implement reusableTokenStream
        *  to retrieve previously saved TokenStreams for re-use
        *  by the same thread. */
-    TokenStream* getPreviousTokenStream();
+    virtual TokenStream* getPreviousTokenStream();
 
     /** Used by Analyzers that implement reusableTokenStream
        *  to save a TokenStream for later re-use by the same
        *  thread. */
-    void setPreviousTokenStream(TokenStream* obj);
+    virtual void setPreviousTokenStream(TokenStream* obj);
 public:
     /**
        * Invoked before indexing a Field instance if
diff --git a/src/core/CLucene/analysis/standard/StandardAnalyzer.cpp 
b/src/core/CLucene/analysis/standard/StandardAnalyzer.cpp
index 3275c1b..d0383f2 100644
--- a/src/core/CLucene/analysis/standard/StandardAnalyzer.cpp
+++ b/src/core/CLucene/analysis/standard/StandardAnalyzer.cpp
@@ -60,7 +60,10 @@ CL_NS_DEF2(analysis,standard)
 
        StandardAnalyzer::~StandardAnalyzer(){
         SavedStreams* t = 
reinterpret_cast<SavedStreams*>(this->getPreviousTokenStream());
-        if (t) _CLDELETE(t->filteredTokenStream);
+        if (t) {
+            _CLDELETE(t->filteredTokenStream);
+            _CLDELETE(t);
+        }
                _CLLDELETE(stopSet);
        }
 
diff --git a/src/core/CLucene/analysis/standard/StandardAnalyzer.h 
b/src/core/CLucene/analysis/standard/StandardAnalyzer.h
index 85e476b..653ac85 100644
--- a/src/core/CLucene/analysis/standard/StandardAnalyzer.h
+++ b/src/core/CLucene/analysis/standard/StandardAnalyzer.h
@@ -22,6 +22,7 @@ CL_NS_DEF2(analysis,standard)
        private:
                CLTCSetList* stopSet;
         int32_t maxTokenLength;
+        TokenStream* prev_tokenStream{};
 
         class SavedStreams;
        public:
@@ -46,6 +47,13 @@ CL_NS_DEF2(analysis,standard)
 
                virtual ~StandardAnalyzer();
 
+
+        TokenStream* getPreviousTokenStream() override {
+            return prev_tokenStream;
+        }
+        void setPreviousTokenStream(TokenStream* obj) override {
+            prev_tokenStream = obj;
+        }
         /**
         * Constructs a StandardTokenizer filtered by a 
         * StandardFilter, a LowerCaseFilter and a StopFilter.


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org