[doris] branch branch-1.1-lts updated: [fix](jsonreader) teach jsonreader to release memory (#13336) (#13394)

[morningman]

This is an automated email from the ASF dual-hosted git repository.

morningman pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
     new abceed6f5d [fix](jsonreader) teach jsonreader to release memory 
(#13336) (#13394)
abceed6f5d is described below

commit abceed6f5d6fef9471415ad82f5b04a884074987
Author: Yongqiang YANG <98214048+dataroar...@users.noreply.github.com>
AuthorDate: Sun Oct 16 23:57:10 2022 +0800

    [fix](jsonreader) teach jsonreader to release memory (#13336) (#13394)
    
    Allocator of rapidjson does not release memory, this fix use allocator with 
local buffer and call Clear to release memory allocated beyond local buffer.
---
 be/src/exec/json_scanner.cpp |  5 +++++
 be/src/exec/json_scanner.h   | 10 +++++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/be/src/exec/json_scanner.cpp b/be/src/exec/json_scanner.cpp
index d651545806..f2273ffca4 100644
--- a/be/src/exec/json_scanner.cpp
+++ b/be/src/exec/json_scanner.cpp
@@ -296,6 +296,9 @@ JsonReader::JsonReader(RuntimeState* state, ScannerCounter* 
counter, RuntimeProf
           _strip_outer_array(strip_outer_array),
           _num_as_string(num_as_string),
           _fuzzy_parse(fuzzy_parse),
+          _value_allocator(_value_buffer, sizeof(_value_buffer)),
+          _parse_allocator(_parse_buffer, sizeof(_parse_buffer)),
+          _origin_json_doc(&_value_allocator, sizeof(_parse_buffer), 
&_parse_allocator),
           _json_doc(nullptr),
           _scanner_eof(scanner_eof) {
     _bytes_read_counter = ADD_COUNTER(_profile, "BytesRead", TUnit::BYTES);
@@ -386,6 +389,8 @@ Status JsonReader::_parse_json_doc(size_t* size, bool* eof) 
{
         return Status::OK();
     }
 
+    // clear memory here.
+    _origin_json_doc.GetAllocator().Clear();
     bool has_parse_error = false;
     // parse jsondata to JsonDoc
 
diff --git a/be/src/exec/json_scanner.h b/be/src/exec/json_scanner.h
index 1a489a5ed8..846d3a24a4 100644
--- a/be/src/exec/json_scanner.h
+++ b/be/src/exec/json_scanner.h
@@ -181,7 +181,15 @@ private:
     std::vector<std::vector<JsonPath>> _parsed_jsonpaths;
     std::vector<JsonPath> _parsed_json_root;
 
-    rapidjson::Document _origin_json_doc; // origin json document object from 
parsed json string
+    char _value_buffer[4 * 1024 * 1024];
+    char _parse_buffer[512 * 1024];
+
+    typedef rapidjson::GenericDocument<rapidjson::UTF8<>, 
rapidjson::MemoryPoolAllocator<>,
+                                       rapidjson::MemoryPoolAllocator<>>
+            Document;
+    rapidjson::MemoryPoolAllocator<> _value_allocator;
+    rapidjson::MemoryPoolAllocator<> _parse_allocator;
+    Document _origin_json_doc;   // origin json document object from parsed 
json string
     rapidjson::Value* _json_doc; // _json_doc equals _final_json_doc iff not 
set `json_root`
     std::unordered_map<std::string, int> _name_map;
     


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org