eldenmoon commented on code in PR #20037: URL: https://github.com/apache/doris/pull/20037#discussion_r1206150999
########## be/src/olap/resource_pool.h: ########## @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <fmt/core.h> +#include <parallel_hashmap/phmap.h> + +#include <algorithm> +#include <memory> +#include <mutex> +#include <vector> + +#include "olap/iterators.h" +#include "olap/olap_common.h" +#include "olap/schema.h" +#include "olap/tablet.h" +#include "olap/tablet_schema.h" + +namespace doris { + +namespace segment_v2 { +class Segment; +class SegmentIterator; +using SegmentIteratorUPtr = std::unique_ptr<SegmentIterator>; +} // namespace segment_v2 + +// The ResourcePool is utilized to cache pre-allocated data structures, +// eliminating the need for frequent allocation and deallocation during usage. +// This caching mechanism proves immensely advantageous, particularly in scenarios +// with high concurrency, where queries are executed simultaneously. +class ResourcePool { +public: + enum class Type { + SEGMENT_READ_SCHEMA = 0, + READ_SCHEMA = 1, + SEGMENT_ITERATOR = 2, + }; + + static ResourcePool* instance() { return _s_instance; } + + static void create_global_instance(size_t capacity); + + // Return the borrowed object back to cache, if reached capacity, return false. + // Do not touch iter if returned success. + static void return_iterator(RowwiseIteratorUPtr&& iter); + + // get cache schema key, delimiter with SCHEMA_DELIMITER + static std::string get_schema_key(int32_t tablet_id, const Schema& schema); + static std::string get_schema_key(int32_t tablet_id, const TabletSchemaSPtr& schema, + const std::vector<uint32_t>& column_ids); + static std::string get_schema_key(int32_t tablet_id, const std::vector<TColumn>& columns); + + // Get a cached segment iter, if not exist, then a new iterator will be constructed + // and put into cache when returned.Otherwise get the target iter and remove from map to prevent from + // multi threads access, since SegmentIterator is stateless and can't be shared. + SegmentIteratorUPtr borrow_segment_iter(int32_t tablet_id, SchemaSPtr schema, + std::shared_ptr<Segment> segment); + + // Get a shared cached Schema from resource pool, schema_key is a subset of column unique ids + SchemaSPtr get_shared_schema(const std::string& schema_key); + + // Get a shared cached tablet Schema from resource pool, schema_key is full column unique ids + TabletSchemaSPtr get_shared_tablet_schema(const std::string& schema_key); + + void insert_schema(const std::string& key, SchemaSPtr schema); + void insert_tablet_schema(const std::string& key, TabletSchemaSPtr schema); + + // Try to prune the cache if expired. + Status prune(); + +private: + ResourcePool(size_t capacity); + static constexpr char SCHEMA_DELIMITER = '-'; + static constexpr int SCHEMA_CAPACITY = 1024; Review Comment: seems reasonable -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
