This is an automated email from the ASF dual-hosted git repository.
liaoxin01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8bf231eeaba [fix](be) Skip file cache for http chunk-response readers
(#66932)
8bf231eeaba is described below
commit 8bf231eeabaaca91b09215a2b07fef80656932cb
Author: wudi <[email protected]>
AuthorDate: Thu Aug 27 11:40:15 2026 +0800
[fix](be) Skip file cache for http chunk-response readers (#66932)
### What problem does this PR solve?
Issue Number: None
Related PR: None
Problem Summary:
The file-block cache assumes that a reader exposes stable, random-access
content. HTTP readers used by CDC TVF chunk responses instead expose a
dynamic streaming response. When file cache is enabled, wrapping such a
reader with `CachedRemoteFileReader` can make the stream appear empty or
reuse invalid cached content.
This change adds a reader capability check and skips the file-cache
wrapper for HTTP chunk-response readers. Other HTTP file readers
continue using the existing cache path. A unit test covers the bypass
behavior.
---
be/src/io/file_factory.cpp | 13 ++++++++---
be/test/io/fs/http_file_reader_test.cpp | 41 +++++++++++++++++++++++++++++++++
2 files changed, 51 insertions(+), 3 deletions(-)
diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp
index cf1cfa443dc..0776b68a154 100644
--- a/be/src/io/file_factory.cpp
+++ b/be/src/io/file_factory.cpp
@@ -52,6 +52,7 @@
#include "service/backend_options.h"
#include "util/s3_uri.h"
#include "util/s3_util.h"
+#include "util/string_util.h"
#include "util/uid_util.h"
namespace doris {
@@ -271,10 +272,16 @@ Result<io::FileReaderSPtr>
FileFactory::_create_file_reader_internal(
});
}
case TFileType::FILE_HTTP: {
+ auto options = reader_options;
+ auto chunk_response =
system_properties.properties.find("http.enable.chunk.response");
+ if (chunk_response != system_properties.properties.end() &&
+ (iequal(chunk_response->second, "true") || chunk_response->second
== "1")) {
+ options.cache_type = io::FileCachePolicy::NO_CACHE;
+ }
return io::HttpFileReader::create(file_description.path,
system_properties.properties,
- reader_options, profile)
- .and_then([&](auto&& reader) {
- return io::create_cached_file_reader(std::move(reader),
reader_options);
+ options, profile)
+ .and_then([&options](auto&& reader) {
+ return io::create_cached_file_reader(std::move(reader),
options);
});
}
default:
diff --git a/be/test/io/fs/http_file_reader_test.cpp
b/be/test/io/fs/http_file_reader_test.cpp
new file mode 100644
index 00000000000..08241d5d5c5
--- /dev/null
+++ b/be/test/io/fs/http_file_reader_test.cpp
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "io/fs/http_file_reader.h"
+
+#include <gtest/gtest.h>
+
+#include "io/file_factory.h"
+
+namespace doris::io {
+
+TEST(HttpFileReaderTest, ChunkResponseDisablesFileCache) {
+ FileSystemProperties properties;
+ properties.system_type = TFileType::FILE_HTTP;
+ properties.properties = {{"http.enable.chunk.response", "true"}};
+ FileDescription file_description;
+ file_description.path = "http://127.0.0.1/stream";;
+ FileReaderOptions opts;
+ opts.cache_type = FileCachePolicy::FILE_BLOCK_CACHE;
+
+ auto reader = FileFactory::create_file_reader(properties,
file_description, opts);
+
+ ASSERT_TRUE(reader.has_value()) << reader.error();
+ EXPECT_NE(std::dynamic_pointer_cast<HttpFileReader>(reader.value()),
nullptr);
+}
+
+} // namespace doris::io
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
