This is an automated email from the ASF dual-hosted git repository.
cmcfarlen pushed a commit to branch 10.1.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/10.1.x by this push:
new b73a29d3d5 Try both old(9.2) and new cache key generation (#12271)
(#12283)
b73a29d3d5 is described below
commit b73a29d3d56448ecf7a0b3c957f5c8ea45d5f58f
Author: Chris McFarlen <[email protected]>
AuthorDate: Wed Jun 11 13:23:58 2025 -0500
Try both old(9.2) and new cache key generation (#12271) (#12283)
(cherry picked from commit 8fbd40122fc4818bf646d282d665da0cd1eebd87)
---
doc/admin-guide/files/records.yaml.en.rst | 13 ++++
include/proxy/hdrs/URL.h | 12 +++
include/proxy/http/HttpConfig.h | 3 +
include/proxy/http/HttpSM.h | 8 ++
src/iocore/cache/P_CacheInternal.h | 18 +++++
src/proxy/hdrs/URL.cc | 117 ++++++++++++++++++++++++++++++
src/proxy/http/HttpConfig.cc | 5 +-
src/proxy/http/HttpSM.cc | 19 ++++-
src/records/RecordsConfig.cc | 7 ++
9 files changed, 200 insertions(+), 2 deletions(-)
diff --git a/doc/admin-guide/files/records.yaml.en.rst
b/doc/admin-guide/files/records.yaml.en.rst
index 4d4a507b0d..edf7d1fa6d 100644
--- a/doc/admin-guide/files/records.yaml.en.rst
+++ b/doc/admin-guide/files/records.yaml.en.rst
@@ -2390,6 +2390,19 @@ Cache Control
Establishes a guaranteed maximum lifetime boundary for object freshness.
Setting this to ``0`` disables the feature.
+.. ts:cv:: CONFIG proxy.config.http.cache.try_compat_key_read INT 0
+ :reloadable:
+
+ When enabled (``1``), |TS| will try to lookup the cached object using the
+ previous cache key generation algorithm, but will always write new objects
+ using the newest key generation. This might be temporarily necessary
+ if a large cache was created by the previous version of ATS but the new
+ version changed the way cache keys are generated. If this is turned on,
+ a metric called `proxy.process.http.cache.compat_key_reads` will be
+ incremented any time the compat cache lookup successfully finds the object.
+ You can monitor this metric and know when its safe to turn this feature off
+ as the cache wraps around.
+
.. ts:cv:: CONFIG proxy.config.http.cache.range.lookup INT 1
:overridable:
diff --git a/include/proxy/hdrs/URL.h b/include/proxy/hdrs/URL.h
index e0143346df..f87780579d 100644
--- a/include/proxy/hdrs/URL.h
+++ b/include/proxy/hdrs/URL.h
@@ -217,6 +217,7 @@ void url_called_set(URLImpl *url);
char *url_string_get_buf(URLImpl *url, char *dstbuf, int dstbuf_size, int
*length);
void url_CryptoHash_get(const URLImpl *url, CryptoHash *hash, bool
ignore_query = false, cache_generation_t generation = -1);
+void url_CryptoHash_get_92(const URLImpl *url, CryptoHash *hash, bool
ignore_query = false, cache_generation_t generation = -1);
void url_host_CryptoHash_get(URLImpl *url, CryptoHash *hash);
constexpr bool USE_STRICT_URI_PARSING = true;
@@ -278,6 +279,7 @@ public:
char *string_get_ref(int *length = nullptr, unsigned normalization_flags =
URLNormalize::NONE) const;
char *string_get_buf(char *dstbuf, int dsbuf_size, int *length = nullptr)
const;
void hash_get(CryptoHash *hash, bool ignore_query = false,
cache_generation_t generation = -1) const;
+ void hash_get92(CryptoHash *hash, bool ignore_query = false,
cache_generation_t generation = -1) const;
void host_hash_get(CryptoHash *hash) const;
const char *scheme_get(int *length);
@@ -496,6 +498,16 @@ URL::hash_get(CryptoHash *hash, bool ignore_query,
cache_generation_t generation
url_CryptoHash_get(m_url_impl, hash, ignore_query, generation);
}
+/*-------------------------------------------------------------------------
+ -------------------------------------------------------------------------*/
+
+inline void
+URL::hash_get92(CryptoHash *hash, bool ignore_query, cache_generation_t
generation) const
+{
+ ink_assert(valid());
+ url_CryptoHash_get_92(m_url_impl, hash, ignore_query, generation);
+}
+
/*-------------------------------------------------------------------------
-------------------------------------------------------------------------*/
diff --git a/include/proxy/http/HttpConfig.h b/include/proxy/http/HttpConfig.h
index 63257470ac..25ef5576ed 100644
--- a/include/proxy/http/HttpConfig.h
+++ b/include/proxy/http/HttpConfig.h
@@ -329,6 +329,7 @@ struct HttpStatsBlock {
Metrics::Counter::AtomicType *origin_server_speed_bytes_per_sec_400M;
Metrics::Counter::AtomicType *origin_server_speed_bytes_per_sec_800M;
Metrics::Counter::AtomicType *origin_server_speed_bytes_per_sec_1G;
+ Metrics::Counter::AtomicType *cache_compat_key_reads;
};
enum CacheOpenWriteFailAction_t {
@@ -817,6 +818,8 @@ public:
MgmtByte http_host_sni_policy = 0;
MgmtByte scheme_proto_mismatch_policy = 2;
+ MgmtByte cache_try_compat_key_read = 0;
+
// noncopyable
/////////////////////////////////////
// operator = and copy constructor //
diff --git a/include/proxy/http/HttpSM.h b/include/proxy/http/HttpSM.h
index bf3fce0fe7..8f81440137 100644
--- a/include/proxy/http/HttpSM.h
+++ b/include/proxy/http/HttpSM.h
@@ -174,6 +174,12 @@ public:
~PostDataBuffers();
};
+enum class CompatibilityCacheLookup {
+ COMPAT_CACHE_LOOKUP_NORMAL = 0,
+ COMPAT_CACHE_LOOKUP_92,
+ COMPAT_CACHE_LAST,
+};
+
class HttpSM : public Continuation, public PluginUserArgs<TS_USER_ARGS_TXN>
{
friend class HttpTransact;
@@ -533,6 +539,8 @@ public:
const char *plugin_tag = nullptr;
int64_t plugin_id = 0;
+ CompatibilityCacheLookup compatibility_cache_lookup =
CompatibilityCacheLookup::COMPAT_CACHE_LOOKUP_NORMAL;
+
private:
HttpTunnel tunnel;
diff --git a/src/iocore/cache/P_CacheInternal.h
b/src/iocore/cache/P_CacheInternal.h
index bf324266de..7044c16ee3 100644
--- a/src/iocore/cache/P_CacheInternal.h
+++ b/src/iocore/cache/P_CacheInternal.h
@@ -470,6 +470,11 @@ struct Cache {
static void generate_key(CryptoHash *hash, CacheURL *url);
static void generate_key(HttpCacheKey *hash, CacheURL *url, bool
ignore_query = false, cache_generation_t generation = -1);
+ // These generate functions are used for backward compatibility with caches
created with ATS9.2
+ // see `proxy.config.http.cache.try_compat_key_read`
+ static void generate_key92(CryptoHash *hash, CacheURL *url);
+ static void generate_key92(HttpCacheKey *hash, CacheURL *url, bool
ignore_query = false, cache_generation_t generation = -1);
+
void vol_initialized(bool result);
int open_done();
@@ -495,6 +500,19 @@ Cache::generate_key(HttpCacheKey *key, CacheURL *url, bool
ignore_query, cache_g
url->hash_get(&key->hash, ignore_query, generation);
}
+inline void
+Cache::generate_key92(CryptoHash *hash, CacheURL *url)
+{
+ url->hash_get92(hash);
+}
+
+inline void
+Cache::generate_key92(HttpCacheKey *key, CacheURL *url, bool ignore_query,
cache_generation_t generation)
+{
+ key->hostname = url->host_get(&key->hostlen);
+ url->hash_get92(&key->hash, ignore_query, generation);
+}
+
inline unsigned int
cache_hash(const CryptoHash &hash)
{
diff --git a/src/proxy/hdrs/URL.cc b/src/proxy/hdrs/URL.cc
index 430d886f81..0fbc39ab40 100644
--- a/src/proxy/hdrs/URL.cc
+++ b/src/proxy/hdrs/URL.cc
@@ -1877,6 +1877,123 @@ url_CryptoHash_get(const URLImpl *url, CryptoHash
*hash, bool ignore_query, cach
}
}
+static inline void
+url_CryptoHash_get_general_92(const URLImpl *url, CryptoContext &ctx,
CryptoHash &hash, bool ignore_query,
+ cache_generation_t generation)
+{
+ char buffer[BUFSIZE];
+ char *p, *e;
+ const char *strs[13], *ends[13];
+ const char *t;
+ in_port_t port;
+ int i, s;
+
+ strs[0] = url->m_ptr_scheme;
+ strs[1] = "://";
+ strs[2] = url->m_ptr_user;
+ strs[3] = ":";
+ strs[4] = url->m_ptr_password;
+ strs[5] = "@";
+ strs[6] = url->m_ptr_host;
+ strs[7] = "/";
+ strs[8] = url->m_ptr_path;
+
+ ends[0] = strs[0] + url->m_len_scheme;
+ ends[1] = strs[1] + 3;
+ ends[2] = strs[2] + url->m_len_user;
+ ends[3] = strs[3] + 1;
+ ends[4] = strs[4] + url->m_len_password;
+ ends[5] = strs[5] + 1;
+ ends[6] = strs[6] + url->m_len_host;
+ ends[7] = strs[7] + 1;
+ ends[8] = strs[8] + url->m_len_path;
+
+ strs[9] = ";";
+ strs[10] = url->m_ptr_params;
+ strs[11] = "?";
+
+ // Special case for the query paramters, allowing us to ignore them if
requested
+ if (!ignore_query) {
+ strs[12] = url->m_ptr_query;
+ ends[12] = strs[12] + url->m_len_query;
+ } else {
+ strs[12] = nullptr;
+ ends[12] = nullptr;
+ }
+
+ ends[9] = strs[9] + 1;
+ ends[10] = strs[10] + url->m_len_params;
+ ends[11] = strs[11] + 1;
+
+ p = buffer;
+ e = buffer + BUFSIZE;
+
+ for (i = 0; i < 13; i++) {
+ if (strs[i]) {
+ t = strs[i];
+ s = 0;
+
+ while (t < ends[i]) {
+ if ((i == 0) || (i == 6)) { // scheme and host
+ unescape_str_tolower(p, e, t, ends[i], s);
+ } else if (i == 8 || i == 10 || i == 12) { // path, params, query
+ // Don't unescape the parts of the URI that are processed by the
+ // origin since it may behave differently based upon whether these
are
+ // escaped or not. Therefore differently encoded strings should be
+ // cached separately via differentiated hashes.
+ int path_len = ends[i] - t;
+ int min_len = std::min(path_len, static_cast<int>(e - p));
+ memcpy(p, t, min_len);
+ p += min_len;
+ t += min_len;
+ } else {
+ unescape_str(p, e, t, ends[i], s);
+ }
+
+ if (p == e) {
+ ctx.update(buffer, BUFSIZE);
+ p = buffer;
+ }
+ }
+ }
+ }
+
+ if (p != buffer) {
+ ctx.update(buffer, p - buffer);
+ }
+ int buffer_len = static_cast<int>(p - buffer);
+ port = url_canonicalize_port(url->m_url_type, url->m_port);
+
+ ctx.update(&port, sizeof(port));
+ if (generation != -1) {
+ ctx.update(&generation, sizeof(generation));
+ Dbg(dbg_ctl_url_cachekey, "Final url string for cache hash key %.*s%d%d",
buffer_len, buffer, port,
+ static_cast<int>(generation));
+ } else {
+ Dbg(dbg_ctl_url_cachekey, "Final url string for cache hash key %.*s%d",
buffer_len, buffer, port);
+ }
+ ctx.finalize(hash);
+}
+
+void
+url_CryptoHash_get_92(const URLImpl *url, CryptoHash *hash, bool ignore_query,
cache_generation_t generation)
+{
+ URLHashContext ctx;
+ if ((url_hash_method != 0) && (url->m_url_type == URL_TYPE_HTTP) &&
+ ((url->m_len_user + url->m_len_password + url->m_len_params +
(ignore_query ? 0 : url->m_len_query)) == 0) &&
+ (3 + 1 + 1 + 1 + 1 + 1 + 2 + url->m_len_scheme + url->m_len_host +
url->m_len_path < BUFSIZE) &&
+ (memchr(url->m_ptr_host, '%', url->m_len_host) == nullptr) &&
(memchr(url->m_ptr_path, '%', url->m_len_path) == nullptr)) {
+ url_CryptoHash_get_fast(url, ctx, hash, generation);
+#ifdef DEBUG
+ CryptoHash hash_general;
+ url_CryptoHash_get_general_92(url, ctx, hash_general, ignore_query,
generation);
+ ink_assert(*hash == hash_general);
+#endif
+ } else {
+ url_CryptoHash_get_general_92(url, ctx, *hash, ignore_query, generation);
+ }
+}
+
#undef BUFSIZE
/*-------------------------------------------------------------------------
diff --git a/src/proxy/http/HttpConfig.cc b/src/proxy/http/HttpConfig.cc
index 0d6e6e87b9..0649be5b23 100644
--- a/src/proxy/http/HttpConfig.cc
+++ b/src/proxy/http/HttpConfig.cc
@@ -591,6 +591,7 @@ register_stat_callbacks()
Metrics::Counter::createPtr("proxy.process.http.origin_server_speed_bytes_per_sec_800M");
http_rsb.origin_server_speed_bytes_per_sec_1G =
Metrics::Counter::createPtr("proxy.process.http.origin_server_speed_bytes_per_sec_1G");
+ http_rsb.cache_compat_key_reads =
Metrics::Counter::createPtr("proxy.process.http.cache.compat_key_reads");
Metrics::Derived::derive({
// Total bytes of client request body + headers
@@ -1108,7 +1109,7 @@ HttpConfig::startup()
HttpEstablishStaticConfigLongLong(c.post_copy_size,
"proxy.config.http.post_copy_size");
HttpEstablishStaticConfigStringAlloc(c.redirect_actions_string,
"proxy.config.http.redirect.actions");
HttpEstablishStaticConfigByte(c.http_host_sni_policy,
"proxy.config.http.host_sni_policy");
-
+ HttpEstablishStaticConfigByte(c.cache_try_compat_key_read,
"proxy.config.http.cache.try_compat_key_read");
HttpEstablishStaticConfigStringAlloc(c.oride.ssl_client_sni_policy,
"proxy.config.ssl.client.sni_policy");
HttpEstablishStaticConfigStringAlloc(c.oride.ssl_client_alpn_protocols,
"proxy.config.ssl.client.alpn_protocols");
HttpEstablishStaticConfigByte(c.scheme_proto_mismatch_policy,
"proxy.config.ssl.client.scheme_proto_mismatch_policy");
@@ -1409,6 +1410,8 @@ HttpConfig::reconfigure()
params->oride.plugin_vc_default_buffer_index =
m_master.oride.plugin_vc_default_buffer_index;
params->oride.plugin_vc_default_buffer_water_mark =
m_master.oride.plugin_vc_default_buffer_water_mark;
+ params->cache_try_compat_key_read = m_master.cache_try_compat_key_read;
+
m_id = configProcessor.set(m_id, params);
}
diff --git a/src/proxy/http/HttpSM.cc b/src/proxy/http/HttpSM.cc
index e24fbadaf5..12b35f3307 100644
--- a/src/proxy/http/HttpSM.cc
+++ b/src/proxy/http/HttpSM.cc
@@ -22,6 +22,8 @@
*/
+#include "proxy/http/HttpConfig.h"
+#include "tsutil/Metrics.h"
#include "tsutil/ts_bw_format.h"
#include "proxy/ProxyTransaction.h"
#include "proxy/http/HttpSM.h"
@@ -2587,6 +2589,10 @@ HttpSM::state_cache_open_read(int event, void *data)
t_state.cache_info.hit_miss_code = SQUID_HIT_DISK;
}
+ if (compatibility_cache_lookup ==
CompatibilityCacheLookup::COMPAT_CACHE_LOOKUP_92) {
+ Metrics::Counter::increment(http_rsb.cache_compat_key_reads);
+ }
+
ink_assert(t_state.cache_info.object_read != nullptr);
call_transact_and_set_next_state(HttpTransact::HandleCacheOpenRead);
break;
@@ -2603,6 +2609,13 @@ HttpSM::state_cache_open_read(int event, void *data)
if (cache_sm.get_last_error() == -ECACHE_DOC_BUSY) {
t_state.cache_lookup_result = HttpTransact::CACHE_LOOKUP_DOC_BUSY;
} else {
+ if (t_state.http_config_param->cache_try_compat_key_read &&
+ compatibility_cache_lookup ==
CompatibilityCacheLookup::COMPAT_CACHE_LOOKUP_NORMAL) {
+ // do the retry
+ compatibility_cache_lookup =
CompatibilityCacheLookup::COMPAT_CACHE_LOOKUP_92;
+ do_cache_lookup_and_read();
+ return 0;
+ }
t_state.cache_lookup_result = HttpTransact::CACHE_LOOKUP_MISS;
}
@@ -5000,7 +5013,11 @@ HttpSM::do_cache_lookup_and_read()
SMDbg(dbg_ctl_http_seq, "Issuing cache lookup for URL %s",
c_url->string_get(&t_state.arena));
HttpCacheKey key;
- Cache::generate_key(&key, c_url, t_state.txn_conf->cache_ignore_query,
t_state.txn_conf->cache_generation_number);
+ if (compatibility_cache_lookup ==
CompatibilityCacheLookup::COMPAT_CACHE_LOOKUP_92) {
+ Cache::generate_key92(&key, c_url, t_state.txn_conf->cache_ignore_query,
t_state.txn_conf->cache_generation_number);
+ } else {
+ Cache::generate_key(&key, c_url, t_state.txn_conf->cache_ignore_query,
t_state.txn_conf->cache_generation_number);
+ }
t_state.hdr_info.cache_request.copy(&t_state.hdr_info.client_request);
HttpTransactHeaders::normalize_accept_encoding(t_state.txn_conf,
&t_state.hdr_info.cache_request);
diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc
index 0022e1c8df..91c0fa5161 100644
--- a/src/records/RecordsConfig.cc
+++ b/src/records/RecordsConfig.cc
@@ -638,6 +638,13 @@ static const RecordElement RecordsConfig[] =
{RECT_CONFIG, "proxy.config.http.cache.guaranteed_max_lifetime", RECD_INT,
"31536000", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
,
+ // ###################
+ // # Cache Compat #
+ // ###################
+ {RECT_CONFIG, "proxy.config.http.cache.try_compat_key_read", RECD_INT, "0",
RECU_NULL, RR_NULL, RECC_INT, "[0-1]", RECA_NULL}
+ ,
+
+
// ###################
// # Error Reporting #
// ###################
