This is an automated email from the ASF dual-hosted git repository. cmcfarlen pushed a commit to branch 10.1.x in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit dc9e0ac849fa37116779508ee8530dd88564816a Author: Kit Chan <[email protected]> AuthorDate: Mon Jun 16 17:59:57 2025 -0700 Add max inclusion depth support to esi plugin (#12295) * Add max inclusion depth support to esi plugin * fix formatting (cherry picked from commit 7f178de7de19498c1c320ea9b62c2f32355f3893) --- doc/admin-guide/plugins/esi.en.rst | 4 +- plugins/esi/esi.cc | 123 ++++++++++++++----- .../pluginTest/esi/esi_nested_include.test.py | 136 +++++++++++++++++++++ .../pluginTest/esi/gold/nested_include_body.gold | 12 ++ 4 files changed, 242 insertions(+), 33 deletions(-) diff --git a/doc/admin-guide/plugins/esi.en.rst b/doc/admin-guide/plugins/esi.en.rst index f15406c81f..21855d4a40 100644 --- a/doc/admin-guide/plugins/esi.en.rst +++ b/doc/admin-guide/plugins/esi.en.rst @@ -76,7 +76,7 @@ Enabling ESI esi.so -2. There are four optional arguments that can be passed to the above ``esi.so`` entry: +2. There are optional arguments that can be passed to the above ``esi.so`` entry: - ``--private-response`` will add private cache control and expires headers to the processed ESI document. - ``--packed-node-support`` will enable the support for using the packed node feature, which will improve the @@ -90,6 +90,8 @@ Enabling ESI in bytes. The number of bytes must be an unsigned decimal integer, and can be followed (with no white space) by a K, to indicate the given number is multiplied by 1024, or by M, to indicate the given number is multiplied by 1024 * 1024. Example values: 500, 5K, 2M. If this option is omitted, the maximum document size defaults to 1M. +- ``--max-inclusion-depth <max-depth>`` controls the maximum depth of recursive ESI inclusion allowed (between 0 and 9). + Default is 3. 3. ``HTTP_COOKIE`` variable support is turned off by default. It can be turned on with ``-f <handler_config>`` or ``-handler <handler_config>``. For example: diff --git a/plugins/esi/esi.cc b/plugins/esi/esi.cc index 869c896eb5..229f68dac0 100644 --- a/plugins/esi/esi.cc +++ b/plugins/esi/esi.cc @@ -58,6 +58,7 @@ struct OptionInfo { bool disable_gzip_output{false}; bool first_byte_flush{false}; unsigned max_doc_size{1024 * 1024}; + unsigned max_inclusion_depth{3}; }; static HandlerManager *gHandlerManager = nullptr; @@ -69,6 +70,9 @@ static Utils::HeaderValueList gAllowlistCookies; #define MIME_FIELD_XESI "X-Esi" #define MIME_FIELD_XESI_LEN 5 +#define MIME_FIELD_XESIDEPTH "X-Esi-Depth" +#define MIME_FIELD_XESIDEPTH_LEN 11 + #define HTTP_VALUE_PRIVATE_EXPIRES "-1" #define HTTP_VALUE_PRIVATE_CC "max-age=0, private" @@ -298,7 +302,9 @@ ContData::getClientState() } TSHandleMLocRelease(bufp, req_hdr_loc, url_loc); } - TSMLoc field_loc = TSMimeHdrFieldGet(req_bufp, req_hdr_loc, 0); + + TSMLoc field_loc = TSMimeHdrFieldGet(req_bufp, req_hdr_loc, 0); + bool depth_field = false; while (field_loc) { TSMLoc next_field_loc; const char *name; @@ -306,38 +312,56 @@ ContData::getClientState() name = TSMimeHdrFieldNameGet(req_bufp, req_hdr_loc, field_loc, &name_len); if (name) { - int n_values; - n_values = TSMimeHdrFieldValuesCount(req_bufp, req_hdr_loc, field_loc); - if (n_values && (n_values != TS_ERROR)) { - const char *value = nullptr; - int value_len = 0; - if (n_values == 1) { - value = TSMimeHdrFieldValueStringGet(req_bufp, req_hdr_loc, field_loc, 0, &value_len); - - if (nullptr != value && value_len) { - if (Utils::areEqual(name, name_len, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING) && - Utils::areEqual(value, value_len, TS_HTTP_VALUE_GZIP, TS_HTTP_LEN_GZIP)) { - gzip_output = true; - } - } + if (Utils::areEqual(name, name_len, MIME_FIELD_XESIDEPTH, MIME_FIELD_XESIDEPTH_LEN)) { + unsigned d = TSMimeHdrFieldValueUintGet(req_bufp, req_hdr_loc, field_loc, -1); + d = (d + 1) % 10; + char dstr[2]; + int const len = snprintf(dstr, sizeof(dstr), "%u", d); + + HttpHeader header; + if (len != 1) { + header = HttpHeader(MIME_FIELD_XESIDEPTH, MIME_FIELD_XESIDEPTH_LEN, "1", 1); } else { - for (int i = 0; i < n_values; ++i) { - value = TSMimeHdrFieldValueStringGet(req_bufp, req_hdr_loc, field_loc, i, &value_len); + header = HttpHeader(MIME_FIELD_XESIDEPTH, MIME_FIELD_XESIDEPTH_LEN, dstr, 1); + } + data_fetcher->useHeader(header); + esi_vars->populate(header); + depth_field = true; + + } else { + int n_values; + n_values = TSMimeHdrFieldValuesCount(req_bufp, req_hdr_loc, field_loc); + if (n_values && (n_values != TS_ERROR)) { + const char *value = nullptr; + int value_len = 0; + if (n_values == 1) { + value = TSMimeHdrFieldValueStringGet(req_bufp, req_hdr_loc, field_loc, 0, &value_len); + if (nullptr != value && value_len) { if (Utils::areEqual(name, name_len, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING) && Utils::areEqual(value, value_len, TS_HTTP_VALUE_GZIP, TS_HTTP_LEN_GZIP)) { gzip_output = true; } } - } + } else { + for (int i = 0; i < n_values; ++i) { + value = TSMimeHdrFieldValueStringGet(req_bufp, req_hdr_loc, field_loc, i, &value_len); + if (nullptr != value && value_len) { + if (Utils::areEqual(name, name_len, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING) && + Utils::areEqual(value, value_len, TS_HTTP_VALUE_GZIP, TS_HTTP_LEN_GZIP)) { + gzip_output = true; + } + } + } - value = TSMimeHdrFieldValueStringGet(req_bufp, req_hdr_loc, field_loc, -1, &value_len); - } + value = TSMimeHdrFieldValueStringGet(req_bufp, req_hdr_loc, field_loc, -1, &value_len); + } - if (value != nullptr) { - HttpHeader header(name, name_len, value, value_len); - data_fetcher->useHeader(header); - esi_vars->populate(header); + if (value != nullptr) { + HttpHeader header(name, name_len, value, value_len); + data_fetcher->useHeader(header); + esi_vars->populate(header); + } } } } @@ -346,6 +370,12 @@ ContData::getClientState() TSHandleMLocRelease(req_bufp, req_hdr_loc, field_loc); field_loc = next_field_loc; } + + if (depth_field == false) { + HttpHeader header(MIME_FIELD_XESIDEPTH, MIME_FIELD_XESIDEPTH_LEN, "1", 1); + data_fetcher->useHeader(header); + esi_vars->populate(header); + } } if (gzip_output) { @@ -1229,7 +1259,7 @@ maskOsCacheHeaders(TSHttpTxn txnp) } static bool -isTxnTransformable(TSHttpTxn txnp, bool is_cache_txn, bool *intercept_header, bool *head_only) +isTxnTransformable(TSHttpTxn txnp, bool is_cache_txn, const OptionInfo *pOptionInfo, bool *intercept_header, bool *head_only) { // We are only interested in transforming "200 OK" responses with a // Content-Type: text/ header and with X-Esi header @@ -1244,6 +1274,21 @@ isTxnTransformable(TSHttpTxn txnp, bool is_cache_txn, bool *intercept_header, bo return false; } + TSMLoc loc; + unsigned d; + + d = 0; + loc = TSMimeHdrFieldFind(bufp, hdr_loc, MIME_FIELD_XESIDEPTH, MIME_FIELD_XESIDEPTH_LEN); + if (loc != TS_NULL_MLOC) { + d = TSMimeHdrFieldValueUintGet(bufp, hdr_loc, loc, -1); + } + TSHandleMLocRelease(bufp, hdr_loc, loc); + if (d >= pOptionInfo->max_inclusion_depth) { + TSError("[esi][%s] The current esi inclusion depth (%u) is larger than or equal to the max (%u)", __FUNCTION__, d, + pOptionInfo->max_inclusion_depth); + return false; + } + int method_len; const char *method; method = TSHttpHdrMethodGet(bufp, hdr_loc, &method_len); @@ -1318,7 +1363,7 @@ isTxnTransformable(TSHttpTxn txnp, bool is_cache_txn, bool *intercept_header, bo } static bool -isCacheObjTransformable(TSHttpTxn txnp, bool *intercept_header, bool *head_only) +isCacheObjTransformable(TSHttpTxn txnp, const OptionInfo *pOptionInfo, bool *intercept_header, bool *head_only) { int obj_status; if (TSHttpTxnCacheLookupStatusGet(txnp, &obj_status) == TS_ERROR) { @@ -1327,7 +1372,7 @@ isCacheObjTransformable(TSHttpTxn txnp, bool *intercept_header, bool *head_only) } if (obj_status == TS_CACHE_LOOKUP_HIT_FRESH) { Dbg(dbg_ctl_local, "[%s] doc found in cache, will add transformation", __FUNCTION__); - return isTxnTransformable(txnp, true, intercept_header, head_only); + return isTxnTransformable(txnp, true, pOptionInfo, intercept_header, head_only); } Dbg(dbg_ctl_local, "[%s] cache object's status is %d; not transformable", __FUNCTION__, obj_status); return false; @@ -1499,7 +1544,7 @@ globalHookHandler(TSCont contp, TSEvent event, void *edata) if (event == TS_EVENT_HTTP_READ_RESPONSE_HDR) { bool mask_cache_headers = false; Dbg(dbg_ctl_local, "[%s] handling read response header event", __FUNCTION__); - if (isTxnTransformable(txnp, false, &intercept_header, &head_only)) { + if (isTxnTransformable(txnp, false, pOptionInfo, &intercept_header, &head_only)) { addTransform(txnp, true, intercept_header, head_only, pOptionInfo); Stats::increment(Stats::N_OS_DOCS); mask_cache_headers = true; @@ -1512,7 +1557,7 @@ globalHookHandler(TSCont contp, TSEvent event, void *edata) } } else { Dbg(dbg_ctl_local, "[%s] handling cache lookup complete event", __FUNCTION__); - if (isCacheObjTransformable(txnp, &intercept_header, &head_only)) { + if (isCacheObjTransformable(txnp, pOptionInfo, &intercept_header, &head_only)) { // we make the assumption above that a transformable cache // object would already have a transformation. We should revisit // that assumption in case we change the statement below @@ -1575,11 +1620,12 @@ esiPluginInit(int argc, const char *argv[], OptionInfo *pOptionInfo) {const_cast<char *>("first-byte-flush"), no_argument, nullptr, 'b'}, {const_cast<char *>("handler-filename"), required_argument, nullptr, 'f'}, {const_cast<char *>("max-doc-size"), required_argument, nullptr, 'd'}, + {const_cast<char *>("max-inclusion-depth"), required_argument, nullptr, 'i'}, {nullptr, 0, nullptr, 0 }, }; int longindex = 0; - while ((c = getopt_long(argc, const_cast<char *const *>(argv), "npzbf:d:", longopts, &longindex)) != -1) { + while ((c = getopt_long(argc, const_cast<char *const *>(argv), "npzbf:d:i:", longopts, &longindex)) != -1) { switch (c) { case 'n': pOptionInfo->packed_node_support = true; @@ -1621,6 +1667,18 @@ esiPluginInit(int argc, const char *argv[], OptionInfo *pOptionInfo) pOptionInfo->max_doc_size = max * coeff; break; } + case 'i': { + unsigned max; + auto num = std::sscanf(optarg, "%u", &max); + if (num != 1) { + TSEmergency("[esi][%s] value for maximum inclusion depth (%s) is not unsigned integer", __FUNCTION__, optarg); + } + if (max > 9) { + TSEmergency("[esi][%s] maximum inclusion depth (%s) large than 9", __FUNCTION__, optarg); + } + pOptionInfo->max_inclusion_depth = max; + break; + } default: TSEmergency("[esi][%s] bad option", __FUNCTION__); return -1; @@ -1630,9 +1688,10 @@ esiPluginInit(int argc, const char *argv[], OptionInfo *pOptionInfo) Dbg(dbg_ctl_local, "[%s] Plugin started, " - "packed-node-support: %d, private-response: %d, disable-gzip-output: %d, first-byte-flush: %d, max-doc-size %u ", + "packed-node-support: %d, private-response: %d, disable-gzip-output: %d, first-byte-flush: %d, max-doc-size %u, " + "max-inclusion-depth %u ", __FUNCTION__, pOptionInfo->packed_node_support, pOptionInfo->private_response, pOptionInfo->disable_gzip_output, - pOptionInfo->first_byte_flush, pOptionInfo->max_doc_size); + pOptionInfo->first_byte_flush, pOptionInfo->max_doc_size, pOptionInfo->max_inclusion_depth); return 0; } diff --git a/tests/gold_tests/pluginTest/esi/esi_nested_include.test.py b/tests/gold_tests/pluginTest/esi/esi_nested_include.test.py new file mode 100644 index 0000000000..906b29e529 --- /dev/null +++ b/tests/gold_tests/pluginTest/esi/esi_nested_include.test.py @@ -0,0 +1,136 @@ +''' +Test nested include for the ESI plugin. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +Test nested include for the ESI plugin. +''' + +Test.SkipUnless(Condition.PluginExists('esi.so'),) + + +class EsiTest(): + """ + A class that encapsulates the configuration and execution of a set of ESI + test cases. + """ + """ static: The same server Process is used across all tests. """ + _server = None + """ static: A counter to keep the ATS process names unique across tests. """ + _ts_counter = 0 + """ static: A counter to keep any output file names unique across tests. """ + _output_counter = 0 + """ The ATS process for this set of test cases. """ + _ts = None + + def __init__(self, plugin_config): + """ + Args: + plugin_config (str): The config line to place in plugin.config for + the ATS process. + """ + if EsiTest._server is None: + EsiTest._server = EsiTest._create_server() + + self._ts = EsiTest._create_ats(self, plugin_config) + + @staticmethod + def _create_server(): + """ + Create and start a server process. + """ + # Configure our server. + server = Test.MakeOriginServer("server", lookup_key="{%uuid}") + + # Generate the set of ESI responses. + request_header = { + "headers": "GET /esi-nested-include.php HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Content-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", + "body": "" + } + esi_body = r'''<p> +<esi:include src="http://www.example.com/esi-nested-include.html"/> +</p> +''' + response_header = { + "headers": + "HTTP/1.1 200 OK\r\n" + "X-Esi: 1\r\n" + "Cache-Control: private\r\n" + "Content-Type: text/html\r\n" + + "Connection: close\r\n" + "Content-Length: {}\r\n".format(len(esi_body)) + "\r\n", + "timestamp": "1469733493.993", + "body": esi_body + } + server.addResponse("sessionfile.log", request_header, response_header) + + # Create a run to start the server. + tr = Test.AddTestRun("Start the server.") + tr.Processes.Default.StartBefore(server) + tr.Processes.Default.Command = "echo starting the server" + tr.Processes.Default.ReturnCode = 0 + tr.StillRunningAfter = server + + return server + + @staticmethod + def _create_ats(self, plugin_config): + """ + Create and start an ATS process. + """ + EsiTest._ts_counter += 1 + + # Configure ATS with a vanilla ESI plugin configuration. + ts = Test.MakeATSProcess("ts{}".format(EsiTest._ts_counter)) + ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|plugin_esi', + }) + ts.Disk.remap_config.AddLine(f'map http://www.example.com/ http://127.0.0.1:{EsiTest._server.Variables.Port}') + ts.Disk.plugin_config.AddLine(plugin_config) + + ts.Disk.diags_log.Content = Testers.ContainsExpression( + r'The current esi inclusion depth \(3\) is larger than or equal to the max \(3\)', + 'Verify the ESI error concerning the max inclusion depth') + + # Create a run to start the ATS process. + tr = Test.AddTestRun("Start the ATS process.") + tr.Processes.Default.StartBefore(ts) + tr.Processes.Default.Command = "echo starting ATS" + tr.Processes.Default.ReturnCode = 0 + tr.StillRunningAfter = ts + return ts + + def run_test(self): + # Test 1: Verify basic ESI functionality without processing internal txn. + tr = Test.AddTestRun("First request") + tr.MakeCurlCommand( + f'http://127.0.0.1:{self._ts.Variables.port}/main.php -H"Host: www.example.com" ' + '-H"Accept: */*" --verbose') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = "gold/nested_include_body.gold" + tr.StillRunningAfter = self._server + tr.StillRunningAfter = self._ts + + +# +# Configure and run the test cases. +# + +# Run the tests with ESI configured with private response. +first_test = EsiTest(plugin_config='esi.so') +first_test.run_test() diff --git a/tests/gold_tests/pluginTest/esi/gold/nested_include_body.gold b/tests/gold_tests/pluginTest/esi/gold/nested_include_body.gold new file mode 100644 index 0000000000..84e0e4c699 --- /dev/null +++ b/tests/gold_tests/pluginTest/esi/gold/nested_include_body.gold @@ -0,0 +1,12 @@ +<p> +<p> +<p> +<p> +<esi:include src="http://www.example.com/esi-nested-include.html"/> +</p> + +</p> + +</p> + +</p>
