This is an automated email from the ASF dual-hosted git repository. markt pushed a commit to branch 11.0.x in repository https://gitbox.apache.org/repos/asf/tomcat.git
The following commit(s) were added to refs/heads/11.0.x by this push: new c899ae319d Fix BZ 69466 - rework handling of HEAD requests c899ae319d is described below commit c899ae319d0e1c307b648d9affeaf2b622739be3 Author: Mark Thomas <ma...@apache.org> AuthorDate: Thu Nov 21 16:14:49 2024 +0000 Fix BZ 69466 - rework handling of HEAD requests Headers explicitly set by users will not be removed and any header present in a HEAD request will also be present in the equivalent GET request. There may be some headers, as per RFC 9110, section 9.3.2, that are present in a GET request that are not present in the equivalent HEAD request. https://bz.apache.org/bugzilla/show_bug.cgi?id=69466 --- .../apache/catalina/connector/OutputBuffer.java | 11 +- java/org/apache/coyote/http11/Http11Processor.java | 18 +- java/org/apache/coyote/http2/StreamProcessor.java | 7 - .../servlet/http/HttpServletDoHeadBaseTest.java | 113 +++++++------ test/jakarta/servlet/http/TestHttpServlet.java | 37 ++--- test/jakarta/servlet/http/TesterConstants.java | 23 +++ .../coyote/http11/TestHttp11ProcessorDoHead.java | 182 +++++++++++++++++++++ webapps/docs/changelog.xml | 7 + 8 files changed, 302 insertions(+), 96 deletions(-) diff --git a/java/org/apache/catalina/connector/OutputBuffer.java b/java/org/apache/catalina/connector/OutputBuffer.java index 04cf82e632..a1adae91aa 100644 --- a/java/org/apache/catalina/connector/OutputBuffer.java +++ b/java/org/apache/catalina/connector/OutputBuffer.java @@ -220,9 +220,14 @@ public class OutputBuffer extends Writer { flushCharBuffer(); } - if ((!coyoteResponse.isCommitted()) && (coyoteResponse.getContentLengthLong() == -1)) { - // If this didn't cause a commit of the response, the final content - // length can be calculated. + // Content length can be calculated if: + // - the response has not been committed + // AND + // - the content length has not been explicitly set + // AND + // - some content has been written OR this is NOT a HEAD request + if ((!coyoteResponse.isCommitted()) && (coyoteResponse.getContentLengthLong() == -1) && + ((bb.remaining() > 0 || !coyoteResponse.getRequest().method().equals("HEAD")))) { coyoteResponse.setContentLength(bb.remaining()); } diff --git a/java/org/apache/coyote/http11/Http11Processor.java b/java/org/apache/coyote/http11/Http11Processor.java index 53b38d8c57..afdef54424 100644 --- a/java/org/apache/coyote/http11/Http11Processor.java +++ b/java/org/apache/coyote/http11/Http11Processor.java @@ -895,7 +895,7 @@ public class Http11Processor extends AbstractProcessor { boolean head = request.method().equals("HEAD"); if (head) { - // No entity body + // Any entity body, if present, should not be sent outputBuffer.addActiveFilter(outputFilters[Constants.VOID_FILTER]); contentDelimitation = true; } @@ -935,6 +935,13 @@ public class Http11Processor extends AbstractProcessor { headers.setValue("Content-Length").setLong(contentLength); outputBuffer.addActiveFilter(outputFilters[Constants.IDENTITY_FILTER]); contentDelimitation = true; + } else if (head) { + /* + * The OutputBuffer can't differentiate between a zero length response because GET would have produced a + * zero length body and a zero length response because HEAD didn't write any content. Therefore skip setting + * the "transfer-encoding: chunked" header as that may not be consistent with what would be seen with the + * equivalent GET. + */ } else { // If the response code supports an entity body and we're on // HTTP 1.1 then we chunk unless we have a Connection: close header @@ -1034,15 +1041,6 @@ public class Http11Processor extends AbstractProcessor { headers.setValue("Server").setString(server); } - // Exclude some HTTP header fields where the value is determined only - // while generating the content as per section 9.3.2 of RFC 9110. - if (head) { - headers.removeHeader("content-length"); - headers.removeHeader("content-range"); - headers.removeHeader("trailer"); - headers.removeHeader("transfer-encoding"); - } - writeHeaders(response.getStatus(), headers); outputBuffer.commit(); diff --git a/java/org/apache/coyote/http2/StreamProcessor.java b/java/org/apache/coyote/http2/StreamProcessor.java index daac6059b6..a853a5dd8e 100644 --- a/java/org/apache/coyote/http2/StreamProcessor.java +++ b/java/org/apache/coyote/http2/StreamProcessor.java @@ -255,13 +255,6 @@ class StreamProcessor extends AbstractProcessor { headers.setValue("Server").setString(server); } } - - // Exclude some HTTP header fields where the value is determined only - // while generating the content as per section 9.3.2 of RFC 9110. - if (coyoteRequest != null && coyoteRequest.method().equals("HEAD")) { - headers.removeHeader("content-length"); - headers.removeHeader("content-range"); - } } diff --git a/test/jakarta/servlet/http/HttpServletDoHeadBaseTest.java b/test/jakarta/servlet/http/HttpServletDoHeadBaseTest.java index 32d2ccf6c4..bfd59f21a9 100644 --- a/test/jakarta/servlet/http/HttpServletDoHeadBaseTest.java +++ b/test/jakarta/servlet/http/HttpServletDoHeadBaseTest.java @@ -22,7 +22,6 @@ import java.io.PrintWriter; import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -55,11 +54,12 @@ public class HttpServletDoHeadBaseTest extends Http2TestBase { private static final String VALID = "** valid data **"; private static final String INVALID = "* invalid data *"; - protected static final Integer BUFFERS[] = new Integer[] { Integer.valueOf (16), Integer.valueOf(8 * 1024), Integer.valueOf(16 * 1024) }; + protected static final Integer BUFFERS[] = + new Integer[] { Integer.valueOf(16), Integer.valueOf(8 * 1024), Integer.valueOf(16 * 1024) }; - protected static final Integer COUNTS[] = new Integer[] { Integer.valueOf(0), Integer.valueOf(1), - Integer.valueOf(511), Integer.valueOf(512), Integer.valueOf(513), - Integer.valueOf(1023), Integer.valueOf(1024), Integer.valueOf(1025) }; + protected static final Integer COUNTS[] = + new Integer[] { Integer.valueOf(0), Integer.valueOf(1), Integer.valueOf(511), Integer.valueOf(512), + Integer.valueOf(513), Integer.valueOf(1023), Integer.valueOf(1024), Integer.valueOf(1025) }; @Parameter(2) public boolean useLegacy; @@ -90,26 +90,26 @@ public class HttpServletDoHeadBaseTest extends Http2TestBase { Assert.assertEquals(HttpServletResponse.SC_OK, rc); out.recycle(); - Map<String,List<String>> headHeaders = new HashMap<>(); + Map<String,List<String>> headHeaders = new CaseInsensitiveKeyMap<>(); rc = headUrl(path, out, headHeaders); Assert.assertEquals(HttpServletResponse.SC_OK, rc); - // Headers should be the same part from: - // - Date header may be different - // Exclude some HTTP header fields where the value is determined only - // while generating the content as per section 9.3.2 of RFC 9110. - // (previously RFC 7231, section 4.3.2) - getHeaders.remove("content-length"); - getHeaders.remove("content-range"); - getHeaders.remove("trailer"); - getHeaders.remove("transfer-encoding"); + // Date header is likely to be different so just remove it from both GET and HEAD. + getHeaders.remove("date"); + headHeaders.remove("date"); + /* + * There are some headers that are optional for HEAD. See RFC 9110, section 9.3.2. If present, they must be the + * same for both GET and HEAD. If not present in HEAD, remove them from GET. + */ + for (String header : TesterConstants.OPTIONAL_HEADERS_WITH_HEAD) { + if (!headHeaders.containsKey(header)) { + getHeaders.remove(header); + } + } Assert.assertEquals(getHeaders.size(), headHeaders.size()); - for (Map.Entry<String, List<String>> getHeader : getHeaders.entrySet()) { + for (Map.Entry<String,List<String>> getHeader : getHeaders.entrySet()) { String headerName = getHeader.getKey(); - if ("date".equalsIgnoreCase(headerName)) { - continue; - } Assert.assertTrue(headerName, headHeaders.containsKey(headerName)); List<String> getValues = getHeader.getValue(); List<String> headValues = headHeaders.get(headerName); @@ -168,18 +168,30 @@ public class HttpServletDoHeadBaseTest extends Http2TestBase { int i = 0; int j = 0; for (; i < getHeaders.length; i++) { - // Exclude some HTTP header fields where the value is determined - // only while generating the content as per section 9.3.2 of RFC - // 9110. - if (getHeaders[i].contains("content-length") || getHeaders[i].contains("content-range") ) { - // Skip - } else { - // Headers should be the same, ignoring the first character which is the steam ID - Assert.assertEquals(getHeaders[i] + "\n" + traceGet + traceHead, '3', getHeaders[i].charAt(0)); - Assert.assertEquals(headHeaders[j] + "\n" + traceGet + traceHead, '5', headHeaders[j].charAt(0)); - Assert.assertEquals(traceGet + traceHead, getHeaders[i].substring(1), headHeaders[j].substring(1)); - j++; + /* + * There are some headers that are optional for HEAD. See RFC 9110, section 9.3.2. If present, they must + * be the same for both GET and HEAD. If not present in HEAD, ignore them in GET. + */ + boolean skip = false; + for (String optional : TesterConstants.OPTIONAL_HEADERS_WITH_HEAD) { + if (getHeaders[i].contains(optional)) { + if (!headHeaders[i].contains(optional)) { + // Skip it + skip = true; + } + // Found it. Don't need to check the remaining headers + break; + } } + if (skip) { + continue; + } + + // Remaining headers should be the same, ignoring the first character which is the steam ID + Assert.assertEquals(getHeaders[i] + "\n" + traceGet + traceHead, '3', getHeaders[i].charAt(0)); + Assert.assertEquals(headHeaders[j] + "\n" + traceGet + traceHead, '5', headHeaders[j].charAt(0)); + Assert.assertEquals(traceGet + traceHead, getHeaders[i].substring(1), headHeaders[j].substring(1)); + j++; } // Stream 5 should have one more trace entry @@ -202,7 +214,8 @@ public class HttpServletDoHeadBaseTest extends Http2TestBase { Tomcat.addServlet(ctxt, "simple", new SimpleServlet()); ctxt.addServletMappingDecoded("/simple", "simple"); - HeadTestServlet s = new HeadTestServlet(bufferSize, useWriter, invalidWriteCount, resetType, validWriteCount, explicitFlush); + HeadTestServlet s = new HeadTestServlet(bufferSize, useWriter, invalidWriteCount, resetType, validWriteCount, + explicitFlush); Wrapper w = Tomcat.addServlet(ctxt, "HeadTestServlet", s); if (useLegacy) { w.addInitParameter(HttpServlet.LEGACY_DO_HEAD, "true"); @@ -247,33 +260,25 @@ public class HttpServletDoHeadBaseTest extends Http2TestBase { /* * Using legacy HEAD handling with a Writer. * - * HttpServlet wraps the response. The test is sensitive to - * buffer sizes. For the tests to pass the number of - * characters that can be written before the response is - * committed needs to be the same. + * HttpServlet wraps the response. The test is sensitive to buffer sizes. For the tests to pass the + * number of characters that can be written before the response is committed needs to be the same. * - * Internally, the Tomcat response buffer defaults to 8192 - * bytes. When a Writer is used, an additional 8192 byte buffer - * is used for char->byte. + * Internally, the Tomcat response buffer defaults to 8192 bytes. When a Writer is used, an additional + * 8192 byte buffer is used for char->byte. * - * The char->byte buffer used by HttpServlet processing HEAD - * requests in legacy mode is created as a 512 byte buffer. + * The char->byte buffer used by HttpServlet processing HEAD requests in legacy mode is created as a 512 + * byte buffer. * - * If the response isn't reset, none of this matters as it is - * just the size of the response buffer and the size of the - * response that determines if chunked encoding is used. - * However, if the response is reset then things get interesting - * as the amount of response data that can be written before - * committing the response is the combination of the char->byte - * buffer and the response buffer. The size of the response - * buffer needs to be adjusted so that both GET and HEAD can - * write the same amount of data before committing the response. - * To make matters worse, to obtain the correct behaviour the - * response buffer size needs to be reset back to 8192 after the - * reset. + * If the response isn't reset, none of this matters as it is just the size of the response buffer and + * the size of the response that determines if chunked encoding is used. However, if the response is + * reset then things get interesting as the amount of response data that can be written before + * committing the response is the combination of the char->byte buffer and the response buffer. The size + * of the response buffer needs to be adjusted so that both GET and HEAD can write the same amount of + * data before committing the response. To make matters worse, to obtain the correct behaviour the + * response buffer size needs to be reset back to 8192 after the reset. * - * This is why the legacy mode is problematic and the new - * approach of the container handling HEAD is better. + * This is why the legacy mode is problematic and the new approach of the container handling HEAD is + * better. */ // Response buffer is always no smaller than originalBufferSize diff --git a/test/jakarta/servlet/http/TestHttpServlet.java b/test/jakarta/servlet/http/TestHttpServlet.java index f895b23f09..fddb138f58 100644 --- a/test/jakarta/servlet/http/TestHttpServlet.java +++ b/test/jakarta/servlet/http/TestHttpServlet.java @@ -45,10 +45,6 @@ import org.apache.tomcat.util.net.TesterSupport.SimpleServlet; public class TestHttpServlet extends TomcatBaseTest { - /* - * Nature of test has changed from original bug report since content-length - * is no longer returned for HEAD requests as allowed by RFC 9110. - */ @Test public void testBug53454() throws Exception { Tomcat tomcat = getTomcatInstance(); @@ -68,7 +64,7 @@ public class TestHttpServlet extends TomcatBaseTest { resHeaders); Assert.assertEquals(HttpServletResponse.SC_OK, rc); - Assert.assertNull(resHeaders.get("Content-Length")); + Assert.assertEquals(LargeBodyServlet.RESPONSE_LENGTH, resHeaders.get("Content-Length").get(0)); } @@ -181,20 +177,29 @@ public class TestHttpServlet extends TomcatBaseTest { int rc = getUrl(path, out, getHeaders); Assert.assertEquals(HttpServletResponse.SC_OK, rc); - removeGeneratingContentHeaders(getHeaders); out.recycle(); - Map<String,List<String>> headHeaders = new HashMap<>(); + Map<String,List<String>> headHeaders = new CaseInsensitiveKeyMap<>(); rc = headUrl(path, out, headHeaders); Assert.assertEquals(HttpServletResponse.SC_OK, rc); + // Date header is likely to be different so just remove it from both GET and HEAD. + getHeaders.remove("date"); + headHeaders.remove("date"); + /* + * There are some headers that are optional for HEAD. See RFC 9110, section 9.3.2. If present, they must be the + * same for both GET and HEAD. If not present in HEAD, remove them from GET. + */ + for (String header : TesterConstants.OPTIONAL_HEADERS_WITH_HEAD) { + if (!headHeaders.containsKey(header)) { + getHeaders.remove(header); + } + } + // Headers should be the same (apart from Date) Assert.assertEquals(getHeaders.size(), headHeaders.size()); for (Map.Entry<String, List<String>> getHeader : getHeaders.entrySet()) { String headerName = getHeader.getKey(); - if ("date".equalsIgnoreCase(headerName)) { - continue; - } Assert.assertTrue(headerName, headHeaders.containsKey(headerName)); List<String> getValues = getHeader.getValue(); List<String> headValues = headHeaders.get(headerName); @@ -208,18 +213,6 @@ public class TestHttpServlet extends TomcatBaseTest { } - /* - * Removes headers that are not expected to appear in the response to the - * equivalent HEAD request. - */ - private void removeGeneratingContentHeaders(Map<String,List<String>> headers) { - headers.remove("content-length"); - headers.remove("content-range"); - headers.remove("trailer"); - headers.remove("transfer-encoding"); - } - - @Test public void testDoOptions() throws Exception { doTestDoOptions(new OptionsServlet(), "GET, HEAD, OPTIONS"); diff --git a/test/jakarta/servlet/http/TesterConstants.java b/test/jakarta/servlet/http/TesterConstants.java new file mode 100644 index 0000000000..f1a76810d9 --- /dev/null +++ b/test/jakarta/servlet/http/TesterConstants.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package jakarta.servlet.http; + +public class TesterConstants { + + public static final String[] OPTIONAL_HEADERS_WITH_HEAD = + new String[] { "content-length", "vary", "transfer-encoding", "trailers" }; +} diff --git a/test/org/apache/coyote/http11/TestHttp11ProcessorDoHead.java b/test/org/apache/coyote/http11/TestHttp11ProcessorDoHead.java new file mode 100644 index 0000000000..fc7967a83c --- /dev/null +++ b/test/org/apache/coyote/http11/TestHttp11ProcessorDoHead.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.coyote.http11; + +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; + +import org.apache.catalina.Context; +import org.apache.catalina.Wrapper; +import org.apache.catalina.startup.Tomcat; +import org.apache.catalina.startup.TomcatBaseTest; +import org.apache.tomcat.util.buf.ByteChunk; + +@RunWith(Parameterized.class) +public class TestHttp11ProcessorDoHead extends TomcatBaseTest { + + private static final int DEFAULT_BUFFER_SIZE = 8192; + + @Parameterized.Parameters(name = "{index}: explicit-cl[{0}], cl[{1}], useContainerHead[{2}], expected-cl[{3}]") + public static Collection<Object[]> parameters() { + + List<Object[]> parameterSets = new ArrayList<>(); + + int[] contentLengths = new int[] { DEFAULT_BUFFER_SIZE / 2, DEFAULT_BUFFER_SIZE -1 , DEFAULT_BUFFER_SIZE, DEFAULT_BUFFER_SIZE + 1, DEFAULT_BUFFER_SIZE * 2}; + + for (Boolean explicitContentLength : booleans) { + for (int contentLength : contentLengths) { + for (HeadSource headSource : HeadSource.values()) { + Integer expectedContentLength = null; + /* + * Tomcat should send a content-length for a head request when the same value would be sent for a + * equivalent GET request. Those circumstances are: + * - The Servlet sets an explicit content-length + * - When the container is providing the HEAD response and the content-length is less than the + * buffer size (so the container would normally set the content-length on a GET) + */ + if (explicitContentLength.booleanValue() || + !HeadSource.SERVLET.equals(headSource) && contentLength <= DEFAULT_BUFFER_SIZE) { + expectedContentLength = Integer.valueOf(contentLength); + } + parameterSets.add(new Object[] { explicitContentLength, Integer.valueOf(contentLength), + headSource, expectedContentLength }); + } + } + } + return parameterSets; + } + + @Parameter(0) + public boolean explicitContentLength; + @Parameter(1) + public int contentLength; + @Parameter(2) + public HeadSource headSource; + @Parameter(3) + public Integer expectedContentLength; + + @SuppressWarnings("removal") + @Test + public void testHeadWithtDoHeadMethod() throws Exception { + Tomcat tomcat = getTomcatInstance(); + + // No file system docBase required + Context ctx = getProgrammaticRootContext(); + + // Add servlet + Wrapper w = Tomcat.addServlet(ctx, "DoHeadServlet", + new DoHeadServlet(explicitContentLength, contentLength, headSource)); + if (HeadSource.CONTAINER_LEGACY.equals(headSource)) { + w.addInitParameter(HttpServlet.LEGACY_DO_HEAD, "true"); + } + + ctx.addServletMappingDecoded("/head", "DoHeadServlet"); + + tomcat.start(); + + ByteChunk responseBody = new ByteChunk(); + Map<String, List<String>> responseHeaders = new HashMap<>(); + int rc = headUrl("http://localhost:" + getPort() + "/head", responseBody, responseHeaders); + + Assert.assertEquals(HttpServletResponse.SC_OK, rc); + + List<String> values = responseHeaders.get("Content-Length"); + if (expectedContentLength == null) { + Assert.assertNull(values); + } else { + Assert.assertNotNull(values); + Assert.assertEquals(1, values.size()); + Assert.assertEquals(expectedContentLength.toString(), values.get(0)); + } + } + + + private static class DoHeadServlet extends HttpServlet { + + private static final long serialVersionUID = 1L; + + private boolean explicitContentLength; + + private int contentLength; + + private HeadSource headSource; + + private DoHeadServlet(boolean explicitContentLength, int contentLength, HeadSource headSource) { + this.explicitContentLength = explicitContentLength; + this.contentLength = contentLength; + this.headSource = headSource; + } + + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + writeResponseHeaders(resp); + PrintWriter pw = resp.getWriter(); + // Efficiency is not a concern here + for (int i = 0; i < contentLength; i++) { + pw.print('X'); + } + } + + + @Override + protected void doHead(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + switch (headSource) { + case SERVLET: + writeResponseHeaders(resp); + break; + case CONTAINER_NEW: + case CONTAINER_LEGACY: + super.doHead(req, resp); + break; + } + } + + + private void writeResponseHeaders(HttpServletResponse resp) { + resp.setContentType("text/plain"); + resp.setCharacterEncoding(StandardCharsets.UTF_8.name()); + if (explicitContentLength) { + resp.setContentLength(contentLength); + } + } + } + + + private enum HeadSource { + SERVLET, + CONTAINER_NEW, + CONTAINER_LEGACY + } +} diff --git a/webapps/docs/changelog.xml b/webapps/docs/changelog.xml index f597e7f33a..0e88b91d14 100644 --- a/webapps/docs/changelog.xml +++ b/webapps/docs/changelog.xml @@ -165,6 +165,13 @@ from external repositories. Prior to this fix, these classes could be incorrectly marked as not found. (markt) </fix> + <fix> + <bug>69466</bug>: Rework handling of HEAD requests. Headers explicitly + set by users will not be removed and any header present in a HEAD + request will also be present in the equivalent GET request. There may be + some headers, as per RFC 9110, section 9.3.2, that are present in a GET + request that are not present in the equivalent HEAD request. (markt) + </fix> </changelog> </subsection> <subsection name="Coyote"> --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@tomcat.apache.org For additional commands, e-mail: dev-h...@tomcat.apache.org