On Thu, Aug 5, 2021 at 8:55 PM Purushottam choudhary
<[email protected]> wrote:
>
> Added 0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch
> to fix large memory usage for large file downloads
> from dynamic backends reuse or release large memory chunks.
>
> This issue is caused by a bug in the lighttpd 1.4.55 version and
> has been fixed in lighttpd 1.4.58. Hence, it is not needed for
> master and hardknott branch because lighttpd has 1.4.59 version.
>
> Link:
> https://redmine.lighttpd.net/projects/lighttpd/repository/14/revisions/7ba521ffb4959f6f74a609d5d4acafc29a038337
This patch is throwing warnings on the autobuilder:
Applying patch 0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch
patching file src/chunk.c
Hunk #2 succeeded at 124 with fuzz 1.
patching file src/chunk.h
patching file src/http-header-glue.c
The context lines in the patches can be updated with devtool:
devtool modify lighttpd
devtool finish --force-patch-refresh lighttpd <layer_path>
Could you fix this and send a V2 so that we can have clean builds on
the autobuilder?
Thanks!
Steve
> Signed-off-by: Purushottam Choudhary <[email protected]>
> ---
> ...large-mem-chunks-fix-mem-usage-fixes-3033.patch | 225
> +++++++++++++++++++++
> meta/recipes-extended/lighttpd/lighttpd_1.4.55.bb | 1 +
> 2 files changed, 226 insertions(+)
> create mode 100644
> meta/recipes-extended/lighttpd/lighttpd/0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch
>
> diff --git
> a/meta/recipes-extended/lighttpd/lighttpd/0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch
>
> b/meta/recipes-extended/lighttpd/lighttpd/0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch
> new file mode 100644
> index 0000000..c4e4a60
> --- /dev/null
> +++
> b/meta/recipes-extended/lighttpd/lighttpd/0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch
> @@ -0,0 +1,225 @@
> +From 40dadb863fd105c4a66b598c2423b1eb11b1f26e Mon Sep 17 00:00:00 2001
> +From: Glenn Strauss <[email protected]>
> +Date: Wed, 23 Dec 2020 23:14:47 -0500
> +Subject: [PATCH] [core] reuse large mem chunks (fix mem usage) (fixes #3033)
> +
> +(cherry picked from commit 7ba521ffb4959f6f74a609d5d4acafc29a038337)
> +
> +(thx flynn)
> +
> +fix large memory usage for large file downloads from dynamic backends
> +
> +reuse or release large memory chunks
> +
> +x-ref:
> + "Memory Growth with PUT and full buffered streams"
> + https://redmine.lighttpd.net/issues/3033
> +
> +Upstream-Status: Backport
> +Comment: No hunk refreshed
> +https://redmine.lighttpd.net/projects/lighttpd/repository/14/revisions/7ba521ffb4959f6f74a609d5d4acafc29a038337
> +Signed-off-by: Purushottam Choudhary <[email protected]>
> +---
> + src/chunk.c | 99 +++++++++++++++++++++++++++++++++---------
> + src/chunk.h | 2 +
> + src/http-header-glue.c | 2 +-
> + 3 files changed, 82 insertions(+), 21 deletions(-)
> +
> +diff --git a/src/chunk.c b/src/chunk.c
> +index 09dd3f14..ffb37951 100644
> +--- a/src/chunk.c
> ++++ b/src/chunk.c
> +@@ -28,16 +28,20 @@
> + static size_t chunk_buf_sz = 4096;
> + static chunk *chunks, *chunks_oversized;
> + static chunk *chunk_buffers;
> ++static int chunks_oversized_n;
> + static array *chunkqueue_default_tempdirs = NULL;
> + static off_t chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE;
> +
> + void chunkqueue_set_chunk_size (size_t sz)
> + {
> +- chunk_buf_sz = sz > 0 ? ((sz + 1023) & ~1023uL) : 4096;
> ++ size_t x = 1024;
> ++ while (x < sz && x < (1u << 30)) x <<= 1;
> ++ chunk_buf_sz = sz > 0 ? x : 4096;
> + }
> +
> + void chunkqueue_set_tempdirs_default_reset (void)
> + {
> ++ chunk_buf_sz = 8192;
> + chunkqueue_default_tempdirs = NULL;
> + chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE;
> + }
> +@@ -120,15 +124,49 @@ static void chunk_free(chunk *c) {
> + free(c);
> + }
> +
> +-buffer * chunk_buffer_acquire(void) {
> ++static chunk * chunk_pop_oversized(size_t sz) {
> ++ /* future: might have buckets of certain sizes, up to socket buf sizes
> */
> ++ if (chunks_oversized && chunks_oversized->mem->size >= sz) {
> ++ --chunks_oversized_n;
> ++ chunk *c = chunks_oversized;
> ++ chunks_oversized = c->next;
> ++ return c;
> ++ }
> ++ return NULL;
> ++}
> ++
> ++static void chunk_push_oversized(chunk * const c, const size_t sz) {
> ++ if (chunks_oversized_n < 64 && chunk_buf_sz >= 4096) {
> ++ ++chunks_oversized_n;
> ++ chunk **co = &chunks_oversized;
> ++ while (*co && sz < (*co)->mem->size) co = &(*co)->next;
> ++ c->next = *co;
> ++ *co = c;
> ++ }
> ++ else
> ++ chunk_free(c);
> ++}
> ++
> ++static buffer * chunk_buffer_acquire_sz(size_t sz) {
> + chunk *c;
> + buffer *b;
> +- if (chunks) {
> +- c = chunks;
> +- chunks = c->next;
> ++ if (sz <= chunk_buf_sz) {
> ++ if (chunks) {
> ++ c = chunks;
> ++ chunks = c->next;
> ++ }
> ++ else
> ++ c = chunk_init(chunk_buf_sz);
> ++ /* future: might choose to pop from chunks_oversized, if
> available
> ++ * (even if larger than sz) rather than allocating new chunk
> ++ * (and if doing so, might replace chunks_oversized_n) */
> + }
> + else {
> +- c = chunk_init(chunk_buf_sz);
> ++ /*(round up to nearest chunk_buf_sz)*/
> ++ sz = (sz + (chunk_buf_sz-1)) & ~(chunk_buf_sz-1);
> ++ c = chunk_pop_oversized(sz);
> ++ if (NULL == c)
> ++ c = chunk_init(sz);
> + }
> + c->next = chunk_buffers;
> + chunk_buffers = c;
> +@@ -137,21 +175,47 @@ buffer * chunk_buffer_acquire(void) {
> + return b;
> + }
> +
> ++buffer * chunk_buffer_acquire(void) {
> ++ return chunk_buffer_acquire_sz(chunk_buf_sz);
> ++}
> ++
> + void chunk_buffer_release(buffer *b) {
> + if (NULL == b) return;
> +- if (b->size >= chunk_buf_sz && chunk_buffers) {
> ++ if (chunk_buffers) {
> + chunk *c = chunk_buffers;
> + chunk_buffers = c->next;
> + c->mem = b;
> +- c->next = chunks;
> +- chunks = c;
> + buffer_clear(b);
> ++ if (b->size == chunk_buf_sz) {
> ++ c->next = chunks;
> ++ chunks = c;
> ++ }
> ++ else if (b->size > chunk_buf_sz)
> ++ chunk_push_oversized(c, b->size);
> ++ else
> ++ chunk_free(c);
> + }
> + else {
> + buffer_free(b);
> + }
> + }
> +
> ++size_t chunk_buffer_prepare_append(buffer * const b, size_t sz) {
> ++ if (sz > chunk_buffer_string_space(b)) {
> ++ sz += b->used ? b->used : 1;
> ++ buffer * const cb = chunk_buffer_acquire_sz(sz);
> ++ /* swap buffer contents and copy original b->ptr into larger b->ptr
> */
> ++ /*(this does more than buffer_move())*/
> ++ buffer tb = *b;
> ++ *b = *cb;
> ++ *cb = tb;
> ++ if ((b->used = tb.used))
> ++ memcpy(b->ptr, tb.ptr, tb.used);
> ++ chunk_buffer_release(cb);
> ++ }
> ++ return chunk_buffer_string_space(b);
> ++}
> ++
> + static chunk * chunk_acquire(size_t sz) {
> + if (sz <= chunk_buf_sz) {
> + if (chunks) {
> +@@ -162,13 +226,10 @@ static chunk * chunk_acquire(size_t sz) {
> + sz = chunk_buf_sz;
> + }
> + else {
> +- sz = (sz + 8191) & ~8191uL;
> +- /* future: might have buckets of certain sizes, up to socket buf
> sizes*/
> +- if (chunks_oversized && chunks_oversized->mem->size >= sz) {
> +- chunk *c = chunks_oversized;
> +- chunks_oversized = c->next;
> +- return c;
> +- }
> ++ /*(round up to nearest chunk_buf_sz)*/
> ++ sz = (sz + (chunk_buf_sz-1)) & ~(chunk_buf_sz-1);
> ++ chunk *c = chunk_pop_oversized(sz);
> ++ if (c) return c;
> + }
> +
> + return chunk_init(sz);
> +@@ -183,10 +244,7 @@ static void chunk_release(chunk *c) {
> + }
> + else if (sz > chunk_buf_sz) {
> + chunk_reset(c);
> +- chunk **co = &chunks_oversized;
> +- while (*co && sz < (*co)->mem->size) co = &(*co)->next;
> +- c->next = *co;
> +- *co = c;
> ++ chunk_push_oversized(c, sz);
> + }
> + else {
> + chunk_free(c);
> +@@ -205,6 +263,7 @@ void chunkqueue_chunk_pool_clear(void)
> + chunk_free(c);
> + }
> + chunks_oversized = NULL;
> ++ chunks_oversized_n = 0;
> + }
> +
> + void chunkqueue_chunk_pool_free(void)
> +diff --git a/src/chunk.h b/src/chunk.h
> +index 4c6b7e47..93f343c3 100644
> +--- a/src/chunk.h
> ++++ b/src/chunk.h
> +@@ -50,6 +50,8 @@ typedef struct {
> + buffer * chunk_buffer_acquire(void);
> + void chunk_buffer_release(buffer *b);
> +
> ++size_t chunk_buffer_prepare_append (buffer *b, size_t sz);
> ++
> + void chunkqueue_chunk_pool_clear(void);
> + void chunkqueue_chunk_pool_free(void);
> +
> +diff --git a/src/http-header-glue.c b/src/http-header-glue.c
> +index d54f00cf..2231fba9 100644
> +--- a/src/http-header-glue.c
> ++++ b/src/http-header-glue.c
> +@@ -1267,7 +1267,7 @@ handler_t http_response_read(server *srv, connection
> *con, http_response_opts *o
> + if (avail < toread) {
> + /*(add avail+toread to reduce allocations when ioctl
> EOPNOTSUPP)*/
> + avail = avail ? avail - 1 + toread : toread;
> +- buffer_string_prepare_append(b, avail);
> ++ avail = chunk_buffer_prepare_append(b, avail);
> + }
> +
> + n = read(fd, b->ptr+buffer_string_length(b), avail);
> +--
> +2.32.0
> diff --git a/meta/recipes-extended/lighttpd/lighttpd_1.4.55.bb
> b/meta/recipes-extended/lighttpd/lighttpd_1.4.55.bb
> index 35a268a..5329465 100644
> --- a/meta/recipes-extended/lighttpd/lighttpd_1.4.55.bb
> +++ b/meta/recipes-extended/lighttpd/lighttpd_1.4.55.bb
> @@ -18,6 +18,7 @@ SRC_URI =
> "http://download.lighttpd.net/lighttpd/releases-1.4.x/lighttpd-${PV}.t
> file://lighttpd.conf \
> file://lighttpd \
>
> file://0001-Use-pkg-config-for-pcre-dependency-instead-of-config.patch \
> +
> file://0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch \
> "
>
> SRC_URI[md5sum] = "be4bda2c28bcbdac6eb941528f6edf03"
> --
> 2.7.4
>
> This message contains information that may be privileged or confidential and
> is the property of the KPIT Technologies Ltd. It is intended only for the
> person to whom it is addressed. If you are not the intended recipient, you
> are not authorized to read, print, retain copy, disseminate, distribute, or
> use this message or any part thereof. If you receive this message in error,
> please notify the sender immediately and delete all copies of this message.
> KPIT Technologies Ltd. does not accept any liability for virus infected mails.
>
>
>
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#154577):
https://lists.openembedded.org/g/openembedded-core/message/154577
Mute This Topic: https://lists.openembedded.org/mt/84703025/21656
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
