[PATCH][RFC] readelf: partial support of ZSTD compression
Support decompression of ZSTD sections and add support
for it when -SWz is used:
...
[30] .debug_abbrevPROGBITS 1f9d 0168 0 C
0 0 1
[ELF ZSTD (2) 02fc 1]
...
One TODO I see is that:
+libelf_so_LDLIBS = $(libelf_so_DEPS) -lz -lzstd
should be conditional based on HAVE_ZSTD. But I don't know how to do that?
And the second part should cover elfcompress where it should support ZSTD
compression,
leaving that for now.
Martin
libelf/ChangeLog:
* Makefile.am: Add -lzstd.
* elf.h (ELFCOMPRESS_ZSTD): Add new value.
* elf_compress.c (__libelf_decompress): Dispatch based
on the compression algorithm.
(__libelf_decompress_zlib): New.
(__libelf_decompress_zstd): New.
(__libelf_decompress_elf): Pass type of compression to
__libelf_decompress.
* elf_compress_gnu.c (elf_compress_gnu): Use ELFCOMPRESS_ZLIB
as .z* sections can be only compressed with ZLIB.
* libelfP.h (__libelf_decompress): Change signature.
m4/ChangeLog:
* zstd.m4: New file.
src/ChangeLog:
* readelf.c (elf_ch_type_name): Use switch and support ZSTD.
---
libelf/Makefile.am| 2 +-
libelf/elf.h | 3 +++
libelf/elf_compress.c | 56 ---
libelf/elf_compress_gnu.c | 2 +-
libelf/libelfP.h | 2 +-
m4/zstd.m4| 23
src/readelf.c | 18 -
7 files changed, 93 insertions(+), 13 deletions(-)
create mode 100644 m4/zstd.m4
diff --git a/libelf/Makefile.am b/libelf/Makefile.am
index 560ed45f..33f3a0a1 100644
--- a/libelf/Makefile.am
+++ b/libelf/Makefile.am
@@ -106,7 +106,7 @@ libelf_pic_a_SOURCES =
am_libelf_pic_a_OBJECTS = $(libelf_a_SOURCES:.c=.os)
libelf_so_DEPS = ../lib/libeu.a
-libelf_so_LDLIBS = $(libelf_so_DEPS) -lz
+libelf_so_LDLIBS = $(libelf_so_DEPS) -lz -lzstd
if USE_LOCKS
libelf_so_LDLIBS += -lpthread
endif
diff --git a/libelf/elf.h b/libelf/elf.h
index 02a1b3f5..f0f0ec7d 100644
--- a/libelf/elf.h
+++ b/libelf/elf.h
@@ -506,6 +506,9 @@ typedef struct
/* Legal values for ch_type (compression algorithm). */
#define ELFCOMPRESS_ZLIB 1 /* ZLIB/DEFLATE algorithm. */
+#define ELFCOMPRESS_ZSTD 2 /* Compressed with zstd */
+ /* (see http://www.zstandard.org). */
+
#define ELFCOMPRESS_LOOS 0x6000 /* Start of OS-specific. */
#define ELFCOMPRESS_HIOS 0x6fff /* End of OS-specific. */
#define ELFCOMPRESS_LOPROC 0x7000 /* Start of processor-specific. */
diff --git a/libelf/elf_compress.c b/libelf/elf_compress.c
index d7f53af2..62b41b20 100644
--- a/libelf/elf_compress.c
+++ b/libelf/elf_compress.c
@@ -39,6 +39,10 @@
#include
#include
+#ifdef USE_ZSTD
+#include
+#endif
+
/* Cleanup and return result. Don't leak memory. */
static void *
do_deflate_cleanup (void *result, z_stream *z, void *out_buf,
@@ -207,7 +211,7 @@ __libelf_compress (Elf_Scn *scn, size_t hsize, int ei_data,
void *
internal_function
-__libelf_decompress (void *buf_in, size_t size_in, size_t size_out)
+__libelf_decompress_zlib (void *buf_in, size_t size_in, size_t size_out)
{
/* Catch highly unlikely compression ratios so we don't allocate
some giant amount of memory for nothing. The max compression
@@ -260,6 +264,50 @@ __libelf_decompress (void *buf_in, size_t size_in, size_t
size_out)
return buf_out;
}
+#ifdef USE_ZSTD
+void *
+internal_function
+__libelf_decompress_zstd (void *buf_in, size_t size_in, size_t size_out)
+{
+ /* Malloc might return NULL when requestion zero size. This is highly
+ unlikely, it would only happen when the compression was forced.
+ But we do need a non-NULL buffer to return and set as result.
+ Just make sure to always allocate at least 1 byte. */
+ void *buf_out = malloc (size_out ?: 1);
+ if (unlikely (buf_out == NULL))
+{
+ __libelf_seterrno (ELF_E_NOMEM);
+ return NULL;
+}
+
+ size_t ret = ZSTD_decompress (buf_out, size_out, buf_in, size_in);
+ if (ZSTD_isError (ret))
+{
+ free (buf_out);
+ __libelf_seterrno (ELF_E_DECOMPRESS_ERROR);
+ return NULL;
+}
+ else
+return buf_out;
+}
+#endif
+
+void *
+internal_function
+__libelf_decompress (int chtype, void *buf_in, size_t size_in, size_t size_out)
+{
+ if (chtype == ELFCOMPRESS_ZLIB)
+return __libelf_decompress_zlib (buf_in, size_in, size_out);
+ else
+{
+#ifdef USE_ZSTD
+return __libelf_decompress_zstd (buf_in, size_in, size_out);
+#else
+return 0;
+#endif
+}
+}
+
void *
internal_function
__libelf_decompress_elf (Elf_Scn *scn, size_t *size_out, size_t *addralign)
@@ -268,7 +316,7 @@ __libelf_decompress_elf (Elf_Scn *scn, size_t *size_out,
size_t *addralign)
if (gelf_getchdr (scn, &chdr) == NULL)
return NULL;
- if (chdr.ch_type != ELFCOMPRESS_ZLIB)
+ if (chdr.ch_type != ELFCOMPRESS_Z
[Bug general/29565] elfutils: support zstd for SHF_COMPRESSED debug sections
https://sourceware.org/bugzilla/show_bug.cgi?id=29565 --- Comment #1 from Martin Liska --- First part patch: https://sourceware.org/pipermail/elfutils-devel/2022q4/005491.html -- You are receiving this mail because: You are on the CC list for the bug.
Re: [PATCH][RFC] readelf: partial support of ZSTD compression
On Mon, Oct 24, 2022 at 01:09:59PM +0200, Martin Liška wrote: [...] > One TODO I see is that: > +libelf_so_LDLIBS = $(libelf_so_DEPS) -lz -lzstd > > should be conditional based on HAVE_ZSTD. But I don't know how to do that? I suppose you're talking about libzstd_LIBS. [...] > diff --git a/m4/zstd.m4 b/m4/zstd.m4 > new file mode 100644 > index ..6da4db68 > --- /dev/null > +++ b/m4/zstd.m4 > @@ -0,0 +1,23 @@ > +dnl Copyright (C) 2022 Free Software Foundation, Inc. > +dnl This file is free software, distributed under the terms of the GNU > +dnl General Public License. As a special exception to the GNU General > +dnl Public License, this file may be distributed as part of a program > +dnl that contains a configuration script generated by Autoconf, under > +dnl the same distribution terms as the rest of that program. > + > +dnl Enable features using the zstd library. > +AC_DEFUN([AC_ZSTD], [ > +AC_ARG_WITH(zstd, > + [AS_HELP_STRING([--with-zstd], [support zstd compressed debug sections > (default=auto)])], > + [], [with_zstd=auto]) Where does this code come from? I though the "AC_" prefix is reserved for the GNU Autoconf. Also, looks like it would be more appropriate to call it --enable-zstd rather than --with-zstd. -- ldv
[Bug tools/29719] New: eu-readelf -s=section is confusing
https://sourceware.org/bugzilla/show_bug.cgi?id=29719
Bug ID: 29719
Summary: eu-readelf -s=section is confusing
Product: elfutils
Version: unspecified
Status: NEW
Severity: normal
Priority: P2
Component: tools
Assignee: unassigned at sourceware dot org
Reporter: mliska at suse dot cz
CC: elfutils-devel at sourceware dot org, mjw at fedoraproject
dot org
Target Milestone: ---
Note there's a difference in between binutils and elfutils where the later on
has an optional argument:
--symbols [section name]
I have a few comments:
1) it's confusing as one expects that all symbols in "section name" will be
printed, but it's not correct. It's the name of '.symtab' section.
2) readelf -sW xxx does not print anything, while binutils does:
The reason is simple, 'W' section is not found in the binary. I would recommend
printing a warning:
./src/readelf -sW /home/marxin/Programming/linux/arch/x86/kernel/head_64.o
WARNING: cannot find section: 'W'
WARNING: cannot find section: 'W'
diff --git a/src/readelf.c b/src/readelf.c
index a206e60e..465526c4 100644
--- a/src/readelf.c
+++ b/src/readelf.c
@@ -2431,6 +2431,7 @@ print_symtab (Ebl *ebl, int type)
/* Find the symbol table(s). For this we have to search through the
section table. */
Elf_Scn *scn = NULL;
+ bool symtab_handled = false;
while ((scn = elf_nextscn (ebl->elf, scn)) != NULL)
{
@@ -2466,8 +2467,13 @@ print_symtab (Ebl *ebl, int type)
elf_ndxscn (scn), elf_errmsg (-1));
}
handle_symtab (ebl, scn, shdr);
+ symtab_handled = true;
}
}
+
+ if (!symtab_handled && symbol_table_section != NULL)
+printf ("WARNING: %s: '%s'\n", _("cannot find section"),
+ symbol_table_section);
}
--
You are receiving this mail because:
You are on the CC list for the bug.
Re: [PATCH][RFC] readelf: partial support of ZSTD compression
On 10/24/22 13:41, Dmitry V. Levin wrote: > On Mon, Oct 24, 2022 at 01:09:59PM +0200, Martin Liška wrote: > [...] >> One TODO I see is that: >> +libelf_so_LDLIBS = $(libelf_so_DEPS) -lz -lzstd >> >> should be conditional based on HAVE_ZSTD. But I don't know how to do that? > > I suppose you're talking about libzstd_LIBS. Hm, can't see it after autoreconf -fi and ./configure. > > [...] >> diff --git a/m4/zstd.m4 b/m4/zstd.m4 >> new file mode 100644 >> index ..6da4db68 >> --- /dev/null >> +++ b/m4/zstd.m4 >> @@ -0,0 +1,23 @@ >> +dnl Copyright (C) 2022 Free Software Foundation, Inc. >> +dnl This file is free software, distributed under the terms of the GNU >> +dnl General Public License. As a special exception to the GNU General >> +dnl Public License, this file may be distributed as part of a program >> +dnl that contains a configuration script generated by Autoconf, under >> +dnl the same distribution terms as the rest of that program. >> + >> +dnl Enable features using the zstd library. >> +AC_DEFUN([AC_ZSTD], [ >> +AC_ARG_WITH(zstd, >> + [AS_HELP_STRING([--with-zstd], [support zstd compressed debug sections >> (default=auto)])], >> + [], [with_zstd=auto]) > > Where does this code come from? > I though the "AC_" prefix is reserved for the GNU Autoconf. It comes from binutils './config/zstd.m4' file. > Also, looks like it would be more appropriate to call it --enable-zstd > rather than --with-zstd. > Ah, I see. Thanks, Martin
[Bug tools/29719] eu-readelf -s=section is confusing
https://sourceware.org/bugzilla/show_bug.cgi?id=29719 --- Comment #1 from Martin Liska --- 3) we may want to implement the semantic: dump all symbols that live in a given section "foo". Can be handy for sections like .hot.text and so on. -- You are receiving this mail because: You are on the CC list for the bug.
Re: [PATCH][RFC] readelf: partial support of ZSTD compression
On Mon, Oct 24, 2022 at 02:17:17PM +0200, Martin Liška wrote: > On 10/24/22 13:41, Dmitry V. Levin wrote: > > On Mon, Oct 24, 2022 at 01:09:59PM +0200, Martin Liška wrote: > > [...] > >> One TODO I see is that: > >> +libelf_so_LDLIBS = $(libelf_so_DEPS) -lz -lzstd > >> > >> should be conditional based on HAVE_ZSTD. But I don't know how to do that? > > > > I suppose you're talking about libzstd_LIBS. > > Hm, can't see it after autoreconf -fi and ./configure. That's because you do PKG_CHECK_MODULES(ZSTD, [libzstd], ...) and this defines ZSTD_CFLAGS and ZSTD_LIBS instead of libzstd_CFLAGS and libzstd_LIBS because PKG_CHECK_MODULES() uses its first argument as the prefix for these variables. -- ldv
Re: [PATCH][RFC] readelf: partial support of ZSTD compression
On 10/24/22 18:48, Dmitry V. Levin wrote:
> On Mon, Oct 24, 2022 at 02:17:17PM +0200, Martin Liška wrote:
>> On 10/24/22 13:41, Dmitry V. Levin wrote:
>>> On Mon, Oct 24, 2022 at 01:09:59PM +0200, Martin Liška wrote:
>>> [...]
One TODO I see is that:
+libelf_so_LDLIBS = $(libelf_so_DEPS) -lz -lzstd
should be conditional based on HAVE_ZSTD. But I don't know how to do that?
>>>
>>> I suppose you're talking about libzstd_LIBS.
>>
>> Hm, can't see it after autoreconf -fi and ./configure.
>
> That's because you do
> PKG_CHECK_MODULES(ZSTD, [libzstd], ...)
> and this defines ZSTD_CFLAGS and ZSTD_LIBS instead of libzstd_CFLAGS
> and libzstd_LIBS because PKG_CHECK_MODULES() uses its first argument
> as the prefix for these variables.
>
>
Thank you. Apparently, I collided with the existing:
eu_ZIPLIB(zstd,ZSTD,zstd,ZSTD_decompress,[ZSTD (zst)])
Anyway, I'm sending V2 that works fine --with-zstd and --without-zstd as
expected.
Ready for master?
Thanks,
Martin
From 4aea412783b9b0dcaf0f887947bf2e8ee6c5368b Mon Sep 17 00:00:00 2001
From: Martin Liska
Date: Mon, 24 Oct 2022 11:53:13 +0200
Subject: [PATCH] readelf: partial support of ZSTD compression
Support decompression of ZSTD sections and add support
for it when -SWz is used:
...
[30] .debug_abbrevPROGBITS 1f9d 0168 0 C 0 0 1
[ELF ZSTD (2) 02fc 1]
...
ChangeLog:
* configure.ac: Add zstd_LIBS.
libelf/ChangeLog:
* Makefile.am: Use zstd_LIBS.
* elf.h (ELFCOMPRESS_ZSTD): Add new value.
* elf_compress.c (__libelf_decompress): Dispatch based
on the compression algorithm.
(__libelf_decompress_zlib): New.
(__libelf_decompress_zstd): New.
(__libelf_decompress_elf): Pass type of compression to
__libelf_decompress.
* elf_compress_gnu.c (elf_compress_gnu): Use ELFCOMPRESS_ZLIB
as .z* sections can be only compressed with ZLIB.
* libelfP.h (__libelf_decompress): Change signature.
src/ChangeLog:
* readelf.c (elf_ch_type_name): Use switch and support ZSTD.
---
configure.ac | 8 +++---
libelf/Makefile.am| 2 +-
libelf/elf.h | 3 +++
libelf/elf_compress.c | 56 ---
libelf/elf_compress_gnu.c | 2 +-
libelf/libelfP.h | 2 +-
src/readelf.c | 18 -
7 files changed, 75 insertions(+), 16 deletions(-)
diff --git a/configure.ac b/configure.ac
index 03b67a9d..803876e2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -410,6 +410,11 @@ dnl Test for bzlib and xz/lzma/zstd, gives BZLIB/LZMALIB/ZSTD .am
dnl conditional and config.h USE_BZLIB/USE_LZMALIB/USE_ZSTD #define.
save_LIBS="$LIBS"
LIBS=
+eu_ZIPLIB(zstd,ZSTD,zstd,ZSTD_decompress,[ZSTD (zst)])
+AS_IF([test "x$with_zstd" = xyes], [LIBZSTD="libzstd"], [LIBLZSTD=""])
+AC_SUBST([LIBZSTD])
+zstd_LIBS="$LIBS"
+AC_SUBST([zstd_LIBS])
eu_ZIPLIB(bzlib,BZLIB,bz2,BZ2_bzdopen,bzip2)
# We need this since bzip2 doesn't have a pkgconfig file.
BZ2_LIB="$LIBS"
@@ -417,9 +422,6 @@ AC_SUBST([BZ2_LIB])
eu_ZIPLIB(lzma,LZMA,lzma,lzma_auto_decoder,[LZMA (xz)])
AS_IF([test "x$with_lzma" = xyes], [LIBLZMA="liblzma"], [LIBLZMA=""])
AC_SUBST([LIBLZMA])
-eu_ZIPLIB(zstd,ZSTD,zstd,ZSTD_decompress,[ZSTD (zst)])
-AS_IF([test "x$with_zstd" = xyes], [LIBZSTD="libzstd"], [LIBLZSTD=""])
-AC_SUBST([LIBZSTD])
zip_LIBS="$LIBS"
LIBS="$save_LIBS"
AC_SUBST([zip_LIBS])
diff --git a/libelf/Makefile.am b/libelf/Makefile.am
index 560ed45f..24c25cf8 100644
--- a/libelf/Makefile.am
+++ b/libelf/Makefile.am
@@ -106,7 +106,7 @@ libelf_pic_a_SOURCES =
am_libelf_pic_a_OBJECTS = $(libelf_a_SOURCES:.c=.os)
libelf_so_DEPS = ../lib/libeu.a
-libelf_so_LDLIBS = $(libelf_so_DEPS) -lz
+libelf_so_LDLIBS = $(libelf_so_DEPS) -lz $(zstd_LIBS)
if USE_LOCKS
libelf_so_LDLIBS += -lpthread
endif
diff --git a/libelf/elf.h b/libelf/elf.h
index 02a1b3f5..f0f0ec7d 100644
--- a/libelf/elf.h
+++ b/libelf/elf.h
@@ -506,6 +506,9 @@ typedef struct
/* Legal values for ch_type (compression algorithm). */
#define ELFCOMPRESS_ZLIB 1 /* ZLIB/DEFLATE algorithm. */
+#define ELFCOMPRESS_ZSTD 2 /* Compressed with zstd */
+ /* (see http://www.zstandard.org). */
+
#define ELFCOMPRESS_LOOS 0x6000 /* Start of OS-specific. */
#define ELFCOMPRESS_HIOS 0x6fff /* End of OS-specific. */
#define ELFCOMPRESS_LOPROC 0x7000 /* Start of processor-specific. */
diff --git a/libelf/elf_compress.c b/libelf/elf_compress.c
index d7f53af2..62b41b20 100644
--- a/libelf/elf_compress.c
+++ b/libelf/elf_compress.c
@@ -39,6 +39,10 @@
#include
#include
+#ifdef USE_ZSTD
+#include
+#endif
+
/* Cleanup and return result. Don't leak memory. */
static void *
do_deflate_cleanup (void *result, z_stream *z, void *out_buf,
@@ -207,7 +211,7 @@ __libelf_compress (Elf_Scn *scn, size_t hsize, int ei_data,
void *
internal_function
-__libelf_decompress (void *buf_in, size_t size_in, size_t size_out)
+__libelf_decompress_zlib (void *buf_in, size_t size_in, size_t size_out)
{
/* Ca
Re: [PATCH] debuginfod: Support queries for ELF/DWARF sections
Hi - Generally looks fine, thanks a lot. A few nits: - use of write(2) to put files onto disk is not quite right; write(2) can be partial, so you need a loop (or a macro wrapping a loop) - not sure I understand why the code worries about dots in or not in section names. Why not just pass them verbatim throughout the code base, and not worry about whether or not there's a dot? Does the ELF standard even require a dot? - not sure whether/why the I queries require a new _query_i view, as opposed to running the _query_d & _query_e views union'd together. I see an ORDER BY that's different here but not sure why bother; if anything, the server could prefer one or the other type, based on the same section-name heuristic as the client - don't really see a need for the X-DEBUGINFOD-SECTION response header, which simply echoes back the very same parameter the client just requested; the other X-DEBUGINFOD-* headers are novel metadata - re. verbose logging in the section vs non-section case, suggest just keeping the code simple (even if it makes the logs more verbose), i.e., not duplicating if (...) clog << STUFF else clog << STUFF; no biggie tho - the webapi docs in debuginfod.8 should document the new query type Otherwise lgtm. Lots of nice work. - FChE
