[Bug debuginfod/29474] New: Server returns 404 for concurrent requests when leading to a same .rpm
https://sourceware.org/bugzilla/show_bug.cgi?id=29474 Bug ID: 29474 Summary: Server returns 404 for concurrent requests when leading to a same .rpm Product: elfutils Version: unspecified Status: NEW Severity: normal Priority: P2 Component: debuginfod Assignee: unassigned at sourceware dot org Reporter: mliska at suse dot cz CC: elfutils-devel at sourceware dot org, fche at redhat dot com Target Milestone: --- Reproducer: $ mkdir /tmp/test && cd /tmp/test && wget wget https://ftp5.gwdg.de/pub/opensuse/tumbleweed/repo/debug/x86_64/cross-aarch64-gcc12-debuginfo-12.1.1%2Bgit287-1.1.x86_64.rpm Clone: https://github.com/marxin/script-misc.git Index the folder: ./debuginfod/debuginfod -F -Z '.rpm=(rpm2cpio|zstdcat)<' /tmp/test -C 32 And then run: $ ./debuginfod-stress.py data/files-local2.txt http://localhost:8002 Running stress test of 5 binaries in 8 threads: Attemp #0 4 666d857e7318e96a79ac1eb6850e47027a644695 Attemp #1 4 666d857e7318e96a79ac1eb6850e47027a644695 and server reports: [Thu 11 Aug 2022 04:47:43 PM GMT] (28119/28122): ::1:50958 UA:pythonrequests/2.28.1 XFF: GET /buildid/671e2a753d37fe5a8e9979c25bbdedef5fa11606/debuginfo 200 3565448 0+1010ms [Thu 11 Aug 2022 04:47:43 PM GMT] (28119/28138): not found [Thu 11 Aug 2022 04:47:43 PM GMT] (28119/28138): ::1:50998 UA:pythonrequests/2.28.1 XFF: GET /buildid/666d857e7318e96a79ac1eb6850e47027a644695/debuginfo 404 9 0+1034ms [Thu 11 Aug 2022 04:47:43 PM GMT] (28119/28124): ::1:50960 UA:pythonrequests/2.28.1 XFF: GET /buildid/4fec8709140eee9d594f6c81823fc481a7464780/debuginfo 200 304712 0+1247ms [Thu 11 Aug 2022 04:47:43 PM GMT] (28119/28149): ::1:50944 UA:pythonrequests/2.28.1 XFF: GET /buildid/c4ad4bb208047e2f7020b0f64f1324ce21600e12/debuginfo 200 3562688 0+1321ms [Thu 11 Aug 2022 04:47:43 PM GMT] (28119/28126): ::1:50990 UA:pythonrequests/2.28.1 XFF: GET /buildid/85b1a85f235f4a3a0e4f1cfc27e480ae89e6c02a/debuginfo 200 1298415 0+1332ms [Thu 11 Aug 2022 04:47:44 PM GMT] (28119/28151): ::1:51004 UA:pythonrequests/2.28.1 XFF: GET /buildid/666d857e7318e96a79ac1eb6850e47027a644695/debuginfo 200 67192 0+0ms -- You are receiving this mail because: You are on the CC list for the bug.
[Bug debuginfod/29474] Server returns 404 for concurrent requests when leading to a same .rpm
https://sourceware.org/bugzilla/show_bug.cgi?id=29474
--- Comment #1 from Martin Liska ---
Have a better knowledge of what happens. Let's assume the following debugging
patch:
diff --git a/debuginfod/debuginfod.cxx b/debuginfod/debuginfod.cxx
index a089d0bd..0320f289 100644
--- a/debuginfod/debuginfod.cxx
+++ b/debuginfod/debuginfod.cxx
@@ -1743,7 +1743,10 @@ handle_buildid_r_match (bool internal_req_p,
continue;
if (fdcache.probe (b_source0, fn)) // skip if already interned
+ {
+obatched(clog) << "SKIPPING due to probe=" << b_source1 << "
filename=" << fn << " r=" << r << endl;
continue;
+ }
// extract this file to a temporary file
char* tmppath = NULL;
@@ -1776,6 +1779,7 @@ handle_buildid_r_match (bool internal_req_p,
{
// NB: now we know we have a complete reusable file; make fdcache
// responsible for unlinking it later.
+ obatched(clog) << "INTERN " << b_source1 << " filename=" << fn <<
endl;
fdcache.intern(b_source0, fn,
tmppath, archive_entry_size(e),
false); // prefetched ones go to the prefetch cache
@@ -1792,6 +1796,7 @@ handle_buildid_r_match (bool internal_req_p,
inc_metric ("http_responses_total","result",archive_extension + "
archive");
r = MHD_create_response_from_fd (archive_entry_size(e), fd);
+ obatched(clog) << "ASSIGN " << b_source1 << " filename=" << fn << " r="
<< r << endl;
if (r == 0)
{
if (verbose)
@@ -1820,6 +1825,7 @@ handle_buildid_r_match (bool internal_req_p,
}
// XXX: rpm/file not found: delete this R entry?
+ obatched(clog) << "RETURN" << b_source1 << " r=" << r << endl;
return r;
}
debuginfod log shows:
...
[Thu 11 Aug 2022 08:02:42 PM GMT] (21225/21257): found mtime=1659137364 stype=R
source0=/home/marxin/BIG/test/cross-aarch64-gcc12-debuginfo-12.1.1+git287-1.1.x86_64.rpm
source1=/usr/lib/debug/.dwz/cross-aarch64-gcc12-12.1.1+git287-1.1.x86_64
[Thu 11 Aug 2022 08:02:42 PM GMT] (21225/21251): found mtime=1659137364 stype=R
source0=/home/marxin/BIG/test/cross-aarch64-gcc12-debuginfo-12.1.1+git287-1.1.x86_64.rpm
source1=/usr/lib/debug/usr/aarch64-suse-linux/sys-root/lib64/libgcc_s.so.1-12.1.1+git287-1.1.x86_64.debug
...
[Thu 11 Aug 2022 08:02:42 PM GMT] (21225/21257): INTERN
/usr/lib/debug/.dwz/cross-aarch64-gcc12-12.1.1+git287-1.1.x86_64
filename=/usr/lib/debug/usr/aarch64-suse-linux/sys-root/lib64/libgcc_s.so.1-12.1.1+git287-1.1.x86_64.debug
<- so here 21257 request interns libgcc_s.so.1-12.1.1+git287-1.1.x86_64.debug
[Thu 11 Aug 2022 08:02:42 PM GMT] (21225/21251): SKIPPING due to
probe=/usr/lib/debug/usr/aarch64-suse-linux/sys-root/lib64/libgcc_s.so.1-12.1.1+git287-1.1.x86_64.debug
filename=/usr/lib/debug/usr/aarch64-suse-linux/sys-root/lib64/libgcc_s.so.1-12.1.1+git287-1.1.x86_64.debug
r=0 <- and due to that it's skipped here and we never assign r = ...
[Thu 11 Aug 2022 08:02:43 PM GMT] (21225/21251):
RETURN/usr/lib/debug/usr/aarch64-suse-linux/sys-root/lib64/libgcc_s.so.1-12.1.1+git287-1.1.x86_64.debug
r=0
...
Note in correct run the following happens:
[Thu 11 Aug 2022 08:16:42 PM GMT] (22289/22319): ASSIGN
/usr/lib/debug/usr/aarch64-suse-linux/sys-root/lib64/libgcc_s.so.1-12.1.1+git287-1.1.x86_64.debug
filename=/usr/lib/debug/usr/aarch64-suse-linux/sys-root/lib64/libgcc_s.so.1-12.1.1+git287-1.1.x86_64.debug
r=0x7fff740548c0
...
[Thu 11 Aug 2022 08:16:43 PM GMT] (22289/22319):
RETURN/usr/lib/debug/usr/aarch64-suse-linux/sys-root/lib64/libgcc_s.so.1-12.1.1+git287-1.1.x86_64.debug
r=0x7fff740548c0
So the problem is somehow related to fdcache.
--
You are receiving this mail because:
You are on the CC list for the bug.
[Bug debuginfod/29474] Server returns 404 for concurrent requests when leading to a same .rpm
https://sourceware.org/bugzilla/show_bug.cgi?id=29474
--- Comment #2 from Martin Liska ---
So it's the prefetching feature as documented here:
// 3) extract some number of prefetched entries (just into fdcache)
...
if (r != 0) // stage 3
{
// NB: now we know we have a complete reusable file; make fdcache
// responsible for unlinking it later.
obatched(clog) << "INTERN " << b_source1 << " filename=" << fn <<
endl;
fdcache.intern(b_source0, fn,
tmppath, archive_entry_size(e),
false); // prefetched ones go to the prefetch cache
prefetch_count --;
close (fd); // we're not saving this fd to make a mhd-response from!
continue;
}
So what's bad is that
handle_buildid_r_match begins with:
// check for a match in the fdcache first
int fd = fdcache.lookup(b_source0, b_source1);
while (fd >= 0) // got one!; NB: this is really an if() with a possible
branch out to the end
{
...
}
which was false, but happens to be true once another thread inters the file :/
--
You are receiving this mail because:
You are on the CC list for the bug.
[Bug debuginfod/29474] Server returns 404 for concurrent requests when leading to a same .rpm
https://sourceware.org/bugzilla/show_bug.cgi?id=29474 --- Comment #3 from Martin Liska --- Created attachment 14271 --> https://sourceware.org/bugzilla/attachment.cgi?id=14271&action=edit Possible patch candidate (needs removal of extra debugging) -- You are receiving this mail because: You are on the CC list for the bug.
[Bug debuginfod/29474] Server returns 404 for concurrent requests when leading to a same .rpm
https://sourceware.org/bugzilla/show_bug.cgi?id=29474 Martin Liska changed: What|Removed |Added Attachment #14271|0 |1 is obsolete|| --- Comment #4 from Martin Liska --- Created attachment 14272 --> https://sourceware.org/bugzilla/attachment.cgi?id=14272&action=edit Possible patch candidate (needs removal of extra debugging) v2 -- You are receiving this mail because: You are on the CC list for the bug.
[Bug tools/29073] program header entry 2: unknown program header entry type 0x70000003
https://sourceware.org/bugzilla/show_bug.cgi?id=29073 Andreas Schwab changed: What|Removed |Added Status|ASSIGNED|RESOLVED Resolution|--- |FIXED --- Comment #4 from Andreas Schwab --- This has been fixed by: 2f275fc1 elflint: Allow zero p_memsz for PT_RISCV_ATTRIBUTES d703772e backends: Handle new RISC-V specific definitions de209d23 libelf: Sync elf.h from glibc -- You are receiving this mail because: You are on the CC list for the bug.
[Bug debuginfod/29469] New: Add /buildid/BUILDID/header WEBAPI
https://sourceware.org/bugzilla/show_bug.cgi?id=29469 Bug ID: 29469 Summary: Add /buildid/BUILDID/header WEBAPI Product: elfutils Version: unspecified Status: UNCONFIRMED Severity: normal Priority: P2 Component: debuginfod Assignee: unassigned at sourceware dot org Reporter: mliska at suse dot cz CC: elfutils-devel at sourceware dot org Target Milestone: --- I'm writing a checking script that iterates binaries and investigates if debuginfod has the binary indexed. Thus, I use /buildid/BUILDID/debuginfo WEBAPI which is expensive regarding the net transport. I would be happy with /header where I only get x-debuginfod-* headers. -- You are receiving this mail because: You are on the CC list for the bug.
[Bug debuginfod/29469] Add /buildid/BUILDID/header WEBAPI
https://sourceware.org/bugzilla/show_bug.cgi?id=29469 Mark Wielaard changed: What|Removed |Added CC|mjw at fedoraproject dot org |mark at klomp dot org --- Comment #1 from Mark Wielaard --- Do you mean support for an HTTP HEAD (instead of GET) request? Or do you actually want/need a new URL ending in /header to get the information? -- You are receiving this mail because: You are on the CC list for the bug.
[Bug libdw/29450] run-low_high_pc.sh fails on i386 against binutils-2.39
https://sourceware.org/bugzilla/show_bug.cgi?id=29450 Sergei Trofimovich changed: What|Removed |Added Resolution|--- |MOVED Status|UNCONFIRMED |RESOLVED --- Comment #8 from Sergei Trofimovich --- Yeah. The gas fix fixed elfutils test suite for me. Let's mark it RESOLVED/MOVED. -- You are receiving this mail because: You are on the CC list for the bug.
[Bug debuginfod/29469] Add /buildid/BUILDID/header WEBAPI
https://sourceware.org/bugzilla/show_bug.cgi?id=29469 --- Comment #2 from Martin Liska --- (In reply to Mark Wielaard from comment #1) > Do you mean support for an HTTP HEAD (instead of GET) request? Yep, that's what I'm seeking for :) > Or do you actually want/need a new URL ending in /header to get the > information? $ wget --method=HEAD https://debuginfod.opensuse.org/buildid/929d83693f513cefd1c38e5075f2b10b12a9f322/debuginfo Spider mode enabled. Check if remote file exists. --2022-08-11 11:16:46-- https://debuginfod.opensuse.org/buildid/929d83693f513cefd1c38e5075f2b10b12a9f322/debuginfo Resolving debuginfod.opensuse.org (debuginfod.opensuse.org)... 91.193.113.65, 2a07:de40:401::65 Connecting to debuginfod.opensuse.org (debuginfod.opensuse.org)|91.193.113.65|:443... connected. HTTP request sent, awaiting response... 400 Bad Request Remote file does not exist -- broken link!!! -- You are receiving this mail because: You are on the CC list for the bug.
[Bug debuginfod/29469] Add /buildid/BUILDID/header WEBAPI
https://sourceware.org/bugzilla/show_bug.cgi?id=29469 --- Comment #3 from Mark Wielaard --- (In reply to Martin Liska from comment #2) > (In reply to Mark Wielaard from comment #1) > > Do you mean support for an HTTP HEAD (instead of GET) request? > > Yep, that's what I'm seeking for :) > > > Or do you actually want/need a new URL ending in /header to get the > > information? > > $ wget --method=HEAD > https://debuginfod.opensuse.org/buildid/ > 929d83693f513cefd1c38e5075f2b10b12a9f322/debuginfo > Spider mode enabled. Check if remote file exists. > --2022-08-11 11:16:46-- > https://debuginfod.opensuse.org/buildid/ > 929d83693f513cefd1c38e5075f2b10b12a9f322/debuginfo > Resolving debuginfod.opensuse.org (debuginfod.opensuse.org)... > 91.193.113.65, 2a07:de40:401::65 > Connecting to debuginfod.opensuse.org > (debuginfod.opensuse.org)|91.193.113.65|:443... connected. > HTTP request sent, awaiting response... 400 Bad Request > Remote file does not exist -- broken link!!! Right, that is this code: if (string(method) != "GET") throw reportable_exception(400, "we support GET only"); in debuginfod.cxx (handler_cb). I think if you just change that to also allow "HEAD" it will just work. But it does mean the debuginfod server might do more work than strictly necessary. But libmicrohttpd should just send the headers and ignore the body content. (Sorry, haven't actually tested any of this, I am currently on a desert island, ehe mountain, with spotty interweb access and compute resources) -- You are receiving this mail because: You are on the CC list for the bug.
[Bug debuginfod/28284] support description functionality through HEAD
https://sourceware.org/bugzilla/show_bug.cgi?id=28284 Frank Ch. Eigler changed: What|Removed |Added CC||mliska at suse dot cz --- Comment #4 from Frank Ch. Eigler --- *** Bug 29469 has been marked as a duplicate of this bug. *** -- You are receiving this mail because: You are on the CC list for the bug.
[Bug debuginfod/29469] Add /buildid/BUILDID/header WEBAPI
https://sourceware.org/bugzilla/show_bug.cgi?id=29469 Frank Ch. Eigler changed: What|Removed |Added Resolution|--- |DUPLICATE Status|UNCONFIRMED |RESOLVED --- Comment #4 from Frank Ch. Eigler --- this need has been identified in another RFE, with some work toward a webapi and c api *** This bug has been marked as a duplicate of bug 28284 *** -- You are receiving this mail because: You are on the CC list for the bug.
[Bug debuginfod/29472] New: Support querying the debuginfod-server for metadata
https://sourceware.org/bugzilla/show_bug.cgi?id=29472
Bug ID: 29472
Summary: Support querying the debuginfod-server for metadata
Product: elfutils
Version: unspecified
Status: NEW
Severity: normal
Priority: P2
Component: debuginfod
Assignee: unassigned at sourceware dot org
Reporter: rgoldber at redhat dot com
CC: elfutils-devel at sourceware dot org
Target Milestone: ---
It would be beneficial to be able to query a debuginfod-server for its metadata
matching certain conditions. This would allow for operations like seeing what
executable files are available without attempting to download them or seeing
which files are larger than a certain size, which could be used to
prefetch+cache large required files, before the user needs them.
The proposed format is as follows:
debuginfod-find [OPTION]... metadata CONDITION QUERYFMT
Query all of the metadata matching the given CONDITION and return it as
described by QUERYFMT
The supported debuginfod-fields are:
BUILDID, FILENAME, MODIFIED_TIME, SIZE, TYPE, SOURCE_TYPE, SOURCE
and should be referenced using {...}, noting that field names are case
insensitive.
CONDITION is a boolean expression composed of debuginfod-fields, constants
(strings and integers)
and the: `( ) = != < > <= >= && ||` operators, following the standard
conditional precedence order
QUERYFMT is a modified version of the standard printf(3) formatting.
The format is made up of static strings (which may include standard C character
escapes for new‐
lines, tabs, and other special characters (not including \0)) and printf(3)
type formatters. In
place of the type specifier in the format string, use the debuginfod-field you
wish to query.
For example:
debuginfod-find metadata "{FILENAME}=/usr/bin/grep && {TYPE}=E"
"{FILENAME}({BUILDID})\t{SOURCE_TYPE}\n"
Will query all executables with the name /usr/bin/grep and might return the
following
/usr/bin/grep(90e7d8894b94f47ad17722ff8658f833f329b035)R
/usr/bin/grep(e81e4e6e322030178260ae4f6055f781cd4997e1)F
debuginfod-find metadata "{FILENAME}=/bin/bash" "{FILENAME}-{TYPE}\n"
Might return
/bin/bash-E
/bin/bash-S
/bin/bash-D
--
You are receiving this mail because:
You are on the CC list for the bug.
[Bug tools/28729] After eu-ar -r added file into archive, it changes file permission
https://sourceware.org/bugzilla/show_bug.cgi?id=28729
--- Comment #2 from panxiaohe ---
When eu-ar generates archive files，numbers of target file's permission which
are writen in archive file are octal numbers. But "eu -ar -tv archive" and
"eu-ar -x archive" commands treat numbers as decimalism. So it seems that "eu
-ar -tv archive" and "eu-ar -x archive" commands change file permission.
The following file is the example of archive file. 100600(octal number) is the
permission of test1.txt.
# cat test.a
!
test1.txt/ 1659685099 0 0 100600 14 `
dalfjdalflala
Anylze file permission in libelf/elf_begin.c.
-
968 #define INT_FIELD(FIELD)
\
969 do
\
970 {
\
971 char buf[sizeof (ar_hdr->FIELD) + 1];
\
972 const char *string = ar_hdr->FIELD;
\
973 if (ar_hdr->FIELD[sizeof (ar_hdr->FIELD) - 1] != ' ')
\
974 {
\
975 *((char *) mempcpy (buf, ar_hdr->FIELD, sizeof (ar_hdr->FIELD)))
\
976 = '\0';
\
977 string = buf;
\
978 }
\
979 if (sizeof (elf_ar_hdr->FIELD) <= sizeof (long int))
\
980 elf_ar_hdr->FIELD = (__typeof (elf_ar_hdr->FIELD)) atol (string);
\
981 else
\
982 elf_ar_hdr->FIELD = (__typeof (elf_ar_hdr->FIELD)) atoll (string);
\
983 }
\
984 while (0)
985
986 INT_FIELD (ar_date);
987 INT_FIELD (ar_uid);
988 INT_FIELD (ar_gid);
989 INT_FIELD (ar_mode);
990 INT_FIELD (ar_size);
-
Use file permission in libelf/elf_begin.c.
-
551 printf ("%c%c%c%c%c%c%c%c%c %u/%u %6ju %s %s\n",
552 (arhdr->ar_mode & S_IRUSR) ? 'r' : '-',
553 (arhdr->ar_mode & S_IWUSR) ? 'w' : '-',
554 (arhdr->ar_mode & S_IXUSR)
555 ? ((arhdr->ar_mode & S_ISUID) ? 's' : 'x')
556 : ((arhdr->ar_mode & S_ISUID) ? 'S' : '-'),
557 (arhdr->ar_mode & S_IRGRP) ? 'r' : '-',
558 (arhdr->ar_mode & S_IWGRP) ? 'w' : '-',
559 (arhdr->ar_mode & S_IXGRP)
560 ? ((arhdr->ar_mode & S_ISGID) ? 's' : 'x')
561 : ((arhdr->ar_mode & S_ISGID) ? 'S' : '-'),
562 (arhdr->ar_mode & S_IROTH) ? 'r' : '-',
563 (arhdr->ar_mode & S_IWOTH) ? 'w' : '-',
564 (arhdr->ar_mode & S_IXOTH)
565 ? ((arhdr->ar_mode & S_ISVTX) ? 't' : 'x')
566 : ((arhdr->ar_mode & S_ISVTX) ? 'T' : '-'),
567 arhdr->ar_uid,
568 arhdr->ar_gid,
569 (uintmax_t) arhdr->ar_size,
570 datestr,
571 arhdr->ar_name);
-
--
You are receiving this mail because:
You are on the CC list for the bug.
