Hi,
Here is a patch for a new one-shot mode for the debuginfod server. In this
mode the first scanning pass of the server will also output the found
executables' paths and buildids to the given file descriptor. This can (and
soon will be used in systemtap) as an easy way to scan archive files and
quickly/easily extract information concerning the executables they contain.
The patch passes all of the tests across the try-buildbots (branch
users/rgoldber/try-one-shot)
Thanks,
Ryan (rgoldber)
From a4e5d1bbd39d2448e5ae208a114e3a68d9f2c28a Mon Sep 17 00:00:00 2001
From: Ryan Goldberg <rgoldber@redhat.com>
Date: Mon, 8 Aug 2022 13:30:08 -0400
Subject: [PATCH] One-shot mode for server
When in one-shot mode, for the first
scan all of the found executables' paths
and buildids will be written to the provided
file descriptor in the given format.
Signed-off-by: Ryan Goldberg <rgoldber@redhat.com>
---
configure.ac | 5 ++
debuginfod/ChangeLog | 9 +++
debuginfod/Makefile.am | 2 +-
debuginfod/debuginfod.cxx | 52 ++++++++++++++-
debuginfod/debuginfod.h.in | 9 +++
doc/ChangeLog | 4 ++
doc/debuginfod.8 | 16 +++++
tests/ChangeLog | 7 ++-
tests/Makefile.am | 5 +-
tests/run-debuginfod-one-shot.sh | 105 +++++++++++++++++++++++++++++++
10 files changed, 209 insertions(+), 5 deletions(-)
create mode 100755 tests/run-debuginfod-one-shot.sh
diff --git a/configure.ac b/configure.ac
index 03b67a9d..aadff4ca 100644
--- a/configure.ac
+++ b/configure.ac
@@ -600,6 +600,11 @@ case "$ac_cv_search__obstack_free" in
esac
AC_SUBST([obstack_LIBS])
+AC_CHECK_LIB(json-c, json_tokener_parse, [
+ AC_DEFINE([ENABLE_ONE_SHOT], [1], [Define if the one-shot feature is enabled])
+ AC_SUBST(jsonc_LIBS, '-ljson-c')
+])
+
dnl The directories with content.
dnl Documentation.
diff --git a/debuginfod/ChangeLog b/debuginfod/ChangeLog
index a8b5d7f4..b6fac996 100644
--- a/debuginfod/ChangeLog
+++ b/debuginfod/ChangeLog
@@ -1,3 +1,12 @@
+2022-08-09 Ryan Goldberg <rgoldber@redhat.com>
+
+ * debuginfod.cxx (archive_classify): New global (one_shot_mtx).
+ (archive_classify, thread_main_fts_source_paths) In one-shot mode
+ for the first scan, output found buildids/paths to given file descriptor.
+ (parse_opt): Add "--one-shot" flag.
+ * debuginfod.h.in: Added in debuginfod_oneshot_header struct,
+ debuginfod_oneshot_header_magic
+
2022-08-02 Josef Cejka <jcejka@suse.de>
* debuginfod.cxx (groom): Don't evaluate regex unless needed.
diff --git a/debuginfod/Makefile.am b/debuginfod/Makefile.am
index 435cb8a6..03cdeed0 100644
--- a/debuginfod/Makefile.am
+++ b/debuginfod/Makefile.am
@@ -70,7 +70,7 @@ bin_PROGRAMS += debuginfod-find
endif
debuginfod_SOURCES = debuginfod.cxx
-debuginfod_LDADD = $(libdw) $(libelf) $(libeu) $(libdebuginfod) $(argp_LDADD) $(fts_LIBS) $(libmicrohttpd_LIBS) $(sqlite3_LIBS) $(libarchive_LIBS) -lpthread -ldl
+debuginfod_LDADD = $(libdw) $(libelf) $(libeu) $(libdebuginfod) $(argp_LDADD) $(fts_LIBS) $(libmicrohttpd_LIBS) $(sqlite3_LIBS) $(libarchive_LIBS) $(jsonc_LIBS) -lpthread -ldl
debuginfod_find_SOURCES = debuginfod-find.c
debuginfod_find_LDADD = $(libdw) $(libelf) $(libeu) $(libdebuginfod) $(argp_LDADD) $(fts_LIBS)
diff --git a/debuginfod/debuginfod.cxx b/debuginfod/debuginfod.cxx
index a089d0bd..04c5f635 100644
--- a/debuginfod/debuginfod.cxx
+++ b/debuginfod/debuginfod.cxx
@@ -54,10 +54,12 @@ extern "C" {
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/vfs.h>
-#include <unistd.h>
#include <fcntl.h>
#include <netdb.h>
+#ifdef ENABLE_ONE_SHOT
+#include <json-c/json.h>
+#endif
/* If fts.h is included before config.h, its indirect inclusions may not
give us the right LFS aliases of these functions, so map them manually. */
@@ -387,6 +389,8 @@ static const struct argp_option options[] =
{ "passive", ARGP_KEY_PASSIVE, NULL, 0, "Do not scan or groom, read-only database.", 0 },
#define ARGP_KEY_DISABLE_SOURCE_SCAN 0x1009
{ "disable-source-scan", ARGP_KEY_DISABLE_SOURCE_SCAN, NULL, 0, "Do not scan dwarf source info.", 0 },
+#define ARGP_KEY_ONE_SHOT 0x100a
+ { "one-shot", ARGP_KEY_ONE_SHOT, "NUM", 0, "For the first scan, output found buildids/paths to given file descriptor", 0 },
{ NULL, 0, NULL, 0, NULL, 0 },
};
@@ -439,6 +443,8 @@ static unsigned forwarded_ttl_limit = 8;
static bool scan_source_info = true;
static string tmpdir;
static bool passive_p = false;
+static int one_shot_fd = -1;
+static mutex one_shot_mtx;
static void set_metric(const string& key, double value);
// static void inc_metric(const string& key);
@@ -639,6 +645,16 @@ parse_opt (int key, char *arg,
// other conflicting options tricky to check
argp_failure(state, 1, EINVAL, "inconsistent options with passive mode");
break;
+ case ARGP_KEY_ONE_SHOT:
+ one_shot_fd = atol(arg);
+ #ifdef ENABLE_ONE_SHOT
+ if (one_shot_fd < 0)
+ argp_failure(state, 1, EINVAL, "one-shot fd");
+ #else
+ argp_failure(state, 1, EINVAL, "Debuginfod in one-shot mode requires libjson-c");
+ #endif
+
+ break;
case ARGP_KEY_DISABLE_SOURCE_SCAN:
scan_source_info = false;
break;
@@ -3110,6 +3126,31 @@ archive_classify (const string& rps, string& archive_extension,
<< (debuginfo_p ? "D" : "")
<< " sourcefiles=" << sourcefiles.size() << endl;
+ #ifdef ENABLE_ONE_SHOT
+ if(one_shot_fd >= 0 && executable_p && "" != fn && "" != buildid){
+ unique_lock<mutex> lock(one_shot_mtx);
+ struct debuginfod_oneshot_header hdr;
+ hdr.magic = debuginfod_oneshot_header_magic;
+ string json_path = "\"path\": \""+fn+"\"";
+ string json_buildid = " \"buildid\": \""+buildid+"\"";
+
+ // Converts the strings to a properly escaped JSON string
+ json_object *json = json_tokener_parse((
+ "{"+
+ json_path+","+
+ json_buildid+
+ "}"
+ ).c_str());
+ const char* payload = json_object_to_json_string(json);
+
+ hdr.payload_len = strlen(payload)+1;
+ if( NULL == payload ||
+ sizeof(hdr) != write(one_shot_fd, &hdr, sizeof(hdr)) ||
+ (ssize_t) hdr.payload_len != write(one_shot_fd, payload, hdr.payload_len) )
+ if (verbose) obatched(clog) << "There was an issue writing the path+buildid in one-shot mode" << endl;
+ json_object_put(json);
+ }
+ #endif
}
catch (const reportable_exception& e)
{
@@ -3470,6 +3511,15 @@ thread_main_fts_source_paths (void* arg)
scanq.done_idle(); // release the hounds
if (interrupted) break;
+ #ifdef ENABLE_ONE_SHOT
+ // If doing a one-shot pass, close the pipe once the first scan is done
+ // Then resume the normal state
+ if(last_rescan != 0 && one_shot_fd >= 0){
+ if(one_shot_fd > 2) close(one_shot_fd);
+ one_shot_fd = -1;
+ }
+ #endif
+
time_t now = time(NULL);
bool rescan_now = false;
if (last_rescan == 0) // at least one initial rescan is documented even for -t0
diff --git a/debuginfod/debuginfod.h.in b/debuginfod/debuginfod.h.in
index c358df4d..3d0c0d49 100644
--- a/debuginfod/debuginfod.h.in
+++ b/debuginfod/debuginfod.h.in
@@ -103,5 +103,14 @@ void debuginfod_end (debuginfod_client *client);
}
#endif
+/* The message format used by debuginfod one-shot mode */
+#include <sys/types.h>
+static const uint debuginfod_oneshot_header_magic = 0xdeb91f0d;
+struct debuginfod_oneshot_header{
+ uint magic;
+ size_t payload_len;
+ char payload[0];
+};
+
#endif /* _DEBUGINFOD_CLIENT_H */
diff --git a/doc/ChangeLog b/doc/ChangeLog
index ceec2467..201a0bdb 100644
--- a/doc/ChangeLog
+++ b/doc/ChangeLog
@@ -1,3 +1,7 @@
+2022-08-09 Ryan Goldberg <rgoldber@redhat.com>
+
+ * debuginfod.8 (--one-shot): Document new flag & operation mode.
+
2022-06-03 Michael Trapp <michael.trapp@sap.com>
* debuginfod.8 (--disable-source-scan): Document.
diff --git a/doc/debuginfod.8 b/doc/debuginfod.8
index 50fce7f5..a905a15d 100644
--- a/doc/debuginfod.8
+++ b/doc/debuginfod.8
@@ -279,6 +279,22 @@ Disable scan of the dwarf source info of debuginfo sections.
If a setup has no access to source code, the source info is not
required.
+.TP
+.B "\-\-one\-shot=FD"
+For the first scan, atomically write the found executables' buildids and
+canonicalized paths in JSON format to the given file descriptor FD.
+The format for each write is as follows:
+.TS
+tab(;) allbox;
+cf s s.
+Magic number (uint) = 0xdeb91f0d
+.T&
+l,al.
+JSON string length (size_t); JSON string (char[])
+.TE
+
+Once completed, FD will be closed and debuginfod will return to the normal state.
+
.TP
.B "\-v"
Increase verbosity of logging to the standard error file descriptor.
diff --git a/tests/ChangeLog b/tests/ChangeLog
index d2952cc9..ee3521af 100644
--- a/tests/ChangeLog
+++ b/tests/ChangeLog
@@ -1,3 +1,9 @@
+2022-08-09 Ryan Goldberg <rgoldber@redhat.com>
+
+ * run-debuginfod-one-shot.sh: New test.
+ * Makefile.am (TESTS): Add run-debuginfod-one-shot.sh.
+ (EXTRA_DIST): Add run-debuginfod-one-shot.sh
+
2022-08-04 Sergei Trofimovich <slyich@gmail.com>
* low_high_pc.c (handle_die): Drop redundant 'lx' suffix.
@@ -10,7 +16,6 @@
* run-readelf-Dd.sh: New test.
* Makefile.am (TESTS): Add run-readelf-Dd.sh.
- (EXTRA_DIST): Likewise.
2022-06-01 Mark Wielaard <mark@klomp.org>
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 87988fb9..20dc2680 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -244,7 +244,8 @@ TESTS += run-debuginfod-dlopen.sh \
run-debuginfod-x-forwarded-for.sh \
run-debuginfod-response-headers.sh \
run-debuginfod-extraction-passive.sh \
- run-debuginfod-webapi-concurrency.sh
+ run-debuginfod-webapi-concurrency.sh \
+ run-debuginfod-one-shot.sh
endif
if !OLD_LIBMICROHTTPD
# Will crash on too old libmicrohttpd
@@ -252,7 +253,6 @@ if !OLD_LIBMICROHTTPD
TESTS += run-debuginfod-federation-metrics.sh
endif
endif
-
EXTRA_DIST = run-arextract.sh run-arsymtest.sh run-ar.sh \
run-show-die-info.sh run-get-files.sh run-get-lines.sh \
run-next-files.sh run-next-lines.sh testfile-only-debug-line.bz2 \
@@ -549,6 +549,7 @@ EXTRA_DIST = run-arextract.sh run-arsymtest.sh run-ar.sh \
run-debuginfod-response-headers.sh \
run-debuginfod-extraction-passive.sh \
run-debuginfod-webapi-concurrency.sh \
+ run-debuginfod-one-shot.sh \
debuginfod-rpms/fedora30/hello2-1.0-2.src.rpm \
debuginfod-rpms/fedora30/hello2-1.0-2.x86_64.rpm \
debuginfod-rpms/fedora30/hello2-debuginfo-1.0-2.x86_64.rpm \
diff --git a/tests/run-debuginfod-one-shot.sh b/tests/run-debuginfod-one-shot.sh
new file mode 100755
index 00000000..185f2e27
--- /dev/null
+++ b/tests/run-debuginfod-one-shot.sh
@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+#
+# Copyright (C) 2019-2021 Red Hat, Inc.
+# This file is part of elfutils.
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# elfutils is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+. $srcdir/debuginfod-subr.sh
+
+DB=${PWD}/.debuginfod_tmp.sqlite
+tempfiles $DB
+
+binoutput="one-shot-output.bin"
+tempfiles $binoutput
+exec 3<>$binoutput # Open FD=3 for RW
+
+# This variable is essential and ensures no time-race for claiming ports occurs
+# set base to a unique multiple of 100 not used in any other 'run-debuginfod-*' test
+base=13100
+get_ports
+mkdir R
+env LD_LIBRARY_PATH=$ldpath DEBUGINFOD_URLS= ${abs_builddir}/../debuginfod/debuginfod $VERBOSE -R \
+ -d $DB -p $PORT1 -t0 -g0 R > vlog$PORT1 2>&1 --one-shot 3 &
+PID1=$!
+tempfiles vlog$PORT1
+errfiles vlog$PORT1
+
+########################################################################
+cp -rvp ${abs_srcdir}/debuginfod-rpms/rhel7 R
+
+correct_output=("/usr/local/bin/hello2:bc1febfd03ca05e030f0d205f7659db29f8a4b30" \
+ "/usr/local/bin/hello:f0aa15b8aba4f3c28cac3c2a73801fefa644a9f2")
+MAGIC="0xdeb91f0d"
+
+cat << EoF > parseData.cxx
+#include <sys/types.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include "${abs_top_builddir}/debuginfod/debuginfod.h"
+#include <stdio.h>
+#include <json-c/json.h>
+
+int main(){
+ int fd = open("$binoutput", 0);
+ if(fd < 0) return 1;
+
+ struct debuginfod_oneshot_header hdr;
+
+ int hdr_sze;
+ while(0 != (hdr_sze = read(fd, &hdr, sizeof(hdr)))){
+ char* payload = (char*)malloc(hdr.payload_len);
+ if(hdr_sze != sizeof(hdr) ||
+ hdr.magic != $MAGIC ||
+ (ssize_t)hdr.payload_len != read(fd, payload, hdr.payload_len))
+ return 1;
+ json_object *json = json_tokener_parse(payload);
+ json_object *json_field;
+ if(!json_object_object_get_ex(json, "path", &json_field)) return 1;
+ printf("%s:",json_object_get_string(json_field));
+ if(!json_object_object_get_ex(json, "buildid", &json_field)) return 1;
+ printf("%s\n",json_object_get_string(json_field));
+ free(payload);
+ json_object_put(json);
+ }
+ close(fd);
+ return 0;
+}
+EoF
+tempfiles parseData.cxx parseData.o tempfiles parseData
+g++ -o parseData -l json-c parseData.cxx || { echo "Can't compile test program"; exit 77; }; # Skip on compilation faliure (probably missing libjson-c.so)
+
+########################################################################
+# Server must become ready with R fully scanned and indexed
+wait_ready $PORT1 'ready' 1
+wait_ready $PORT1 'thread_work_total{role="traverse"}' 1
+wait_ready $PORT1 'thread_work_pending{role="scan"}' 0
+wait_ready $PORT1 'thread_busy{role="scan"}' 0
+
+exec 3>&- # close fd
+tempfiles output.txt
+# Check the header format is correct
+testrun ./parseData > output.txt
+
+cat output.txt
+
+# Check that each of the output lines corresponds to one of the correct outputs
+while read -r line; do
+ printf '%s\0' "${correct_output[@]}" | grep -Fxqz -- $line;
+done < output.txt
+
+kill $PID1
+wait $PID1
+PID1=0
+exit 0
--
2.37.1
