Hi Frank,
And finally the debuginfod server code itself.
On Wed, 2023-04-12 at 16:31 -0400, Frank Ch. Eigler via Elfutils-devel
wrote:
> + * debuginfod.cxx (DEBUGINFOD_SQLITE_DDL): Add an index or two.
> + (metadata_maxtime_s, parse_opt): New parameter for load control.
> + (add_client_federation_headers): New refactored function.
> + (handle_metadata): New function.
> + (handler_cb): Call it for /metadata URL. Trace it.
> + (groom): Tweak sqlite_ps object lifetimes.
> [...]
> diff --git a/debuginfod/debuginfod.cxx b/debuginfod/debuginfod.cxx
> index 5ef6cc32189b..000820fec5ea 100644
> --- a/debuginfod/debuginfod.cxx
> +++ b/debuginfod/debuginfod.cxx
> @@ -1,5 +1,5 @@
> /* Debuginfo-over-http server.
> - Copyright (C) 2019-2021 Red Hat, Inc.
> + Copyright (C) 2019-2023 Red Hat, Inc.
> Copyright (C) 2021, 2022 Mark J. Wielaard
> This file is part of elfutils.
>
> @@ -68,6 +68,7 @@ extern "C" {
> #include
> #include
> #include
> +#include
>
>
> /* If fts.h is included before config.h, its indirect inclusions may not
> @@ -127,6 +128,9 @@ using namespace std;
> #define tid() pthread_self()
> #endif
>
> +#ifdef HAVE_JSON_C
> + #include
> +#endif
>
> inline bool
> string_endswith(const string& haystack, const string& needle)
> @@ -185,7 +189,7 @@ static const char DEBUGINFOD_SQLITE_DDL[] =
>"foreign key (buildid) references " BUILDIDS "_buildids(id) on
> update cascade on delete cascade,\n"
>"primary key (buildid, file, mtime)\n"
>") " WITHOUT_ROWID ";\n"
> - // Index for faster delete by file identifier
> + // Index for faster delete by file identifier and metadata searches
>"create index if not exists " BUILDIDS "_f_de_idx on " BUILDIDS "_f_de
> (file, mtime);\n"
>"create table if not exists " BUILDIDS "_f_s (\n"
>"buildid integer not null,\n"
> @@ -211,6 +215,8 @@ static const char DEBUGINFOD_SQLITE_DDL[] =
>") " WITHOUT_ROWID ";\n"
>// Index for faster delete by archive file identifier
>"create index if not exists " BUILDIDS "_r_de_idx on " BUILDIDS "_r_de
> (file, mtime);\n"
> + // Index for metadata searches
> + "create index if not exists " BUILDIDS "_r_de_idx2 on " BUILDIDS "_r_de
> (content);\n"
>"create table if not exists " BUILDIDS "_r_sref (\n" // outgoing dwarf
> sourcefile references from rpm
>"buildid integer not null,\n"
>"artifactsrc integer not null,\n"
OK, and extra index on content in the _r_de table. Which is an index
into the _files table.
In general I find is hard to keep the whole sql structure in my head.
Is this described somewhere apart from just reading the whole
DEBUGINFOD_SQLITE_DDL?
> @@ -398,6 +404,9 @@ static const struct argp_option options[] =
> { "passive", ARGP_KEY_PASSIVE, NULL, 0, "Do not scan or groom, read-only
> database.", 0 },
> #define ARGP_KEY_DISABLE_SOURCE_SCAN 0x1009
> { "disable-source-scan", ARGP_KEY_DISABLE_SOURCE_SCAN, NULL, 0, "Do not
> scan dwarf source info.", 0 },
> +#define ARGP_KEY_METADATA_MAXTIME 0x100A
> + { "metadata-maxtime", ARGP_KEY_METADATA_MAXTIME, "SECONDS", 0,
> + "Number of seconds to limit metadata query run time, 0=unlimited.", 0 },
> { NULL, 0, NULL, 0, NULL, 0 },
>};
Ack. New --metadata-maxtime argument.
> @@ -452,6 +461,8 @@ static unsigned forwarded_ttl_limit = 8;
> static bool scan_source_info = true;
> static string tmpdir;
> static bool passive_p = false;
> +static unsigned metadata_maxtime_s = 5;
> +
OK. Defaults to 5 seconds.
> static void set_metric(const string& key, double value);
> // static void inc_metric(const string& key);
> @@ -653,6 +664,9 @@ parse_opt (int key, char *arg,
> case ARGP_KEY_DISABLE_SOURCE_SCAN:
>scan_source_info = false;
>break;
> +case ARGP_KEY_METADATA_MAXTIME:
> + metadata_maxtime_s = (unsigned) atoi(arg);
> + break;
>// case 'h': argp_state_help (state, stderr,
> ARGP_HELP_LONG|ARGP_HELP_EXIT_OK);
> default: return ARGP_ERR_UNKNOWN;
> }
OK. Can be set by user.
> @@ -2040,6 +2054,58 @@ handle_buildid_r_match (bool internal_req_p,
>return r;
> }
>
> +void
> +add_client_federation_headers(debuginfod_client *client, MHD_Connection*
> conn){
> + // Transcribe incoming User-Agent:
> + string ua = MHD_lookup_connection_value (conn, MHD_HEADER_KIND,
> "User-Agent") ?: "";
> + string ua_complete = string("User-Agent: ") + ua;
> + debuginfod_add_http_header (client, ua_complete.c_str());
> +
> + // Compute larger XFF:, for avoiding info loss during
> + // federation, and for future cyclicity detection.
> + string xff = MHD_lookup_connection_value (conn, MHD_HEADER_KIND,
> "X-Forwarded-For") ?: "";
> + if (xff != "")
> +xff += string(", "); // comma separated list
> +
> + unsigned int xff_count = 0;
> + for (auto&& i : xff){
> +if (i == ',') xff_count++;
> + }
> +
> + // if X-Forwarded-For: exceeds N hops,
> + //
