avoiding private headers -

[Brian Dodge via Elfutils-devel]

Hello,

I am attempting to use elfutils / libdw to create a library for tagging 
function's local/automatic variable storage. I hope to use dwarf info to 
get the stack frame offsets, etc. for function's local vars.


I started with readelf.c as an example for iterating through CU/DIE.  
Since my code isn't "part of" elfutils I am not including private 
headers such as libdwP.h.  Because of that some of the code derived from 
readelf won't compile of course.


Are these structure accesses simply short-cuts and it is possible to get 
the info needed via API functions?  For things like "cu->version" it 
seems I can get that via the API dwarf_cu_info, but for some things I 
don't quite see what I'd need in the public headers. For example, in the 
(readelf.c) function get_indexed_addr, the line:


Elf_Data *debug_addr = cu->dbg->sectiondata[IDX_debug_addr];

uses the (private) structure Dwarf_CU to get at the (private) structure 
Dwarf, to get at the sectiondata.  I can get the Dwarf via 
dwarf_cu_getdwarf, but Is there another way to get at the section data 
(and IDX_debug_addr) ?


Thanks and apologies for my newbie-ness here.  Please let me know if 
there is a better email address to send this question to.


Brian




--
This email and any attachments may contain confidential information. 
Unauthorized use or distribution is prohibited. If you are not an intended 
recipient, please promptly advise the sender and kindly delete all copies. 
Thank you.

[PATCH] PR25756 more debuginfod metrics

[Frank Ch. Eigler via Elfutils-devel]

Subject: [PATCH 1/2] PR26756: add more prometheus metrics to debuginfod

From: Frank Ch. Eigler 

Add an error_count{} family of metrics for each libc/libarchive/http
exception instance created during operation.  Add a family of fdcache*
metrics for tracking fdcache operations and status.  Test via a
injecting a permission-000 empty nothing.rpm in the testsuite.

Signed-off-by: Frank Ch. Eigler 
---
 debuginfod/ChangeLog |  6 
 debuginfod/debuginfod.cxx| 56 +++-
 tests/ChangeLog  |  6 
 tests/run-debuginfod-find.sh | 12 ++--
 4 files changed, 63 insertions(+), 17 deletions(-)

diff --git a/debuginfod/ChangeLog b/debuginfod/ChangeLog
index 8cb89967e9d1..236e2ca3270f 100644
--- a/debuginfod/ChangeLog
+++ b/debuginfod/ChangeLog
@@ -1,3 +1,9 @@
+2020-10-20  Frank Ch. Eigler  
+
+   PR26756: more prometheus metrics
+   * debuginfod.cxx (*_exception): Add counters for error occurrences.
+   (fdcache::*): Add counters for fdcache operations and status.
+
 2020-09-18  Frank Ch. Eigler 
 
* debuginfod.cxx (scan_source_file, archive_classify): Store only
diff --git a/debuginfod/debuginfod.cxx b/debuginfod/debuginfod.cxx
index 140b7789de3b..a5334d2a7fc5 100644
--- a/debuginfod/debuginfod.cxx
+++ b/debuginfod/debuginfod.cxx
@@ -548,23 +548,31 @@ struct sqlite_exception: public reportable_exception
 struct libc_exception: public reportable_exception
 {
   libc_exception(int rc, const string& msg):
-reportable_exception(string("libc error: ") + msg + ": " + 
string(strerror(rc) ?: "?")) {}
+reportable_exception(string("libc error: ") + msg + ": " + 
string(strerror(rc) ?: "?")) {
+inc_metric("error_count","libc",strerror(rc));
+  }
 };
 
 
 struct archive_exception: public reportable_exception
 {
   archive_exception(const string& msg):
-reportable_exception(string("libarchive error: ") + msg) {}
+reportable_exception(string("libarchive error: ") + msg) {
+  inc_metric("error_count","libarchive",msg);
+  }
   archive_exception(struct archive* a, const string& msg):
-reportable_exception(string("libarchive error: ") + msg + ": " + 
string(archive_error_string(a) ?: "?")) {}
+reportable_exception(string("libarchive error: ") + msg + ": " + 
string(archive_error_string(a) ?: "?")) {
+inc_metric("error_count","libarchive",msg);
+  }
 };
 
 
 struct elfutils_exception: public reportable_exception
 {
   elfutils_exception(int rc, const string& msg):
-reportable_exception(string("elfutils error: ") + msg + ": " + 
string(elf_errmsg(rc) ?: "?")) {}
+reportable_exception(string("elfutils error: ") + msg + ": " + 
string(elf_errmsg(rc) ?: "?")) {
+inc_metric("error_count","elfutils",elf_errmsg(rc));
+  }
 };
 
 
@@ -1083,6 +1091,15 @@ class libarchive_fdcache
   long max_mbs;
 
 public:
+  void set_metrics()
+  {
+double total_mb = 0.0;
+for (auto i = lru.begin(); i < lru.end(); i++)
+  total_mb += i->fd_size_mb;
+set_metric("fdcache_bytes", (int64_t)(total_mb*1024.0*1024.0));
+set_metric("fdcache_count", lru.size());
+  }
+
   void intern(const string& a, const string& b, string fd, off_t sz, bool 
front_p)
   {
 {
@@ -1093,19 +1110,24 @@ class libarchive_fdcache
 {
   unlink (i->fd.c_str());
   lru.erase(i);
+  inc_metric("fdcache_op_count","op","dequeue");
   break; // must not continue iterating
 }
 }
   double mb = (sz+65535)/1048576.0; // round up to 64K block
   fdcache_entry n = { a, b, fd, mb };
-  if (front_p)
+  if (front_p) {
+inc_metric("fdcache_op_count","op","enqueue_front");
 lru.push_front(n);
-  else
+  } else {
+inc_metric("fdcache_op_count","op","enqueue_back");
 lru.push_back(n);
-if (verbose > 3)
-  obatched(clog) << "fdcache interned a=" << a << " b=" << b
- << " fd=" << fd << " mb=" << mb << " front=" << front_p 
<< endl;
+  }
+  if (verbose > 3)
+obatched(clog) << "fdcache interned a=" << a << " b=" << b
+   << " fd=" << fd << " mb=" << mb << " front=" << front_p 
<< endl;
 }
+set_metrics();
 
 // NB: we age the cache at lookup time too
 if (front_p)
@@ -1124,7 +1146,7 @@ class libarchive_fdcache
   fdcache_entry n = *i;
   lru.erase(i); // invalidates i, so no more iteration!
   lru.push_front(n);
-
+  inc_metric("fdcache_op_count","op","requeue_front");
   fd = open(n.fd.c_str(), O_RDONLY); // NB: no problem if dup() 
fails; looks like cache miss
   break;
 }
@@ -1142,9 +1164,12 @@ class libarchive_fdcache
 unique_lock lock(fdcache_lock);
 for (auto i = lru.begin(); i < lru.end(); i++)
   {
-if (i->archive == a && i->entry == b)
+if (i->archive == a && i->entry == b) {
+  inc_metric("fdcache_op_count","op","probe_hit

[PATCH] debuginfo fdcache prefetching tweak

[Frank Ch. Eigler via Elfutils-devel]

Subject: [PATCH 2/2] debuginfod: suppress fdcache prefetching during dwz
 lookup

From: Frank Ch. Eigler 

During a recent from-scratch reindexing of the rpm/deb corpus at
debuginfod.elfutils.org, we found the fdcache chewed up an abnormal
amount of $TMPDIR space.  This was due to internal .dwz lookups, which
triggered fdcache prefetching as for a webapi query, but there was not
a timely fdcache eviction pass to clean it up again.  Rather than add
that pass, it's better to suppress the prefetching completely, as an
internal .dwz search will only ever need that file, not any others
from the same archive.

Signed-off-by: Frank Ch. Eigler 
---
 debuginfod/ChangeLog  |  5 +
 debuginfod/debuginfod.cxx | 20 +---
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/debuginfod/ChangeLog b/debuginfod/ChangeLog
index 236e2ca3270f..fd67cc8bcccd 100644
--- a/debuginfod/ChangeLog
+++ b/debuginfod/ChangeLog
@@ -1,3 +1,8 @@
+2020-10-20  Frank Ch. Eigler  
+
+   * debuginfod.cxx (handle_buildid*): Add a parameter for detecting
+   internally-originated lookups for dwz resolution.
+
 2020-10-20  Frank Ch. Eigler  
 
PR26756: more prometheus metrics
diff --git a/debuginfod/debuginfod.cxx b/debuginfod/debuginfod.cxx
index a5334d2a7fc5..7f9fde171777 100644
--- a/debuginfod/debuginfod.cxx
+++ b/debuginfod/debuginfod.cxx
@@ -898,10 +898,12 @@ add_mhd_last_modified (struct MHD_Response *resp, time_t 
mtime)
 
 
 static struct MHD_Response*
-handle_buildid_f_match (int64_t b_mtime,
+handle_buildid_f_match (bool internal_req_t,
+int64_t b_mtime,
 const string& b_source0,
 int *result_fd)
 {
+  (void) internal_req_t; // ignored
   int fd = open(b_source0.c_str(), O_RDONLY);
   if (fd < 0)
 {
@@ -1254,7 +1256,8 @@ string canonicalized_archive_entry_pathname(struct 
archive_entry *e)
 
 
 static struct MHD_Response*
-handle_buildid_r_match (int64_t b_mtime,
+handle_buildid_r_match (bool internal_req_p,
+int64_t b_mtime,
 const string& b_source0,
 const string& b_source1,
 int *result_fd)
@@ -1358,7 +1361,8 @@ handle_buildid_r_match (int64_t b_mtime,
   // 3) extract some number of prefetched entries (just into fdcache)
   // 4) abort any further processing
   struct MHD_Response* r = 0; // will set in stage 2
-  unsigned prefetch_count = fdcache_prefetch; // will decrement in stage 3
+  unsigned prefetch_count =
+internal_req_p ? 0 : fdcache_prefetch;// will decrement in stage 3
 
   while(r == 0 || prefetch_count > 0) // stage 1, 2, or 3
 {
@@ -1452,16 +1456,17 @@ handle_buildid_r_match (int64_t b_mtime,
 
 
 static struct MHD_Response*
-handle_buildid_match (int64_t b_mtime,
+handle_buildid_match (bool internal_req_p,
+  int64_t b_mtime,
   const string& b_stype,
   const string& b_source0,
   const string& b_source1,
   int *result_fd)
 {
   if (b_stype == "F")
-return handle_buildid_f_match(b_mtime, b_source0, result_fd);
+return handle_buildid_f_match(internal_req_p, b_mtime, b_source0, 
result_fd);
   else if (b_stype == "R")
-return handle_buildid_r_match(b_mtime, b_source0, b_source1, result_fd);
+return handle_buildid_r_match(internal_req_p, b_mtime, b_source0, 
b_source1, result_fd);
   else
 return 0;
 }
@@ -1559,7 +1564,8 @@ handle_buildid (MHD_Connection* conn,
 
   // Try accessing the located match.
   // XXX: in case of multiple matches, attempt them in parallel?
-  auto r = handle_buildid_match (b_mtime, b_stype, b_source0, b_source1, 
result_fd);
+  auto r = handle_buildid_match (conn ? false : true,
+ b_mtime, b_stype, b_source0, b_source1, 
result_fd);
   if (r)
 return r;
 }
-- 
2.26.2