https://github.com/GeorgeHuyubo created
https://github.com/llvm/llvm-project/pull/134563
This PR try to store the download time statistic with a FileSpec, which can be
later used when we report statistic to report the time we used to download the
file. For now, only debuginfod symbol locator is using it, can be easily extend
to other plugins to track download time for each downloaded file.
Sample statistic dump output after this change:
```
Command: statistics dump
===============Output===============
{
"commands": {
"command container add": 1,
"command script add": 51,
"command script import": 59,
"statistics dump": 1,
"target create": 1,
"type summary add": 36,
"type synthetic add": 21
},
"memory": {
"strings": {
"bytesTotal": 3145728,
"bytesUnused": 1554746,
"bytesUsed": 1590982
}
},
"modules": [
{
"debugInfoByteSize": 244905,
"debugInfoEnabled": true,
"debugInfoHadIncompleteTypes": false,
"debugInfoHadVariableErrors": false,
"debugInfoIndexLoadedFromCache": false,
"debugInfoIndexSavedToCache": false,
"debugInfoIndexTime": 1.0277849999999999,
"debugInfoParseTime": 0.00036000000000000002,
"identifier": 94819923889600,
"path":
"/home/hyubo/.cache/llvm-debuginfod/client/llvmcache-e9243b31f158441000d1e6310bd82104bdbe362b-c4crasher",
"symbolDownloadTime": 2.1573409999999997,
"symbolTableIndexTime": 0.0034009999999999999,
"symbolTableLoadedFromCache": false,
"symbolTableParseTime": 0.0055459999999999997,
"symbolTableSavedToCache": false,
"symbolTableStripped": false,
"triple": "x86_64--linux",
"uuid": "E9243B31-F158-4410-00D1-E6310BD82104-BDBE362B"
},
{
"debugInfoByteSize": 0,
"debugInfoEnabled": false,
"debugInfoHadIncompleteTypes": false,
"debugInfoHadVariableErrors": false,
"debugInfoIndexLoadedFromCache": false,
"debugInfoIndexSavedToCache": false,
"debugInfoIndexTime": 0,
"debugInfoParseTime": 0,
"identifier": 94819937577008,
"path": "[vdso](0x00007ffdca346000)",
"symbolDownloadTime": 0,
"symbolTableIndexTime": 2.6999999999999999e-05,
"symbolTableLoadedFromCache": false,
"symbolTableParseTime": 0.00044299999999999998,
"symbolTableSavedToCache": false,
"symbolTableStripped": false,
"triple": "x86_64--linux",
"uuid": "4D1F38BD-BD34-DFB3-C9A5-B49A2A912219-AC713763"
},
{
"debugInfoByteSize": 0,
"debugInfoEnabled": false,
"debugInfoHadIncompleteTypes": false,
"debugInfoHadVariableErrors": false,
"debugInfoIndexLoadedFromCache": false,
"debugInfoIndexSavedToCache": false,
"debugInfoIndexTime": 0,
"debugInfoParseTime": 0,
"identifier": 94819924328256,
"path": "/usr/local/fbcode/platform010/lib/libc.so.6",
"symbolDownloadTime": 0,
"symbolTableIndexTime": 0.0091170000000000001,
"symbolTableLoadedFromCache": false,
"symbolTableParseTime": 0.086711999999999997,
"symbolTableSavedToCache": false,
"symbolTableStripped": false,
"triple": "x86_64--linux",
"uuid": "ACE9DF01-8872-3A35-6D14-3C92527EF739-4BE32C75"
},
{
"debugInfoByteSize": 0,
"debugInfoEnabled": false,
"debugInfoHadIncompleteTypes": false,
"debugInfoHadVariableErrors": false,
"debugInfoIndexLoadedFromCache": false,
"debugInfoIndexSavedToCache": false,
"debugInfoIndexTime": 0,
"debugInfoParseTime": 0,
"identifier": 94819937844080,
"path": "/usr/local/fbcode/platform010/lib/libm.so.6",
"symbolDownloadTime": 0,
"symbolTableIndexTime": 0.0026340000000000001,
"symbolTableLoadedFromCache": false,
"symbolTableParseTime": 0.022778,
"symbolTableSavedToCache": false,
"symbolTableStripped": false,
"triple": "x86_64--linux",
"uuid": "85932B54-0DE7-4FC1-23B0-FB09AD1A5A61-8E1098B7"
},
{
"debugInfoByteSize": 0,
"debugInfoEnabled": false,
"debugInfoHadIncompleteTypes": false,
"debugInfoHadVariableErrors": false,
"debugInfoIndexLoadedFromCache": false,
"debugInfoIndexSavedToCache": false,
"debugInfoIndexTime": 0,
"debugInfoParseTime": 0,
"identifier": 94819939787376,
"path": "/usr/local/fbcode/platform010/lib/libmvec.so.1",
"symbolDownloadTime": 0,
"symbolTableIndexTime": 0.00055500000000000005,
"symbolTableLoadedFromCache": false,
"symbolTableParseTime": 0.0034259999999999998,
"symbolTableSavedToCache": false,
"symbolTableStripped": false,
"triple": "x86_64--linux",
"uuid": "4537CA22-8E6E-495B-7A03-FC2CEDCAD71C-8A7B2067"
},
{
"debugInfoByteSize": 71978,
"debugInfoEnabled": true,
"debugInfoHadIncompleteTypes": false,
"debugInfoHadVariableErrors": false,
"debugInfoIndexLoadedFromCache": false,
"debugInfoIndexSavedToCache": false,
"debugInfoIndexTime": 0.320467,
"debugInfoParseTime": 0.0051050000000000002,
"identifier": 94819941522944,
"path": "/usr/local/fbcode/platform010/lib/libatomic.so.1",
"symbolDownloadTime": 0,
"symbolFilePath":
"/usr/local/fbcode/platform010/lib/libatomic.so.1.2.0.debuginfo",
"symbolTableIndexTime": 0.00030699999999999998,
"symbolTableLoadedFromCache": false,
"symbolTableParseTime": 0.0032499999999999999,
"symbolTableSavedToCache": false,
"symbolTableStripped": false,
"triple": "x86_64--",
"uuid": "F90F9111-BBD1-C2A9-972A-34EB75ABE62E-3BDED9CF"
},
{
"debugInfoByteSize": 450101,
"debugInfoEnabled": true,
"debugInfoHadIncompleteTypes": false,
"debugInfoHadVariableErrors": false,
"debugInfoIndexLoadedFromCache": false,
"debugInfoIndexSavedToCache": false,
"debugInfoIndexTime": 0.33496100000000001,
"debugInfoParseTime": 0.011282,
"identifier": 94819938686464,
"path": "/usr/local/fbcode/platform010/lib/libgcc_s.so.1",
"symbolDownloadTime": 0,
"symbolFilePath":
"/usr/local/fbcode/platform010/lib/libgcc_s.so.1.debuginfo",
"symbolTableIndexTime": 0.00039199999999999999,
"symbolTableLoadedFromCache": false,
"symbolTableParseTime": 0.003754,
"symbolTableSavedToCache": false,
"symbolTableStripped": false,
"triple": "x86_64--",
"uuid": "1C715A92-40BE-BE95-E148-1B401749BAB8-15D09F9D"
},
{
"debugInfoByteSize": 16485198,
"debugInfoEnabled": true,
"debugInfoHadIncompleteTypes": false,
"debugInfoHadVariableErrors": false,
"debugInfoIndexLoadedFromCache": false,
"debugInfoIndexSavedToCache": false,
"debugInfoIndexTime": 0.39336700000000002,
"debugInfoParseTime": 0.37287100000000001,
"identifier": 94819924448176,
"path": "/usr/local/fbcode/platform010/lib/libstdc++.so.6",
"symbolDownloadTime": 0,
"symbolFilePath":
"/usr/local/fbcode/platform010/lib/libstdc++.so.6.0.29.debuginfo",
"symbolTableIndexTime": 0.040395,
"symbolTableLoadedFromCache": false,
"symbolTableParseTime": 0.086318000000000006,
"symbolTableSavedToCache": false,
"symbolTableStripped": false,
"triple": "x86_64--",
"uuid": "C9DAEE53-F607-981B-6BED-F2042833BFC7-71A1E66C"
},
{
"debugInfoByteSize": 0,
"debugInfoEnabled": false,
"debugInfoHadIncompleteTypes": false,
"debugInfoHadVariableErrors": false,
"debugInfoIndexLoadedFromCache": false,
"debugInfoIndexSavedToCache": false,
"debugInfoIndexTime": 0,
"debugInfoParseTime": 0,
"identifier": 94819926907648,
"path": "/usr/local/fbcode/platform010/lib/ld.so",
"symbolDownloadTime": 0,
"symbolTableIndexTime": 0.00054799999999999998,
"symbolTableLoadedFromCache": false,
"symbolTableParseTime": 0.0058370000000000002,
"symbolTableSavedToCache": false,
"symbolTableStripped": false,
"triple": "x86_64--linux",
"uuid": "CCA86CF4-E4FF-42C8-7056-2F7D8B83AEE0-530B4095"
}
],
"targets": [
{
"breakpoints": [],
"dyldPluginName": "dump-modulelist-dyld",
"expressionEvaluation": {
"failures": 0,
"successes": 0
},
"frameVariable": {
"failures": 0,
"successes": 0
},
"mismatchCoredumpModuleCount": 0,
"moduleIdentifiers": [
94819923889600,
94819937577008,
94819924328256,
94819937844080,
94819939787376,
94819941522944,
94819938686464,
94819924448176,
94819926907648
],
"signals": [
{
"SIGABRT": 1
}
],
"sourceMapDeduceCount": 0,
"sourceRealpathAttemptCount": 0,
"sourceRealpathCompatibleCount": 0,
"stopCount": 1,
"summaryProviderStatistics": [],
"targetCreateTime": 4.0000000000000003e-05,
"totalBreakpointResolveTime": 0,
"totalSharedLibraryEventHitCount": 0
}
],
"totalDebugInfoByteSize": 17252182,
"totalDebugInfoEnabled": 4,
"totalDebugInfoIndexLoadedFromCache": 0,
"totalDebugInfoIndexSavedToCache": 0,
"totalDebugInfoIndexTime": 2.0765799999999999,
"totalDebugInfoParseTime": 0.38961800000000002,
"totalModuleCount": 9,
"totalModuleCountHasDebugInfo": 4,
"totalModuleCountWithIncompleteTypes": 0,
"totalModuleCountWithVariableErrors": 0,
"totalSymbolDownloadTime": 2.1573409999999997,
"totalSymbolTableIndexTime": 0.057376000000000003,
"totalSymbolTableParseTime": 0.21806400000000001,
"totalSymbolTableStripped": 0,
"totalSymbolTablesLoadedFromCache": 0,
"totalSymbolTablesSavedToCache": 0,
"transcript": [
{
"command": "symsrv",
"commandArguments": "",
"commandName": "symsrv",
"durationInSeconds": 0.00066600000000000003,
"error": "",
"output": "",
"timestampInEpochSeconds": 1743824829
},
{
"command": "settings set symbols.enable-external-lookup true",
"commandArguments": "symbols.enable-external-lookup true",
"commandName": "settings set",
"durationInSeconds": 3.8000000000000002e-05,
"error": "",
"output": "",
"timestampInEpochSeconds": 1743824829
},
{
"command": "settings insert-before
plugin.symbol-locator.debuginfod.server-urls 0
https://our.intern.facebook.com/intern/debuginfod";,
"commandArguments": "plugin.symbol-locator.debuginfod.server-urls 0
https://our.intern.facebook.com/intern/debuginfod";,
"commandName": "settings insert-before",
"durationInSeconds": 5.0000000000000002e-05,
"error": "",
"output": "",
"timestampInEpochSeconds": 1743824829
},
{
"command": "target create --core
\"/var/tmp/cores/c4crasher.crasher.3277089\"",
"commandArguments": "--core \"/var/tmp/cores/c4crasher.crasher.3277089\"",
"commandName": "target create",
"durationInSeconds": 8.309571,
"error": "",
"output": "Core file '/var/tmp/cores/c4crasher.crasher.3277089' (x86_64)
was loaded.\n",
"timestampInEpochSeconds": 1743824832
},
{
"command": "fbpaste statistics dump",
"commandArguments": "statistics dump",
"commandName": "fbpaste",
"timestampInEpochSeconds": 1743824843
},
{
"command": "statistics dump",
"commandArguments": "",
"commandName": "statistics dump",
"timestampInEpochSeconds": 1743824843
}
]
}
```
>From 7125e9340250b719927628a978117cf1ba66c11f Mon Sep 17 00:00:00 2001
From: George Hu <georgehuy...@gmail.com>
Date: Fri, 4 Apr 2025 13:37:36 -0700
Subject: [PATCH] Add download time for each module in statistics
---
lldb/include/lldb/Symbol/SymbolFile.h | 5 +++
lldb/include/lldb/Target/Statistics.h | 1 +
lldb/include/lldb/Utility/FileSpec.h | 8 +++++
.../SymbolFile/DWARF/SymbolFileDWARF.cpp | 7 ++++
.../SymbolFile/DWARF/SymbolFileDWARF.h | 3 ++
.../Debuginfod/SymbolLocatorDebuginfod.cpp | 32 ++++++++++++-------
lldb/source/Target/Statistics.cpp | 8 +++++
lldb/source/Utility/FileSpec.cpp | 6 ++++
8 files changed, 59 insertions(+), 11 deletions(-)
diff --git a/lldb/include/lldb/Symbol/SymbolFile.h
b/lldb/include/lldb/Symbol/SymbolFile.h
index f35d3ee9f22ae..29fa99ba18323 100644
--- a/lldb/include/lldb/Symbol/SymbolFile.h
+++ b/lldb/include/lldb/Symbol/SymbolFile.h
@@ -422,6 +422,11 @@ class SymbolFile : public PluginInterface {
/// hasn't been indexed yet, or a valid duration if it has.
virtual StatsDuration::Duration GetDebugInfoIndexTime() { return {}; }
+ /// Return the time it took to download any extra symbol files.
+ ///
+ /// \returns 0.0 if no extra symbol files need to be downloaded
+ virtual double GetSymbolDownloadTime() { return 0.0; }
+
/// Reset the statistics for the symbol file.
virtual void ResetStatistics() {}
diff --git a/lldb/include/lldb/Target/Statistics.h
b/lldb/include/lldb/Target/Statistics.h
index ee365357fcf31..2d3a7fcceec53 100644
--- a/lldb/include/lldb/Target/Statistics.h
+++ b/lldb/include/lldb/Target/Statistics.h
@@ -122,6 +122,7 @@ struct ModuleStats {
double symtab_index_time = 0.0;
double debug_parse_time = 0.0;
double debug_index_time = 0.0;
+ double symbol_download_time = 0.0;
uint64_t debug_info_size = 0;
bool symtab_loaded_from_cache = false;
bool symtab_saved_to_cache = false;
diff --git a/lldb/include/lldb/Utility/FileSpec.h
b/lldb/include/lldb/Utility/FileSpec.h
index 3fa89b1dcff28..b59fa299c7f9b 100644
--- a/lldb/include/lldb/Utility/FileSpec.h
+++ b/lldb/include/lldb/Utility/FileSpec.h
@@ -75,6 +75,9 @@ class FileSpec {
/// \see FileSpec::SetFile (const char *path)
explicit FileSpec(llvm::StringRef path, Style style = Style::native);
+ explicit FileSpec(llvm::StringRef path, const double download_time,
+ Style style = Style::native);
+
explicit FileSpec(llvm::StringRef path, const llvm::Triple &triple);
bool DirectoryEquals(const FileSpec &other) const;
@@ -416,6 +419,9 @@ class FileSpec {
/// The lifetime of the StringRefs is tied to the lifetime of the FileSpec.
std::vector<llvm::StringRef> GetComponents() const;
+ /// Get the download time of the file.
+ double GetDownloadTime() { return m_download_time; }
+
protected:
// Convenience method for setting the file without changing the style.
void SetFile(llvm::StringRef path);
@@ -430,6 +436,8 @@ class FileSpec {
No
};
+ /// The download time of the file.
+ double m_download_time = 0.0;
/// The unique'd directory path.
ConstString m_directory;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index b95159d882bc7..8afb7b9dfb01c 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -4349,6 +4349,13 @@ LanguageType
SymbolFileDWARF::GetLanguageFamily(DWARFUnit &unit) {
return LanguageTypeFromDWARF(lang);
}
+double SymbolFileDWARF::GetSymbolDownloadTime() {
+ double time = GetObjectFile()->GetFileSpec().GetDownloadTime();
+ if (m_dwp_symfile)
+ time += m_dwp_symfile->GetObjectFile()->GetFileSpec().GetDownloadTime();
+ return time;
+}
+
StatsDuration::Duration SymbolFileDWARF::GetDebugInfoIndexTime() {
if (m_index)
return m_index->GetIndexTime();
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
index 7309f7a86b659..16f609ee10807 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
@@ -309,6 +309,9 @@ class SymbolFileDWARF : public SymbolFileCommon {
StatsDuration::Duration GetDebugInfoParseTime() override {
return m_parse_time;
}
+
+ double GetSymbolDownloadTime() override;
+
StatsDuration::Duration GetDebugInfoIndexTime() override;
StatsDuration &GetDebugInfoParseTimeRef() { return m_parse_time; }
diff --git
a/lldb/source/Plugins/SymbolLocator/Debuginfod/SymbolLocatorDebuginfod.cpp
b/lldb/source/Plugins/SymbolLocator/Debuginfod/SymbolLocatorDebuginfod.cpp
index f9aa6b1a98765..9b6b72ef20816 100644
--- a/lldb/source/Plugins/SymbolLocator/Debuginfod/SymbolLocatorDebuginfod.cpp
+++ b/lldb/source/Plugins/SymbolLocator/Debuginfod/SymbolLocatorDebuginfod.cpp
@@ -188,17 +188,27 @@ GetFileForModule(const ModuleSpec &module_spec,
std::string cache_file_name = llvm::toHex(build_id, true);
if (!file_name.empty())
cache_file_name += "-" + file_name.str();
- llvm::Expected<std::string> result = llvm::getCachedOrDownloadArtifact(
- cache_file_name, url_path, cache_path, debuginfod_urls, timeout);
- if (result)
- return FileSpec(*result);
-
- Log *log = GetLog(LLDBLog::Symbols);
- auto err_message = llvm::toString(result.takeError());
- LLDB_LOGV(log,
- "Debuginfod failed to download symbol artifact {0} with error {1}",
- url_path, err_message);
- return {};
+ StatsDuration duration;
+ ElapsedTime elapased(duration);
+ std::string local_path;
+ {
+ ElapsedTime elapsed(duration);
+ llvm::Expected<std::string> result = llvm::getCachedOrDownloadArtifact(
+ cache_file_name, url_path, cache_path, debuginfod_urls, timeout);
+
+ if (!result) {
+ Log *log = GetLog(LLDBLog::Symbols);
+ auto err_message = llvm::toString(result.takeError());
+ LLDB_LOGV(
+ log,
+ "Debuginfod failed to download symbol artifact {0} with error {1}",
+ url_path, err_message);
+ return {};
+ }
+ local_path = *result;
+ }
+
+ return FileSpec(local_path, duration.get().count());
}
std::optional<ModuleSpec> SymbolLocatorDebuginfod::LocateExecutableObjectFile(
diff --git a/lldb/source/Target/Statistics.cpp
b/lldb/source/Target/Statistics.cpp
index b5d2e7bda1edf..73e96b6df36ec 100644
--- a/lldb/source/Target/Statistics.cpp
+++ b/lldb/source/Target/Statistics.cpp
@@ -71,6 +71,7 @@ json::Value ModuleStats::ToJSON() const {
module.try_emplace("debugInfoHadIncompleteTypes",
debug_info_had_incomplete_types);
module.try_emplace("symbolTableStripped", symtab_stripped);
+ module.try_emplace("symbolDownloadTime", symbol_download_time);
if (!symfile_path.empty())
module.try_emplace("symbolFilePath", symfile_path);
@@ -288,6 +289,7 @@ llvm::json::Value DebuggerStats::ReportStatistics(
json::Array json_targets;
json::Array json_modules;
+ double symbol_download_time = 0.0;
double symtab_parse_time = 0.0;
double symtab_index_time = 0.0;
double debug_parse_time = 0.0;
@@ -345,6 +347,10 @@ llvm::json::Value DebuggerStats::ReportStatistics(
++debug_index_saved;
module_stat.debug_index_time = sym_file->GetDebugInfoIndexTime().count();
module_stat.debug_parse_time = sym_file->GetDebugInfoParseTime().count();
+ module_stat.symbol_download_time += sym_file->GetSymbolDownloadTime();
+ if (sym_file->GetObjectFile() != module->GetObjectFile())
+ module_stat.symbol_download_time +=
+ module->GetObjectFile()->GetFileSpec().GetDownloadTime();
module_stat.debug_info_size =
sym_file->GetDebugInfoSize(load_all_debug_info);
module_stat.symtab_stripped = module->GetObjectFile()->IsStripped();
@@ -361,6 +367,7 @@ llvm::json::Value DebuggerStats::ReportStatistics(
if (module_stat.debug_info_had_variable_errors)
++num_modules_with_variable_errors;
}
+ symbol_download_time += module_stat.symbol_download_time;
symtab_parse_time += module_stat.symtab_parse_time;
symtab_index_time += module_stat.symtab_index_time;
debug_parse_time += module_stat.debug_parse_time;
@@ -391,6 +398,7 @@ llvm::json::Value DebuggerStats::ReportStatistics(
}
json::Object global_stats{
+ {"totalSymbolDownloadTime", symbol_download_time},
{"totalSymbolTableParseTime", symtab_parse_time},
{"totalSymbolTableIndexTime", symtab_index_time},
{"totalSymbolTablesLoadedFromCache", symtabs_loaded},
diff --git a/lldb/source/Utility/FileSpec.cpp b/lldb/source/Utility/FileSpec.cpp
index bb2b8647342b8..89c3eba44c306 100644
--- a/lldb/source/Utility/FileSpec.cpp
+++ b/lldb/source/Utility/FileSpec.cpp
@@ -72,6 +72,12 @@ FileSpec::FileSpec(llvm::StringRef path, Style style) :
m_style(style) {
SetFile(path, style);
}
+FileSpec::FileSpec(llvm::StringRef path, const double download_time,
+ Style style)
+ : m_download_time(download_time) {
+ SetFile(path, style);
+}
+
FileSpec::FileSpec(llvm::StringRef path, const llvm::Triple &triple)
: FileSpec{path, triple.isOSWindows() ? Style::windows : Style::posix} {}
_______________________________________________
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
