clayborg created this revision. clayborg added reviewers: jingham, labath, JDevlieghere, wallace, aadsm. Herald added subscribers: mgorny, emaste. clayborg requested review of this revision. Herald added subscribers: lldb-commits, MaskRay. Herald added a project: LLDB.
Added the ability to cache the finalized symbol tables as a proof of concept and as a way to discuss caching items for a given module to allow subsequent debug sessions to start faster. Module caching must be enabled by the user before this can be used: (lldb) settings set symbols.enable-lldb-modules-cache true There is also a setting that allows the user to specify a module cache directory that defaults to a directory that defaults to being next to the symbols.clang-modules-cache-path directory in a temp directory: (lldb) settings show symbols.lldb-modules-cache-path If this setting is enabled, the finalized symbol tables will be serialized and saved to disc so they can be quickly loaded next time you debug. Each module gets a cache directory in the "symbols.lldb-modules-cache-path" directory. A directory is created in the cache directory that uses the module's basename as a directory and then a hash is created for the module which incorporates the full path to the executable, the architecture, the object name offset and modification time (if any). This allows universal mach-o files to support caching multuple architectures in the same module cache directory. Making the directory based on the this info allows this directory's contents to be deleted and replaced when the file gets updated on disk. This keeps the cache from growing over time during the compile/edit/debug cycle and prevents out of space issues. If the cache is enabled, the symbol table will be loaded from the cache the next time you debug if the module has not changed. Repository: rG LLVM Github Monorepo https://reviews.llvm.org/D113789 Files: lldb/include/lldb/Core/Mangled.h lldb/include/lldb/Core/Module.h lldb/include/lldb/Core/ModuleList.h lldb/include/lldb/Host/FileSystem.h lldb/include/lldb/Symbol/ObjectFile.h lldb/include/lldb/Symbol/Symbol.h lldb/include/lldb/Symbol/Symtab.h lldb/packages/Python/lldbsuite/test/lldbtest.py lldb/source/API/SBDebugger.cpp lldb/source/Core/CoreProperties.td lldb/source/Core/Mangled.cpp lldb/source/Core/Module.cpp lldb/source/Core/ModuleList.cpp lldb/source/Host/common/FileSystem.cpp lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp lldb/source/Symbol/CMakeLists.txt lldb/source/Symbol/ObjectFile.cpp lldb/source/Symbol/Symbol.cpp lldb/source/Symbol/Symtab.cpp lldb/test/API/functionalities/module_cache/bsd/Makefile lldb/test/API/functionalities/module_cache/bsd/TestModuleCacheBSD.py lldb/test/API/functionalities/module_cache/bsd/a.c lldb/test/API/functionalities/module_cache/bsd/b.c lldb/test/API/functionalities/module_cache/bsd/c.c lldb/test/API/functionalities/module_cache/bsd/main.c lldb/test/API/functionalities/module_cache/simple_exe/Makefile lldb/test/API/functionalities/module_cache/simple_exe/TestModuleCacheSimple.py lldb/test/API/functionalities/module_cache/simple_exe/main.c lldb/test/API/functionalities/module_cache/universal/Makefile lldb/test/API/functionalities/module_cache/universal/TestModuleCacheUniversal.py lldb/test/API/functionalities/module_cache/universal/main.c lldb/test/API/macosx/universal/main.c lldb/unittests/Symbol/CMakeLists.txt lldb/unittests/Symbol/SymbolTest.cpp
Index: lldb/unittests/Symbol/SymbolTest.cpp =================================================================== --- /dev/null +++ lldb/unittests/Symbol/SymbolTest.cpp @@ -0,0 +1,261 @@ +//===-- SymbolTest.cpp ----------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "gtest/gtest.h" + +#include "lldb/Core/Section.h" +#include "lldb/Symbol/Symbol.h" +#include "lldb/Symbol/Symtab.h" +#include "lldb/Utility/DataExtractor.h" + +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/GSYM/FileWriter.h" +#include "llvm/Support/raw_ostream.h" + +using namespace lldb; +using namespace lldb_private; + +static llvm::support::endianness GetLLVMByteOrder(bool little_endian) { + return little_endian ? llvm::support::little : llvm::support::big; +} + +static ByteOrder GetLLDBByteOrder(bool little_endian) { + return little_endian ? eByteOrderLittle : eByteOrderBig; +} + +static void EncodeDecode(const Symbol &object, const SectionList *sect_list, + bool little_endian) { + llvm::SmallString<512> str; + llvm::raw_svector_ostream strm(str); + llvm::gsym::FileWriter file(strm, GetLLVMByteOrder(little_endian)); + object.Encode(file); + llvm::StringRef bytes = strm.str(); + DataExtractor data(bytes.data(), bytes.size(), + GetLLDBByteOrder(little_endian), 4); + Symbol decoded_object; + offset_t data_offset = 0; + decoded_object.Decode(data, &data_offset, sect_list); + EXPECT_EQ(object, decoded_object); +} + +static void EncodeDecode(const Symbol &object, const SectionList *sect_list) { + EncodeDecode(object, sect_list, /*little_endian=*/true); + EncodeDecode(object, sect_list, /*little_endian=*/false); +} + +TEST(SymbolTest, EncodeDecodeSymbol) { + + SectionSP sect_sp(new Section( + /*module_sp=*/ModuleSP(), + /*obj_file=*/nullptr, + /*sect_id=*/1, + /*name=*/ConstString(".text"), + /*sect_type=*/eSectionTypeCode, + /*file_vm_addr=*/0x1000, + /*vm_size=*/0x1000, + /*file_offset=*/0, + /*file_size=*/0, + /*log2align=*/5, + /*flags=*/0x10203040)); + + SectionList sect_list; + sect_list.AddSection(sect_sp); + + Symbol symbol( + /*symID=*/0x10203040, + /*name=*/"main", + /*type=*/eSymbolTypeCode, + /*bool external=*/false, + /*bool is_debug=*/false, + /*bool is_trampoline=*/false, + /*bool is_artificial=*/false, + /*section_sp=*/sect_sp, + /*offset=*/0x0, + /*size=*/0x100, + /*size_is_valid=*/true, + /*contains_linker_annotations=*/false, + /*flags=*/0x11223344); + + // Test encoding a symbol with an address. + EncodeDecode(symbol, §_list); + + // Test that encoding the bits in the bitfield works for all endianness + // combos. + + // Test Symbol.m_is_synthetic + symbol.SetIsSynthetic(true); + EncodeDecode(symbol, §_list); + symbol.SetIsSynthetic(false); + + // Test Symbol.m_is_debug + symbol.SetDebug(true); + EncodeDecode(symbol, §_list); + symbol.SetDebug(false); + + // Test Symbol.m_is_external + symbol.SetExternal(true); + EncodeDecode(symbol, §_list); + symbol.SetExternal(false); + + // Test Symbol.m_size_is_sibling + symbol.SetSizeIsSibling(true); + EncodeDecode(symbol, §_list); + symbol.SetSizeIsSibling(false); + + // Test Symbol.m_size_is_synthesized + symbol.SetSizeIsSynthesized(true); + EncodeDecode(symbol, §_list); + symbol.SetSizeIsSynthesized(false); + + // Test Symbol.m_size_is_synthesized + symbol.SetByteSize(0); + EncodeDecode(symbol, §_list); + symbol.SetByteSize(0x100); + + // Test Symbol.m_demangled_is_synthesized + symbol.SetDemangledNameIsSynthesized(true); + EncodeDecode(symbol, §_list); + symbol.SetDemangledNameIsSynthesized(false); + + // Test Symbol.m_contains_linker_annotations + symbol.SetContainsLinkerAnnotations(true); + EncodeDecode(symbol, §_list); + symbol.SetContainsLinkerAnnotations(false); + + // Test Symbol.m_is_weak + symbol.SetIsWeak(true); + EncodeDecode(symbol, §_list); + symbol.SetIsWeak(false); + + // Test encoding a symbol with no address. + symbol.GetAddressRef().SetSection(SectionSP()); + EncodeDecode(symbol, §_list); +} + +static void EncodeDecode(const Mangled &object, bool little_endian) { + llvm::SmallString<512> str; + llvm::raw_svector_ostream strm(str); + llvm::gsym::FileWriter file(strm, GetLLVMByteOrder(little_endian)); + object.Encode(file); + llvm::StringRef bytes = strm.str(); + DataExtractor data(bytes.data(), bytes.size(), + GetLLDBByteOrder(little_endian), 4); + Mangled decoded_object; + offset_t data_offset = 0; + decoded_object.Decode(data, &data_offset); + EXPECT_EQ(object, decoded_object); +} + +static void EncodeDecode(const Mangled &object) { + EncodeDecode(object, /*little_endian=*/true); + EncodeDecode(object, /*little_endian=*/false); +} + +TEST(SymbolTest, EncodeDecodeMangled) { + Mangled mangled; + // Test encoding and decoding an empty mangled object. + EncodeDecode(mangled); + + // Test encoding a mangled object that hasn't demangled its name yet. + mangled.SetMangledName(ConstString("_Z3fooi")); + EncodeDecode(mangled); + + // Test encoding a mangled object that has demangled its name by computing it. + mangled.GetDemangledName(); + // EncodeDecode(mangled); + + // Test encoding a mangled object that has just a demangled name + mangled.SetMangledName(ConstString()); + mangled.SetDemangledName(ConstString("hello")); + EncodeDecode(mangled); + + // Test encoding a mangled name that has both a mangled and demangled name + // that are not mangled/demangled counterparts of each other. + mangled.SetMangledName(ConstString("world")); + EncodeDecode(mangled); +} + +static void EncodeDecode(const Symtab &object, bool little_endian) { + llvm::SmallString<512> str; + llvm::raw_svector_ostream strm(str); + llvm::gsym::FileWriter file(strm, GetLLVMByteOrder(little_endian)); + object.Encode(file); + llvm::StringRef bytes = strm.str(); + DataExtractor data(bytes.data(), bytes.size(), + GetLLDBByteOrder(little_endian), 4); + Symtab decoded_object(nullptr); + offset_t data_offset = 0; + decoded_object.Decode(data, &data_offset); + ASSERT_EQ(object.GetNumSymbols(), decoded_object.GetNumSymbols()); + for (size_t i = 0; i < object.GetNumSymbols(); ++i) + EXPECT_EQ(*object.SymbolAtIndex(i), *decoded_object.SymbolAtIndex(i)); +} + +static void EncodeDecode(const Symtab &object) { + EncodeDecode(object, /*little_endian=*/true); + EncodeDecode(object, /*little_endian=*/false); +} + +TEST(SymbolTest, EncodeDecodeSymtab) { + Symtab symtab(/*objfile=*/nullptr); + + Symbol symbol1( + /*symID=*/1, + /*name=*/"symbol1", + /*type=*/eSymbolTypeCode, + /*bool external=*/false, + /*bool is_debug=*/false, + /*bool is_trampoline=*/false, + /*bool is_artificial=*/false, + /*section_sp=*/SectionSP(), + /*value=*/0x1001101010000000, + /*size=*/0, + /*size_is_valid=*/false, + /*contains_linker_annotations=*/false, + /*flags=*/0x10011010); + + Symbol symbol2( + /*symID=*/2, + /*name=*/"symbol2", + /*type=*/eSymbolTypeData, + /*bool external=*/false, + /*bool is_debug=*/false, + /*bool is_trampoline=*/false, + /*bool is_artificial=*/false, + /*section_sp=*/SectionSP(), + /*value=*/0x2002202020000000, + /*size=*/0, + /*size_is_valid=*/false, + /*contains_linker_annotations=*/false, + /*flags=*/0x20022020); + + Symbol symbol3( + /*symID=*/3, + /*name=*/"symbol3", + /*type=*/eSymbolTypeCode, + /*bool external=*/false, + /*bool is_debug=*/false, + /*bool is_trampoline=*/false, + /*bool is_artificial=*/false, + /*section_sp=*/SectionSP(), + /*value=*/0x3003303030000000, + /*size=*/0, + /*size_is_valid=*/false, + /*contains_linker_annotations=*/false, + /*flags=*/0x30033030); + + // Test encoding and decoding an empty symbol table. + EncodeDecode(symtab); + + symtab.AddSymbol(symbol1); + symtab.AddSymbol(symbol2); + symtab.AddSymbol(symbol3); + // Test encoding and decoding a symbol table with 3 symbols. + EncodeDecode(symtab); +} Index: lldb/unittests/Symbol/CMakeLists.txt =================================================================== --- lldb/unittests/Symbol/CMakeLists.txt +++ lldb/unittests/Symbol/CMakeLists.txt @@ -1,6 +1,7 @@ add_lldb_unittest(SymbolTests LocateSymbolFileTest.cpp PostfixExpressionTest.cpp + SymbolTest.cpp TestTypeSystem.cpp TestTypeSystemClang.cpp TestClangASTImporter.cpp Index: lldb/test/API/macosx/universal/main.c =================================================================== --- lldb/test/API/macosx/universal/main.c +++ lldb/test/API/macosx/universal/main.c @@ -1,21 +1,3 @@ -#include <stdio.h> -#include <unistd.h> -#include <string.h> - -void -call_me() -{ - sleep(1); -} - -int -main (int argc, char **argv) -{ - printf ("Hello there!\n"); // Set break point at this line. - if (argc == 2 && strcmp(argv[1], "keep_waiting") == 0) - while (1) - { - call_me(); - } +int main (int argc, char **argv) { return 0; } Index: lldb/test/API/functionalities/module_cache/universal/TestModuleCacheUniversal.py =================================================================== --- /dev/null +++ lldb/test/API/functionalities/module_cache/universal/TestModuleCacheUniversal.py @@ -0,0 +1,58 @@ +"""Test the LLDB module cache funcionality for universal mach-o files.""" + +import glob +import lldb +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbutil +import os +import time + + +class ModuleCacheTestcaseUniversal(TestBase): + + mydir = TestBase.compute_mydir(__file__) + + def setUp(self): + # Call super's setUp(). + TestBase.setUp(self) + # Find the line number in a(int) to break at. + self.cache_dir = os.path.join(self.getBuildDir(), 'lldb-module-cache') + print('build-dir: ' + self.getBuildDir()) + # Set the lldb module cache directory to a directory inside the build + # artifacts directory so no other tests are interfered with. + self.runCmd('settings set symbols.lldb-modules-cache-path "%s"' % (self.cache_dir)) + self.runCmd('settings set symbols.enable-lldb-modules-cache true') + self.build() + + + def get_module_cache_dirs(self, basename): + module_cache_glob = os.path.join(self.cache_dir, basename, "*") + return glob.glob(module_cache_glob) + + + # Doesn't depend on any specific debug information. + @no_debug_info_test + @skipUnlessDarwin + @skipIfDarwinEmbedded # this test file assumes we're targetting an x86 system + def test(self): + """ + Test module cache functionality for a universal mach-o files. + + This will test that if we enable the module cache, we can create + lldb module caches for each slice of a universal mach-o file and + they will each have a unique directory. + """ + exe_basename = "testit" + exe = self.getBuildArtifact(exe_basename) + + # Create a module with no depedencies. + self.runCmd('target create -d --arch x86_64 %s' % (exe)) + self.runCmd('image dump symtab %s' % (exe_basename)) + self.runCmd('target create -d --arch arm64 %s' % (exe)) + self.runCmd('image dump symtab %s' % (exe_basename)) + + cache_dirs = self.get_module_cache_dirs(exe_basename) + + self.assertEqual(len(cache_dirs), 2, + "make sure there are two files in the module cache directory for %s" % (exe_basename)) Index: lldb/test/API/functionalities/module_cache/universal/Makefile =================================================================== --- /dev/null +++ lldb/test/API/functionalities/module_cache/universal/Makefile @@ -0,0 +1,20 @@ +EXE := testit + +include Makefile.rules + +all: testit + +testit: testit.x86_64 testit.arm64 + lipo -create -o testit $^ + +testit.arm64: testit.arm64.o + $(CC) -isysroot $(SDKROOT) -target arm64-apple-macosx10.9 -o testit.arm64 $< + +testit.x86_64: testit.x86_64.o + $(CC) -isysroot $(SDKROOT) -target x86_64-apple-macosx10.9 -o testit.x86_64 $< + +testit.arm64.o: main.c + $(CC) -isysroot $(SDKROOT) -g -O0 -target arm64-apple-macosx10.9 -c -o testit.arm64.o $< + +testit.x86_64.o: main.c + $(CC) -isysroot $(SDKROOT) -g -O0 -target x86_64-apple-macosx10.9 -c -o testit.x86_64.o $< Index: lldb/test/API/functionalities/module_cache/simple_exe/main.c =================================================================== --- /dev/null +++ lldb/test/API/functionalities/module_cache/simple_exe/main.c @@ -0,0 +1,2 @@ +int main (int argc, char const *argv[]) { +} Index: lldb/test/API/functionalities/module_cache/simple_exe/TestModuleCacheSimple.py =================================================================== --- /dev/null +++ lldb/test/API/functionalities/module_cache/simple_exe/TestModuleCacheSimple.py @@ -0,0 +1,136 @@ +"""Test the LLDB module cache funcionality.""" + +import glob +import lldb +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbutil +import os +import time + + +class ModuleCacheTestcase(TestBase): + + mydir = TestBase.compute_mydir(__file__) + + def setUp(self): + # Call super's setUp(). + TestBase.setUp(self) + # Find the line number in a(int) to break at. + self.cache_dir = os.path.join(self.getBuildDir(), 'lldb-module-cache') + # Set the lldb module cache directory to a directory inside the build + # artifacts directory so no other tests are interfered with. + self.runCmd('settings set symbols.lldb-modules-cache-path "%s"' % (self.cache_dir)) + self.runCmd('settings set symbols.enable-lldb-modules-cache true') + self.build() + + + def get_module_cache_dir(self, basename): + module_cache_glob = os.path.join(self.cache_dir, basename, "*") + module_cache_files = glob.glob(module_cache_glob) + self.assertEqual(len(module_cache_files), 1, + "make sure the module cache has only 1 directory") + return module_cache_files[0] + + + # Doesn't depend on any specific debug information. + @no_debug_info_test + def test(self): + """ + Test module cache functionality for a simple object file. + + This will test that if we enable the module cache, we have a + corresponding cache entry for the executable with a "info.json" file + that has a modification time that matches that of the executable and + that there is a symbol table cache created as well. It also removes + the executable, rebuilds so that the modification time of the binary + gets updated, and then creates a new target and causes the cache to + get updated. + """ + exe = self.getBuildArtifact("a.out") + + # Create a module with no depedencies. + target = self.createTestTarget(load_dependent_modules=False) + + # Get the executable module and get the number of symbols to make + # sure the symbol table gets parsed and cached. The module cache is + # enabled in the setUp() function. + main_module = target.GetModuleAtIndex(0) + self.assertTrue(main_module.IsValid()) + # Make sure the symbol table gets loaded and cached + main_module.GetNumSymbols() + cache_dir = self.get_module_cache_dir("a.out") + cache_dir_files = os.listdir(cache_dir) + self.assertEqual(len(cache_dir_files), 2, + "make sure there are two files in the module cache directory") + # Make sure the "info.json" file is in the cache directory + info_json_path = None + symtab_cache_path = None + found_info_json = False + found_symtab_cache = False + for basename in cache_dir_files: + if basename == "info.json": + found_info_json = True + info_json_path = os.path.join(cache_dir, basename) + if basename.startswith("symtab-"): + found_symtab_cache = True + symtab_cache_path = os.path.join(cache_dir, basename) + + self.assertEqual(found_info_json, True, + "check for info.json in executable cache dir") + self.assertEqual(found_symtab_cache, True, + "check for info.json in executable cache dir") + exe_mtime_1 = os.path.getmtime(exe) + info_mtime_1 = os.path.getmtime(info_json_path) + symtab_mtime_1 = os.path.getmtime(symtab_cache_path) + self.assertEqual(exe_mtime_1, + info_mtime_1, + "check that the 'info.json' modification time matches the executable modification time") + # Now remove the executable and sleep for a few seconds to make sure we + # get a different creation and modification time for the file since some + # OSs store the modification time in seconds since Jan 1, 1970. + os.remove(exe) + self.assertEqual(os.path.exists(exe), False, + 'make sure we were able to remove the executable') + time.sleep(2) + # Now rebuild the binary so it has a different modification time. + self.build() + self.assertEqual(os.path.exists(exe), True, + 'make sure executable exists after rebuild') + # Make sure the modification time has changed or this test will fail. + exe_mtime_2 = os.path.getmtime(exe) + self.assertNotEqual( + exe_mtime_1, + exe_mtime_2, + "make sure the modification time of the executable has changed") + # Makre sure the module cache still has an out of date cache. + self.assertNotEqual(exe_mtime_2, + os.path.getmtime(info_json_path), + "check that the 'info.json' modification time doesn't match the executable modification time after rebuild") + # Create a new target and get the symbols again, and make sure the cache + # gets updated. All files in the module cache except the "info.json" + # file should get deleted and re-created. + target = self.createTestTarget(load_dependent_modules=False) + main_module = target.GetModuleAtIndex(0) + self.assertTrue(main_module.IsValid()) + main_module.GetNumSymbols() + self.assertEqual(os.path.exists(info_json_path), True, + 'make sure "info.json" exists after cache is updated') + self.assertEqual(os.path.exists(symtab_cache_path), True, + 'make sure "info.json" exists after cache is updated') + + info_mtime_2 = os.path.getmtime(info_json_path) + symtab_mtime_2 = os.path.getmtime(symtab_cache_path) + + self.assertNotEqual( + info_mtime_1, + info_mtime_2, + 'make sure modification time of "info.json" changed') + self.assertNotEqual( + symtab_mtime_1, + symtab_mtime_2, + 'make sure modification time of "symtab-..." changed') + self.assertEqual( + info_mtime_2, + exe_mtime_2, + "check that the 'info.json' modification time matches the executable modification time after rebuild and cache update") Index: lldb/test/API/functionalities/module_cache/simple_exe/Makefile =================================================================== --- /dev/null +++ lldb/test/API/functionalities/module_cache/simple_exe/Makefile @@ -0,0 +1,3 @@ +C_SOURCES := main.c + +include Makefile.rules Index: lldb/test/API/functionalities/module_cache/bsd/main.c =================================================================== --- /dev/null +++ lldb/test/API/functionalities/module_cache/bsd/main.c @@ -0,0 +1,11 @@ +#include <stdio.h> + +extern int a(int); +extern int b(int); +extern int c(int); +int main (int argc, char const *argv[]) +{ + printf ("a(1) returns %d\n", a(1)); + printf ("b(2) returns %d\n", b(2)); + printf ("c(2) returns %d\n", c(2)); +} Index: lldb/test/API/functionalities/module_cache/bsd/c.c =================================================================== --- /dev/null +++ lldb/test/API/functionalities/module_cache/bsd/c.c @@ -0,0 +1,6 @@ +static int __c_global = 3; + +int c(int arg) { + int result = arg + __c_global; + return result; // Set file and line breakpoint inside c(). +} Index: lldb/test/API/functionalities/module_cache/bsd/b.c =================================================================== --- /dev/null +++ lldb/test/API/functionalities/module_cache/bsd/b.c @@ -0,0 +1,6 @@ +static int __b_global = 2; + +int b(int arg) { + int result = arg + __b_global; + return result; // Set file and line breakpoint inside b(). +} Index: lldb/test/API/functionalities/module_cache/bsd/a.c =================================================================== --- /dev/null +++ lldb/test/API/functionalities/module_cache/bsd/a.c @@ -0,0 +1,6 @@ +int __a_global = 1; + +int a(int arg) { + int result = arg + __a_global; + return result; // Set file and line breakpoint inside a(). +} Index: lldb/test/API/functionalities/module_cache/bsd/TestModuleCacheBSD.py =================================================================== --- /dev/null +++ lldb/test/API/functionalities/module_cache/bsd/TestModuleCacheBSD.py @@ -0,0 +1,85 @@ +"""Test the LLDB module cache funcionality.""" + +import glob +import lldb +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbutil +import os +import time + + +class ModuleCacheTestcase(TestBase): + + mydir = TestBase.compute_mydir(__file__) + + def setUp(self): + # Call super's setUp(). + TestBase.setUp(self) + # Find the line number in a(int) to break at. + self.line_a = line_number( + 'a.c', '// Set file and line breakpoint inside a().') + self.line_b = line_number( + 'b.c', '// Set file and line breakpoint inside b().') + self.line_c = line_number( + 'c.c', '// Set file and line breakpoint inside c().') + self.cache_dir = os.path.join(self.getBuildDir(), 'lldb-module-cache') + # Set the lldb module cache directory to a directory inside the build + # artifacts directory so no other tests are interfered with. + self.runCmd('settings set symbols.lldb-modules-cache-path "%s"' % (self.cache_dir)) + self.runCmd('settings set symbols.enable-lldb-modules-cache true') + self.build() + + + def get_module_cache_dirs(self, basename, object): + module_cache_glob = os.path.join(self.cache_dir, basename, object, "*") + return glob.glob(module_cache_glob) + + + # Requires no dSYM, so we let the Makefile make the right stuff for us + @no_debug_info_test + @skipUnlessDarwin + def test(self): + """ + Test module cache functionality for bsd archive object files. + + This will test that if we enable the module cache, we have a + corresponding cache entry for the .o files in libfoo.a. + + The static library has two entries for "a.o": + - one from a.c + - one from c.c which had c.o renamed to a.o and then put into the + libfoo.a as an extra .o file with different contents from the + original a.o + + We do this to test that we can correctly cache duplicate .o files + that appear in .a files. + + This test only works on darwin because of the way DWARF is stored + where the debug map will refer to .o files inside of .a files. + """ + exe = self.getBuildArtifact("a.out") + + # Create a module with no depedencies. + target = self.createTestTarget(load_dependent_modules=False) + + self.runCmd('breakpoint set -f a.c -l %d' % (self.line_a)) + self.runCmd('breakpoint set -f b.c -l %d' % (self.line_b)) + self.runCmd('breakpoint set -f c.c -l %d' % (self.line_c)) + + # Get the executable module and get the number of symbols to make + # sure the symbol table gets parsed and cached. The module cache is + # enabled in the setUp() function. + main_module = target.GetModuleAtIndex(0) + self.assertTrue(main_module.IsValid()) + # Make sure the symbol table gets loaded and cached + main_module.GetNumSymbols() + a_o_cache_dirs = self.get_module_cache_dirs("libfoo.a", "a.o") + b_o_cache_dirs = self.get_module_cache_dirs("libfoo.a", "b.o") + # We expect the directory for a.o to have two cache directories: + # - 1 for the a.o with a earlier mod time + # - 1 for the a.o that was renamed from c.o that should be 2 seconds older + self.assertEqual(len(a_o_cache_dirs), 2, + "make sure there are two files in the module cache directory for libfoo.a(a.o)") + self.assertEqual(len(b_o_cache_dirs), 1, + "make sure there are two files in the module cache directory for libfoo.a(b.o)") Index: lldb/test/API/functionalities/module_cache/bsd/Makefile =================================================================== --- /dev/null +++ lldb/test/API/functionalities/module_cache/bsd/Makefile @@ -0,0 +1,27 @@ +C_SOURCES := main.c a.c b.c c.c +EXE := # Define a.out explicitly +MAKE_DSYM := NO + +all: a.out + +a.out: main.o libfoo.a + $(LD) $(LDFLAGS) $^ -o $@ + +lib_ab.a: a.o b.o + $(AR) $(ARFLAGS) $@ $^ + $(RM) $^ + +# Here we make a .a file that has two a.o files with different modification +# times and different content by first creating libfoo.a with only a.o and b.o, +# then we sleep for 2 seconds, touch c.o to ensure it has a different +# modification time, and then rename c.o to a.o and then add it to the .a file +# again. This is to help test that the module cache will create different +# directories for the two different a.o files. +libfoo.a: lib_ab.a c.o + sleep 2 + touch c.o + mv c.o a.o + $(AR) $(ARFLAGS) $@ lib_ab.a a.o + $(RM) a.o + +include Makefile.rules Index: lldb/source/Symbol/Symtab.cpp =================================================================== --- lldb/source/Symbol/Symtab.cpp +++ lldb/source/Symbol/Symtab.cpp @@ -22,6 +22,7 @@ #include "lldb/Utility/Timer.h" #include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/GSYM/FileWriter.h" using namespace lldb; using namespace lldb_private; @@ -1149,3 +1150,68 @@ } return nullptr; } + +llvm::Optional<FileSpec> Symtab::GetSymtabCacheFile() { + // This function might return no path if caching is disabled or the module + // isn't suitable for caching. + if (auto file = m_objfile->GetModule()->GetCacheDirectory()) { + StreamString strm; + strm.Format("symtab-{0}", m_objfile->GetCacheHash()); + FileSpec cache_file(*file); + cache_file.AppendPathComponent(strm.GetString()); + return cache_file; + } + return llvm::None; +} + +void Symtab::SaveToCache() { + using namespace llvm; + llvm::Optional<FileSpec> cache_file = GetSymtabCacheFile(); + if (!cache_file) + return; + std::error_code strm_err; + raw_fd_ostream strm(cache_file->GetPath(), strm_err); + if (!strm_err) { + const auto byte_order = llvm::support::endian::system_endianness(); + gsym::FileWriter file(strm, byte_order); + Encode(file); + } +} + +void Symtab::Encode(llvm::gsym::FileWriter &file) const { + file.writeU32(m_symbols.size()); + for (const auto &symbol: m_symbols) + symbol.Encode(file); +} + +bool Symtab::Decode(const DataExtractor &data, lldb::offset_t *offset_ptr) { + if (!data.ValidOffsetForDataOfSize(*offset_ptr, 4)) + return false; + const uint32_t num_symbols = data.GetU32(offset_ptr); + if (num_symbols == 0) + return true; + m_symbols.resize(num_symbols); + SectionList *sections = nullptr; + if (m_objfile) + sections = m_objfile->GetModule()->GetSectionList(); + for (uint32_t i=0; i<num_symbols; ++i) { + if (!m_symbols[i].Decode(data, offset_ptr, sections)) + return false; + } + return true; +} + +bool Symtab::LoadFromCache() { + llvm::Optional<FileSpec> cache_file = GetSymtabCacheFile(); + if (!cache_file) + return false; + if (!FileSystem::Instance().Exists(*cache_file)) + return false; + auto data_sp = FileSystem::Instance().CreateDataBuffer(cache_file->GetPath()); + if (!data_sp) + return false; + DataExtractor data(data_sp, m_objfile->GetByteOrder(), + m_objfile->GetAddressByteSize()); + lldb::offset_t offset = 0; + return Decode(data, &offset); +} Index: lldb/source/Symbol/Symbol.cpp =================================================================== --- lldb/source/Symbol/Symbol.cpp +++ lldb/source/Symbol/Symbol.cpp @@ -19,6 +19,8 @@ #include "lldb/Target/Target.h" #include "lldb/Utility/Stream.h" +#include "llvm/DebugInfo/GSYM/FileWriter.h" + using namespace lldb; using namespace lldb_private; @@ -595,3 +597,82 @@ m_mangled.SetDemangledName(ConstString(os.str())); } } + +bool Symbol::Decode(const DataExtractor &data, lldb::offset_t *offset_ptr, + const SectionList *section_list) { + if (!data.ValidOffsetForDataOfSize(*offset_ptr, 8)) + return false; + m_uid = data.GetU32(offset_ptr); + m_type_data = data.GetU16(offset_ptr); + (&m_type_data)[1] = data.GetU16(offset_ptr); + if (!m_mangled.Decode(data, offset_ptr)) + return false; + if (!data.ValidOffsetForDataOfSize(*offset_ptr, 20)) + return false; + const bool is_addr = data.GetU8(offset_ptr) != 0; + const uint64_t value = data.GetU64(offset_ptr); + if (is_addr) { + m_addr_range.GetBaseAddress().ResolveAddressUsingFileSections( + value, section_list); + } else { + m_addr_range.GetBaseAddress().Clear(); + m_addr_range.GetBaseAddress().SetOffset(value); + } + m_addr_range.SetByteSize(data.GetU64(offset_ptr)); + m_flags = data.GetU32(offset_ptr); + return true; +} + +void Symbol::Encode(llvm::gsym::FileWriter &file) const { + file.writeU32(m_uid); + file.writeU16(m_type_data); + file.writeU16((&m_type_data)[1]); + m_mangled.Encode(file); + // A symbol's value might be an address, or it might be a constant. If the + // symbol's base address doesn't have a section, then it is a constant value. + // If it does have a section, we will encode the file address and re-resolve + // the address when we decode it. + bool is_addr = m_addr_range.GetBaseAddress().GetSection().get() != NULL; + file.writeU8(is_addr); + file.writeU64(m_addr_range.GetBaseAddress().GetFileAddress()); + file.writeU64(m_addr_range.GetByteSize()); + file.writeU32(m_flags); +} + +bool Symbol::operator==(const Symbol &rhs) const { + if (m_uid != rhs.m_uid) + return false; + if (m_type_data != rhs.m_type_data) + return false; + if (m_type_data_resolved != rhs.m_type_data_resolved) + return false; + if (m_is_synthetic != rhs.m_is_synthetic) + return false; + if (m_is_debug != rhs.m_is_debug) + return false; + if (m_is_external != rhs.m_is_external) + return false; + if (m_size_is_sibling != rhs.m_size_is_sibling) + return false; + if (m_size_is_synthesized != rhs.m_size_is_synthesized) + return false; + if (m_size_is_valid != rhs.m_size_is_valid) + return false; + if (m_demangled_is_synthesized != rhs.m_demangled_is_synthesized) + return false; + if (m_contains_linker_annotations != rhs.m_contains_linker_annotations) + return false; + if (m_is_weak != rhs.m_is_weak) + return false; + if (m_type != rhs.m_type) + return false; + if (m_mangled != rhs.m_mangled) + return false; + if (m_addr_range.GetBaseAddress() != rhs.m_addr_range.GetBaseAddress()) + return false; + if (m_addr_range.GetByteSize() != rhs.m_addr_range.GetByteSize()) + return false; + if (m_flags != rhs.m_flags) + return false; + return true; +} Index: lldb/source/Symbol/ObjectFile.cpp =================================================================== --- lldb/source/Symbol/ObjectFile.cpp +++ lldb/source/Symbol/ObjectFile.cpp @@ -23,6 +23,8 @@ #include "lldb/Utility/Timer.h" #include "lldb/lldb-private.h" +#include "llvm/Support/DJB.h" + using namespace lldb; using namespace lldb_private; @@ -715,3 +717,12 @@ break; } } + +uint32_t ObjectFile::GetCacheHash() { + if (m_cache_hash) + return *m_cache_hash; + StreamString strm; + strm.Format("{0}-{1}-{2}", m_file, GetType(), GetStrata()); + m_cache_hash = llvm::djbHash(strm.GetString()); + return *m_cache_hash; +} Index: lldb/source/Symbol/CMakeLists.txt =================================================================== --- lldb/source/Symbol/CMakeLists.txt +++ lldb/source/Symbol/CMakeLists.txt @@ -49,4 +49,5 @@ LINK_COMPONENTS Support + DebugInfoGSYM ) Index: lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp =================================================================== --- lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp +++ lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp @@ -598,6 +598,8 @@ ElapsedTime elapsed(module_sp->GetSymtabParseTime()); SectionList *sect_list = GetSectionList(); m_symtab_up = std::make_unique<Symtab>(this); + if (m_symtab_up->LoadFromCache()) + return m_symtab_up.get(); std::lock_guard<std::recursive_mutex> guard(m_symtab_up->GetMutex()); const uint32_t num_syms = m_coff_header.nsyms; @@ -719,6 +721,7 @@ } } m_symtab_up->CalculateSymbolSizes(); + m_symtab_up->SaveToCache(); } } return m_symtab_up.get(); Index: lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp =================================================================== --- lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp +++ lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp @@ -1320,8 +1320,11 @@ m_symtab_up = std::make_unique<Symtab>(this); std::lock_guard<std::recursive_mutex> symtab_guard( m_symtab_up->GetMutex()); + if (m_symtab_up->LoadFromCache()) + return m_symtab_up.get(); ParseSymtab(); m_symtab_up->Finalize(); + m_symtab_up->SaveToCache(); } } return m_symtab_up.get(); @@ -2489,7 +2492,7 @@ // We shouldn't have exports data from both the LC_DYLD_INFO command // AND the LC_DYLD_EXPORTS_TRIE command in the same binary: - lldbassert(!((dyld_info.export_size > 0) + lldbassert(!((dyld_info.export_size > 0) && (exports_trie_load_command.datasize > 0))); if (dyld_info.export_size > 0) { dyld_trie_data.SetData(m_data, dyld_info.export_off, Index: lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp =================================================================== --- lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp +++ lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp @@ -2719,6 +2719,8 @@ section_list->FindSectionByType(eSectionTypeELFSymbolTable, true).get(); if (symtab) { m_symtab_up = std::make_unique<Symtab>(symtab->GetObjectFile()); + if (m_symtab_up->LoadFromCache()) + return m_symtab_up.get(); symbol_id += ParseSymbolTable(m_symtab_up.get(), symbol_id, symtab); } @@ -2833,6 +2835,7 @@ } m_symtab_up->CalculateSymbolSizes(); + m_symtab_up->SaveToCache(); } return m_symtab_up.get(); Index: lldb/source/Host/common/FileSystem.cpp =================================================================== --- lldb/source/Host/common/FileSystem.cpp +++ lldb/source/Host/common/FileSystem.cpp @@ -126,6 +126,19 @@ return status->getLastModificationTime(); } +bool +FileSystem::SetAccessAndModificationTime(const FileSpec &file_spec, + sys::TimePoint<> time) { + if (!file_spec) + return false; + auto file_or_err = Open(file_spec, File::OpenOptions::eOpenOptionReadWrite, lldb::eFilePermissionsUserRead | lldb::eFilePermissionsUserWrite, /*should_close_fd=*/true); + if (file_or_err) + return !llvm::sys::fs::setLastAccessAndModificationTime( + (*file_or_err)->GetDescriptor(), time); + consumeError(file_or_err.takeError()); + return false; +} + uint64_t FileSystem::GetByteSize(const FileSpec &file_spec) const { if (!file_spec) return 0; @@ -513,3 +526,39 @@ void FileSystem::SetHomeDirectory(std::string home_directory) { m_home_directory = std::move(home_directory); } + +Status FileSystem::CreateDirectory(const FileSpec &file_spec) { + return CreateDirectory(file_spec.GetPath()); +} + +Status FileSystem::CreateDirectory(const llvm::Twine &path) { + return Status(llvm::sys::fs::create_directories(path)); +} + +Status FileSystem::RemoveDirectory(const FileSpec &file_spec) { + return RemoveDirectory(file_spec.GetPath()); +} + +Status FileSystem::RemoveDirectory(const llvm::Twine &path) { + Status error; + if (!IsDirectory(path)) + error.SetErrorStringWithFormatv("\"{0}\" is a not a directory", + path.str().c_str()); + else + error = Status(llvm::sys::fs::remove_directories(path)); + return error; +} + +Status FileSystem::Remove(const FileSpec &file_spec) { + return Remove(file_spec.GetPath()); +} + +Status FileSystem::Remove(const llvm::Twine &path) { + Status error; + if (IsDirectory(path)) + error.SetErrorStringWithFormatv("\"{0}\" is a directory", + path.str().c_str()); + else + error = Status(llvm::sys::fs::remove(path)); + return error; +} Index: lldb/source/Core/ModuleList.cpp =================================================================== --- lldb/source/Core/ModuleList.cpp +++ lldb/source/Core/ModuleList.cpp @@ -85,6 +85,13 @@ if (clang::driver::Driver::getDefaultModuleCachePath(path)) { lldbassert(SetClangModulesCachePath(FileSpec(path))); } + + path.clear(); + if (llvm::sys::path::cache_directory(path)) { + llvm::sys::path::append(path, "lldb"); + llvm::sys::path::append(path, "ModuleCache"); + lldbassert(SetLLDBModulesCachePath(FileSpec(path))); + } } bool ModuleListProperties::GetEnableExternalLookup() const { @@ -110,6 +117,29 @@ nullptr, ePropertyClangModulesCachePath, path); } +FileSpec ModuleListProperties::GetLLDBModulesCachePath() const { + return m_collection_sp + ->GetPropertyAtIndexAsOptionValueFileSpec(nullptr, false, + ePropertyLLDBModulesCachePath) + ->GetCurrentValue(); +} + +bool ModuleListProperties::SetLLDBModulesCachePath(const FileSpec &path) { + return m_collection_sp->SetPropertyAtIndexAsFileSpec( + nullptr, ePropertyLLDBModulesCachePath, path); +} + +bool ModuleListProperties::GetEnableLLDBModulesCache() const { + const uint32_t idx = ePropertyEnableLLDBModulesCache; + return m_collection_sp->GetPropertyAtIndexAsBoolean( + nullptr, idx, g_modulelist_properties[idx].default_uint_value != 0); +} + +bool ModuleListProperties::SetEnableLLDBModulesCache(bool new_value) { + return m_collection_sp->SetPropertyAtIndexAsBoolean( + nullptr, ePropertyEnableLLDBModulesCache, new_value); +} + void ModuleListProperties::UpdateSymlinkMappings() { FileSpecList list = m_collection_sp ->GetPropertyAtIndexAsOptionValueFileSpecList( Index: lldb/source/Core/Module.cpp =================================================================== --- lldb/source/Core/Module.cpp +++ lldb/source/Core/Module.cpp @@ -55,10 +55,14 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/Support/Compiler.h" +#include "llvm/Support/DJB.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/JSON.h" #include "llvm/Support/Signals.h" #include "llvm/Support/raw_ostream.h" + #include <cassert> #include <cinttypes> #include <cstdarg> @@ -1658,3 +1662,132 @@ return false; } + +uint32_t Module::Hash() { + std::string identifier; + llvm::raw_string_ostream id_strm(identifier); + id_strm << m_file.GetPath() << m_arch.GetTriple().str(); + if (m_object_name) + id_strm << m_object_name.GetStringRef(); + if (m_object_offset > 0) + id_strm << m_object_offset; + const auto mtime = llvm::sys::toTimeT(m_object_mod_time); + if (mtime > 0) + id_strm << mtime; + return llvm::djbHash(id_strm.str()); +} + +llvm::Optional<FileSpec> Module::GetCacheDirectory() { + using namespace llvm; + std::lock_guard<std::recursive_mutex> guard(m_mutex); + if (!ModuleList::GetGlobalModuleListProperties().GetEnableLLDBModulesCache()) + return None; + // Check the cache directory has a value _and_ if the filespec that was stored + // there is valid. If the optional has a value and the FileSpec is empty, it + // means something went wrong when trying to create the cache directory and we + // don't want to keep trying to create the directory over and over. + if (m_cache_directory.hasValue()) { + // Check if the FileSpec is not empty and if so return the cached value. + if (*m_cache_directory) + return m_cache_directory; + // The FileSpec is invalid, so return None. + return None; + } + // Initialize the optional value with an empty FileSpec in case anything goes + // wrong. If something goes wrong, then the next time this function is called + // we will enter the above if statement and return the cached value or + // None if the cache directory is empty. + m_cache_directory = FileSpec(); + FileSpec cache_dir = + ModuleList::GetGlobalModuleListProperties().GetLLDBModulesCachePath(); + // Append the file basename to the path as a directory. + cache_dir.AppendPathComponent(GetFileSpec().GetFilename().GetStringRef()); + // Append the object name to the path as a directory. The object name will be + // valie if we have an object file from a BSD archive like "foo.a(bar.o)". + if (m_object_name) + cache_dir.AppendPathComponent(m_object_name.GetStringRef()); + // Append the hash as a directory in case a file on disk contains multiple + // architectures. + std::string str; + raw_string_ostream strm(str); + strm << format_hex(Hash(), 10); + cache_dir.AppendPathComponent(strm.str()); + + // Create the directories needed for this module cache path. + FileSystem &fs = FileSystem::Instance(); + if (fs.CreateDirectory(cache_dir).Fail()) + return None; + sys::TimePoint<> module_mtime, cache_mtime; + FileSpec info_file(cache_dir); + module_mtime = GetModificationTime(); + constexpr StringRef info_basename("info.json"); + info_file.AppendPathComponent(info_basename); + if (fs.Exists(info_file)) { + cache_mtime = fs.GetModificationTime(info_file); + if (cache_mtime != module_mtime) { + // Module has been updated and doesn't match the cached information. + // Delete all files except the info file in the cache directory. + fs.EnumerateDirectory( + cache_dir.GetPath(), + /*find_directories=*/false, + /*file_files=*/true, + /*find_other=*/false, + [](void *baton, + sys::fs::file_type ft, + StringRef path) -> FileSystem::EnumerateDirectoryResult { + if (!path.endswith("info.json")) + FileSystem::Instance().Remove(path); + return FileSystem::EnumerateDirectoryResult::eEnumerateDirectoryResultNext; + }, + nullptr); + // Update the modification time on the info file. + if (!fs.SetAccessAndModificationTime(info_file, module_mtime)) + return None; + } + } else { + // Create the info file in the module cache directory and set its + // modification time to match the module's modification time. + auto file_or_err = fs.Open( + info_file, + File::OpenOptions::eOpenOptionWriteOnly | + File::OpenOptions::eOpenOptionCanCreateNewOnly, + lldb::eFilePermissionsUserRead | lldb::eFilePermissionsUserWrite, + /*should_close_fd=*/true); + if (!file_or_err) { + consumeError(file_or_err.takeError()); + return None; + } + lldb_private::File *file = file_or_err->get(); + // Create the context of the "info.json" file with all of the information + // needed to fill out a ModuleSpec for the module that is being cached in + // this directory. This can later be used by cache management and purging + // commands by allowing us to check if the file still exists. The + // modification time of the file in "path" will be set to the modification + // time of the "info.json" file to allow quick and easy testing to see if + // the cache is up to date. + json::Object info{ + {"path", GetFileSpec().GetPath() }, + {"triple", GetArchitecture().GetTriple().str() } + }; + if (m_object_name) + info.try_emplace("object-name", m_object_name.GetStringRef()); + if (m_object_offset > 0) + info.try_emplace("object-offset", m_object_offset); + const auto mtime = llvm::sys::toTimeT(m_object_mod_time); + if (mtime > 0) + info.try_emplace("object-mtime", mtime); + + std::string json_str(formatv("{0:2}\n", json::Value(std::move(info))).str()); + size_t count = json_str.size(); + if (file->Write(json_str.data(), count).Fail() || count != json_str.size()) + return None; + file->Close(); + // Update the modification time on the info file. + if (!fs.SetAccessAndModificationTime(info_file, module_mtime)) + return None; + } + // Everything went well, set the cache directory to the successfully + // created cache directory. + m_cache_directory = cache_dir; + return m_cache_directory; +} Index: lldb/source/Core/Mangled.cpp =================================================================== --- lldb/source/Core/Mangled.cpp +++ lldb/source/Core/Mangled.cpp @@ -18,6 +18,7 @@ #include "lldb/lldb-enumerations.h" #include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/GSYM/FileWriter.h" #include "llvm/Demangle/Demangle.h" #include "llvm/Support/Compiler.h" @@ -391,3 +392,80 @@ s << ", demangled = <error>"; return s; } + + +// When encoding Mangled objects we can get away with encoding as little +// information as is required. The enumeration below helps us to efficiently +// encode Mangled objects. +enum MangledEncoding { + /// If the Mangled object has neither a mangled name or demangled name we can + /// encode the object with one zero byte using the Empty enumeration. + Empty = 0u, + /// If the Mangled object has only a demangled name and no mangled named, we + /// can encode only the demangled name. + DemangledOnly = 1u, + /// If the mangle name can calculate the demangled name (it is the + /// mangled/demangled counterpart), then we only need to encode the mangled + /// name as the demangled name can be recomputed. + MangledOnly = 2u, + /// If we have a Mangled object with two different names that are not related + /// then we need to save both strings. + MangledAndDemangled = 3u +}; + +bool Mangled::Decode(const DataExtractor &data, + lldb::offset_t *offset_ptr) { + m_mangled.Clear(); + m_demangled.Clear(); + MangledEncoding encoding = (MangledEncoding)data.GetU8(offset_ptr); + switch (encoding) { + case Empty: + return true; + + case DemangledOnly: + m_demangled.SetCString(data.GetCStr(offset_ptr)); + return true; + + case MangledOnly: + m_mangled.SetCString(data.GetCStr(offset_ptr)); + return true; + + case MangledAndDemangled: + m_mangled.SetCString(data.GetCStr(offset_ptr)); + m_demangled.SetCString(data.GetCStr(offset_ptr)); + return true; + } + return false; +} + +void Mangled::Encode(llvm::gsym::FileWriter &file) const { + MangledEncoding encoding = Empty; + if (m_mangled) { + encoding = MangledOnly; + if (m_demangled) { + // We have both mangled and demangled names. If the demangled name is the + // counterpart of the mangled name, then we only need to save the mangled + // named. If they are different, we need to save both. + ConstString s; + if (!(m_mangled.GetMangledCounterpart(s) && s == m_demangled)) + encoding = MangledAndDemangled; + } + } else if (m_demangled) { + encoding = DemangledOnly; + } + file.writeU8(encoding); + switch (encoding) { + case Empty: + break; + case DemangledOnly: + file.writeNullTerminated(m_demangled.GetStringRef()); + break; + case MangledOnly: + file.writeNullTerminated(m_mangled.GetStringRef()); + break; + case MangledAndDemangled: + file.writeNullTerminated(m_mangled.GetStringRef()); + file.writeNullTerminated(m_demangled.GetStringRef()); + break; + } +} Index: lldb/source/Core/CoreProperties.td =================================================================== --- lldb/source/Core/CoreProperties.td +++ lldb/source/Core/CoreProperties.td @@ -13,6 +13,14 @@ Global, DefaultStringValue<"">, Desc<"Debug info path which should be resolved while parsing, relative to the host filesystem.">; + def EnableLLDBModulesCache: Property<"enable-lldb-modules-cache", "Boolean">, + Global, + DefaultFalse, + Desc<"Enable module caching for debug sessions in LLDB. LLDB can cache data for each module for improved performance in subsequent debug sessions.">; + def LLDBModulesCachePath: Property<"lldb-modules-cache-path", "FileSpec">, + Global, + DefaultStringValue<"">, + Desc<"The path to the LLDB module cache directory.">; } let Definition = "debugger" in { Index: lldb/source/API/SBDebugger.cpp =================================================================== --- lldb/source/API/SBDebugger.cpp +++ lldb/source/API/SBDebugger.cpp @@ -859,15 +859,15 @@ // The version of CreateTarget that takes an ArchSpec won't accept an // empty ArchSpec, so when the arch hasn't been specified, we need to // call the target triple version. - error = m_opaque_sp->GetTargetList().CreateTarget(*m_opaque_sp, filename, + error = m_opaque_sp->GetTargetList().CreateTarget(*m_opaque_sp, filename, arch_cstr, eLoadDependentsYes, nullptr, target_sp); } else { PlatformSP platform_sp = m_opaque_sp->GetPlatformList() .GetSelectedPlatform(); - ArchSpec arch = Platform::GetAugmentedArchSpec(platform_sp.get(), + ArchSpec arch = Platform::GetAugmentedArchSpec(platform_sp.get(), arch_cstr); if (arch.IsValid()) - error = m_opaque_sp->GetTargetList().CreateTarget(*m_opaque_sp, filename, + error = m_opaque_sp->GetTargetList().CreateTarget(*m_opaque_sp, filename, arch, eLoadDependentsYes, platform_sp, target_sp); else error.SetErrorStringWithFormat("invalid arch_cstr: %s", arch_cstr); Index: lldb/packages/Python/lldbsuite/test/lldbtest.py =================================================================== --- lldb/packages/Python/lldbsuite/test/lldbtest.py +++ lldb/packages/Python/lldbsuite/test/lldbtest.py @@ -2513,7 +2513,8 @@ self.fail(self._formatMessage(msg, "'{}' is not success".format(error))) - def createTestTarget(self, file_path=None, msg=None): + def createTestTarget(self, file_path=None, msg=None, + load_dependent_modules=True): """ Creates a target from the file found at the given file path. Asserts that the resulting target is valid. @@ -2527,7 +2528,6 @@ error = lldb.SBError() triple = "" platform = "" - load_dependent_modules = True target = self.dbg.CreateTarget(file_path, triple, platform, load_dependent_modules, error) if error.Fail(): Index: lldb/include/lldb/Symbol/Symtab.h =================================================================== --- lldb/include/lldb/Symbol/Symtab.h +++ lldb/include/lldb/Symbol/Symtab.h @@ -35,6 +35,11 @@ Symtab(ObjectFile *objfile); ~Symtab(); + llvm::Optional<FileSpec> GetSymtabCacheFile(); + void SaveToCache(); + void Encode(llvm::gsym::FileWriter &file) const; + bool Decode(const DataExtractor &data, lldb::offset_t *offset_ptr); + bool LoadFromCache(); void PreloadSymbols(); void Reserve(size_t count); Symbol *Resize(size_t count); Index: lldb/include/lldb/Symbol/Symbol.h =================================================================== --- lldb/include/lldb/Symbol/Symbol.h +++ lldb/include/lldb/Symbol/Symbol.h @@ -15,6 +15,12 @@ #include "lldb/Utility/UserID.h" #include "lldb/lldb-private.h" +namespace llvm { +namespace gsym { + class FileWriter; +} +} + namespace lldb_private { class Symbol : public SymbolContextScope { @@ -235,6 +241,12 @@ return "___lldb_unnamed_symbol"; } + bool Decode(const DataExtractor &data, lldb::offset_t *offset_ptr, + const SectionList *section_list); + void Encode(llvm::gsym::FileWriter &file) const; + + bool operator==(const Symbol &rhs) const; + protected: // This is the internal guts of ResolveReExportedSymbol, it assumes // reexport_name is not null, and that module_spec is valid. We track the Index: lldb/include/lldb/Symbol/ObjectFile.h =================================================================== --- lldb/include/lldb/Symbol/ObjectFile.h +++ lldb/include/lldb/Symbol/ObjectFile.h @@ -19,6 +19,7 @@ #include "lldb/Utility/FileSpec.h" #include "lldb/Utility/UUID.h" #include "lldb/lldb-private.h" +#include "llvm/ADT/Optional.h" #include "llvm/Support/VersionTuple.h" namespace lldb_private { @@ -692,23 +693,32 @@ return false; } + /// Get a hash that can be used for caching object file releated information. + /// + /// Data for object files can be cached between runs of debug sessions and + /// a module can end up using a main file and a symbol file, both of which + /// can be object files. So we need a unique hash that identifies an object + /// file when storing cached data. + uint32_t GetCacheHash(); + protected: // Member variables. FileSpec m_file; Type m_type; Strata m_strata; - lldb::addr_t m_file_offset; ///< The offset in bytes into the file, or the - ///address in memory - lldb::addr_t m_length; ///< The length of this object file if it is known (can - ///be zero if length is unknown or can't be - ///determined). - DataExtractor - m_data; ///< The data for this object file so things can be parsed lazily. + /// The offset in bytes into the file, or the address in memory. + lldb::addr_t m_file_offset; + /// The length of this object file if it is known. This can be zero if length + /// is unknown or can't be determined. + lldb::addr_t m_length; + /// The data for this object file so things can be parsed lazily. + DataExtractor m_data; lldb::ProcessWP m_process_wp; const lldb::addr_t m_memory_addr; std::unique_ptr<lldb_private::SectionList> m_sections_up; std::unique_ptr<lldb_private::Symtab> m_symtab_up; uint32_t m_synthetic_symbol_idx; + llvm::Optional<uint32_t> m_cache_hash; /// Sets the architecture for a module. At present the architecture can /// only be set if it is invalid. It is not allowed to switch from one Index: lldb/include/lldb/Host/FileSystem.h =================================================================== --- lldb/include/lldb/Host/FileSystem.h +++ lldb/include/lldb/Host/FileSystem.h @@ -89,6 +89,10 @@ llvm::sys::TimePoint<> GetModificationTime(const llvm::Twine &path) const; /// \} + /// Set the access and modification time of the given file. + bool SetAccessAndModificationTime(const FileSpec &file_spec, + llvm::sys::TimePoint<> time); + /// Returns the on-disk size of the given file in bytes. /// \{ uint64_t GetByteSize(const FileSpec &file_spec) const; @@ -142,6 +146,28 @@ void Resolve(FileSpec &file_spec); /// \} + /// Create a directory. + /// + /// Create as many directories as needed in the path. + /// \{ + Status CreateDirectory(const FileSpec &file_spec); + Status CreateDirectory(const llvm::Twine &path); + /// \} + + /// Remove a single directory that must be empty. + /// \{ + Status RemoveDirectory(const FileSpec &file_spec); + Status RemoveDirectory(const llvm::Twine &path); + /// \} + + /// Remove a single file. + /// + /// The path must specify a file an not a directory. + /// \{ + Status Remove(const FileSpec &file_spec); + Status Remove(const llvm::Twine &path); + /// \} + //// Create memory buffer from path. /// \{ std::shared_ptr<DataBufferLLVM> CreateDataBuffer(const llvm::Twine &path, Index: lldb/include/lldb/Core/ModuleList.h =================================================================== --- lldb/include/lldb/Core/ModuleList.h +++ lldb/include/lldb/Core/ModuleList.h @@ -60,6 +60,11 @@ bool SetClangModulesCachePath(const FileSpec &path); bool GetEnableExternalLookup() const; bool SetEnableExternalLookup(bool new_value); + bool GetEnableLLDBModulesCache() const; + bool SetEnableLLDBModulesCache(bool new_value); + + FileSpec GetLLDBModulesCachePath() const; + bool SetLLDBModulesCachePath(const FileSpec &path); PathMappingList GetSymlinkMappings() const; }; Index: lldb/include/lldb/Core/Module.h =================================================================== --- lldb/include/lldb/Core/Module.h +++ lldb/include/lldb/Core/Module.h @@ -876,7 +876,7 @@ /// The value is returned as a reference to allow it to be updated by the /// ElapsedTime RAII object. StatsDuration &GetSymtabParseTime() { return m_symtab_parse_time; } - + /// Accessor for the symbol table index time metric. /// /// The value is returned as a reference to allow it to be updated by the @@ -946,6 +946,41 @@ bool m_match_name_after_lookup = false; }; + /// Get a unique hash for this module. + /// + /// The hash should be enough to indentify the file on disk and the + /// architecture of the file. If the module represents an object inside of a + /// file, then the hash should include the object name and object offset to + /// ensure a unique hash. Some examples: + /// - just a regular object file (mach-o, elf, coff, etc) should create a hash + /// - a universal mach-o file that contains to multiple architectures, + /// each architecture slice should have a unique hash even though they come + /// from the same file + /// - a .o file inside of a BSD archive. Each .o file will have a object name + /// and object offset that should produce a unique hash. The object offset + /// is needed as BSD archive files can contain multiple .o files that have + /// the same name. + uint32_t Hash(); + + /// Get the cache directory for caching data for this module. + /// + /// LLDB can cache data for a module between runs. This cache directory can be + /// used to stored data that must be manually created each time you debug. + /// Examples include debug information indexes, symbol table indexes, symbol + /// tables, and more. + /// + /// The cache directory should be unique to a module's path on disk, + /// architecture and optional object name so the module cache can easily be + /// cleared and updated if the file gets updated. This keeps the module cache + /// from growing too large over time by only allowing one entry in the cache + /// for the same file/arch/object. + /// + /// \returns + /// A valid FileSpec if caching is enabled and the cache directory was + /// able to be created. llvm::None if caching is disabled or there was a + /// failure to create the cache directory. + llvm::Optional<FileSpec> GetCacheDirectory(); + protected: // Member Variables mutable std::recursive_mutex m_mutex; ///< A mutex to keep this object happy @@ -973,6 +1008,9 @@ uint64_t m_object_offset = 0; llvm::sys::TimePoint<> m_object_mod_time; + /// The cache directory for caching data between LLDB processes. + llvm::Optional<FileSpec> m_cache_directory; + /// DataBuffer containing the module image, if it was provided at /// construction time. Otherwise the data will be retrieved by mapping /// one of the FileSpec members above. Index: lldb/include/lldb/Core/Mangled.h =================================================================== --- lldb/include/lldb/Core/Mangled.h +++ lldb/include/lldb/Core/Mangled.h @@ -12,6 +12,7 @@ #include "lldb/lldb-enumerations.h" #include "lldb/lldb-forward.h" +#include "lldb/lldb-types.h" #include "lldb/Utility/ConstString.h" @@ -20,6 +21,12 @@ #include <cstddef> #include <memory> +namespace llvm { +namespace gsym { +class FileWriter; +} +} // namespace llvm + namespace lldb_private { /// \class Mangled Mangled.h "lldb/Core/Mangled.h" @@ -63,6 +70,16 @@ explicit Mangled(llvm::StringRef name); + bool operator==(const Mangled &rhs) const { + return m_mangled == rhs.m_mangled && + GetDemangledName() == rhs.GetDemangledName(); + } + + bool operator!=(const Mangled &rhs) const { + return m_mangled != rhs.m_mangled || + GetDemangledName() != rhs.GetDemangledName(); + } + /// Convert to pointer operator. /// /// This allows code to check a Mangled object to see if it contains a valid @@ -269,6 +286,9 @@ /// for s, otherwise the enumerator for the mangling scheme detected. static Mangled::ManglingScheme GetManglingScheme(llvm::StringRef const name); + bool Decode(const DataExtractor &data, lldb::offset_t *offset_ptr); + void Encode(llvm::gsym::FileWriter &file) const; + private: /// Mangled member variables. ConstString m_mangled; ///< The mangled version of the name
_______________________________________________ lldb-commits mailing list lldb-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
