https://github.com/huangjd updated https://github.com/llvm/llvm-project/pull/66164
>From dcaa197c8b4e80fba83fc3905b0488a44fe89cc5 Mon Sep 17 00:00:00 2001 From: William Huang <williamjhu...@google.com> Date: Fri, 15 Sep 2023 23:08:46 +0000 Subject: [PATCH 1/2] Rebase with main --- .../include/llvm/ProfileData/ProfileFuncRef.h | 225 ++++++++++++++++++ llvm/include/llvm/ProfileData/SampleProf.h | 130 +++++----- .../llvm/ProfileData/SampleProfReader.h | 27 +-- .../llvm/ProfileData/SampleProfWriter.h | 12 +- .../llvm/Transforms/IPO/ProfiledCallGraph.h | 24 +- .../Transforms/IPO/SampleContextTracker.h | 27 ++- llvm/lib/ProfileData/SampleProf.cpp | 25 +- llvm/lib/ProfileData/SampleProfReader.cpp | 76 +++--- llvm/lib/ProfileData/SampleProfWriter.cpp | 30 ++- llvm/lib/Target/X86/X86InsertPrefetch.cpp | 3 +- .../Transforms/IPO/SampleContextTracker.cpp | 64 ++--- llvm/lib/Transforms/IPO/SampleProfile.cpp | 90 ++++--- llvm/tools/llvm-profdata/llvm-profdata.cpp | 16 +- llvm/tools/llvm-profgen/CSPreInliner.cpp | 8 +- llvm/tools/llvm-profgen/CSPreInliner.h | 7 +- llvm/tools/llvm-profgen/ProfileGenerator.cpp | 19 +- llvm/tools/llvm-profgen/ProfiledBinary.cpp | 15 +- llvm/tools/llvm-profgen/ProfiledBinary.h | 20 +- llvm/unittests/ProfileData/SampleProfTest.cpp | 45 ++-- 19 files changed, 555 insertions(+), 308 deletions(-) create mode 100644 llvm/include/llvm/ProfileData/ProfileFuncRef.h diff --git a/llvm/include/llvm/ProfileData/ProfileFuncRef.h b/llvm/include/llvm/ProfileData/ProfileFuncRef.h new file mode 100644 index 000000000000000..60be6232372b8c6 --- /dev/null +++ b/llvm/include/llvm/ProfileData/ProfileFuncRef.h @@ -0,0 +1,225 @@ +//===--- ProfileFuncRef.h - Sample profile function name ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the StringRefOrHashCode class. It is to represent function +// names in a sample profile, which can be in one of two forms - either a +// regular string, or a 64-bit hash code. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_PROFILEDATA_PROFILEFUNCREF_H +#define LLVM_PROFILEDATA_PROFILEFUNCREF_H + +#include "llvm/ADT/DenseMapInfo.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/MD5.h" +#include "llvm/Support/raw_ostream.h" +#include <cstdint> + +namespace llvm { +namespace sampleprof { + +/// This class represents a function name that is read from a sample profile. It +/// comes with two forms: a string or a hash code. For efficient storage, a +/// sample profile may store function names as 64-bit MD5 values, so when +/// reading the profile, this class can represnet them without converting it to +/// a string first. +/// When representing a hash code, we utilize the Length field to store it, and +/// Data is set to null. When representing a string, it is same as StringRef, +/// and can be pointer-casted as one. +/// We disallow implicit cast to StringRef because there are too many instances +/// that it may cause break the code, such as using it in a StringMap. +class ProfileFuncRef { + + const char *Data = nullptr; + + /// Use uint64_t instead of size_t so that it can also hold a MD5 value. + uint64_t Length = 0; + + /// Extension to memcmp to handle hash code representation. If both are hash + /// values, Lhs and Rhs are both null, function returns 0 (and needs an extra + /// comparison using getIntValue). If only one is hash code, it is considered + /// less than the StringRef one. Otherwise perform normal string comparison. + static int compareMemory(const char *Lhs, const char *Rhs, uint64_t Length) { + if (Lhs == Rhs) + return 0; + if (!Lhs) + return -1; + if (!Rhs) + return 1; + return ::memcmp(Lhs, Rhs, (size_t)Length); + } + +public: + ProfileFuncRef() = default; + + /// Constructor from a StringRef. + explicit ProfileFuncRef(StringRef Str) + : Data(Str.data()), Length(Str.size()) {} + + /// Constructor from a hash code. + explicit ProfileFuncRef(uint64_t HashCode) + : Data(nullptr), Length(HashCode) { + assert(HashCode != 0); + } + + /// Constructor from a string. Check if Str is a number, which is generated by + /// converting a MD5 sample profile to a format that does not support MD5, and + /// if so, convert the numerical string to a hash code first. We assume that + /// no function name (from a profile) can be a pure number. + explicit ProfileFuncRef(const std::string &Str) + : Data(Str.data()), Length(Str.size()) { + // Only need to check for base 10 digits, fail faster if otherwise. + if (Str.length() > 0 && isdigit(Str[0]) && + !StringRef(Str).getAsInteger(10, Length)) + Data = nullptr; + } + + /// Check for equality. Similar to StringRef::equals, but will also cover for + /// the case where one or both are hash codes. Comparing their int values are + /// sufficient. A hash code ProfileFuncName is considered not equal to a + /// StringRef ProfileFuncName regardless of actual contents. + bool equals(const ProfileFuncRef &Other) const { + return Length == Other.Length && + compareMemory(Data, Other.Data, Length) == 0; + } + + /// Total order comparison. If both ProfileFuncName are StringRef, this is the + /// same as StringRef::compare. If one of them is StringRef, it is considered + /// greater than the hash code ProfileFuncName. Otherwise this is the the + /// same as comparing their int values. + int compare(const ProfileFuncRef &Other) const { + auto Res = compareMemory(Data, Other.Data, std::min(Length, Other.Length)); + if (Res != 0) + return Res; + if (Length == Other.Length) + return 0; + return Length < Other.Length ? -1 : 1; + } + + /// Convert to a string, usually for output purpose. + std::string str() const { + if (Data) + return std::string(Data, Length); + if (Length != 0) + return std::to_string(Length); + return std::string(); + } + + /// Convert to StringRef, a backing buffer must be provided in case this + /// object represents a hash code, which will be converted to a string into + /// the buffer. If this object represents a StringRef, the buffer is not used. + StringRef stringRef(std::string &Buffer) const { + if (Data) + return StringRef(Data, Length); + if (Length != 0) { + Buffer = std::to_string(Length); + return Buffer; + } + return StringRef(); + } + + friend raw_ostream &operator<<(raw_ostream &OS, const ProfileFuncRef &Obj); + + /// Get hash code of this object. Returns this object's hash code if it is + /// already representing one, otherwise returns the MD5 of its string content. + /// Note that it is not the same as std::hash because we want to keep the + /// consistency that the same sample profile function in string form or MD5 + /// form has the same hash code. + uint64_t getHashCode() const { + if (Data) + return MD5Hash(StringRef(Data, Length)); + return Length; + } + + bool empty() const { return Length == 0; } + + /// Check if this object represents a StringRef, or a hash code. + bool isStringRef() const { return Data != nullptr; } +}; + +inline bool operator==(const ProfileFuncRef &LHS, const ProfileFuncRef &RHS) { + return LHS.equals(RHS); +} + +inline bool operator!=(const ProfileFuncRef &LHS, const ProfileFuncRef &RHS) { + return !LHS.equals(RHS); +} + +inline bool operator<(const ProfileFuncRef &LHS, const ProfileFuncRef &RHS) { + return LHS.compare(RHS) < 0; +} + +inline bool operator<=(const ProfileFuncRef &LHS, const ProfileFuncRef &RHS) { + return LHS.compare(RHS) <= 0; +} + +inline bool operator>(const ProfileFuncRef &LHS, const ProfileFuncRef &RHS) { + return LHS.compare(RHS) > 0; +} + +inline bool operator>=(const ProfileFuncRef &LHS, const ProfileFuncRef &RHS) { + return LHS.compare(RHS) >= 0; +} + +inline raw_ostream &operator<<(raw_ostream &OS, const ProfileFuncRef &Obj) { + if (Obj.Data) + return OS << StringRef(Obj.Data, Obj.Length); + if (Obj.Length != 0) + return OS << Obj.Length; + return OS; +} + +inline uint64_t MD5Hash(const sampleprof::ProfileFuncRef &Obj) { + return Obj.getHashCode(); +} + +inline hash_code hash_value(const sampleprof::ProfileFuncRef &Obj) { + return Obj.getHashCode(); +} + +} // end namespace sampleprof + +/// Template specialization for ProfileFuncName so that it can be used in LLVM +/// map containers. +template <> struct DenseMapInfo<sampleprof::ProfileFuncRef, void> { + + static inline sampleprof::ProfileFuncRef getEmptyKey() { + return sampleprof::ProfileFuncRef(~0ULL); + } + + static inline sampleprof::ProfileFuncRef getTombstoneKey() { + return sampleprof::ProfileFuncRef(~1ULL); + } + + static unsigned getHashValue(const sampleprof::ProfileFuncRef &Val) { + return Val.getHashCode(); + } + + static bool isEqual(const sampleprof::ProfileFuncRef &LHS, + const sampleprof::ProfileFuncRef &RHS) { + return LHS == RHS; + } +}; + +} // end namespace llvm + +namespace std { + +/// Template specialization for ProfileFuncName so that it can be used in STL +/// containers. +template <> struct hash<llvm::sampleprof::ProfileFuncRef> { + size_t operator()(const llvm::sampleprof::ProfileFuncRef &Val) const { + return Val.getHashCode(); + } +}; + +} // end namespace std + +#endif // LLVM_PROFILEDATA_PROFILEFUNCREF_H diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h index 9b2416e39b6cc9a..cae38de7dfb9835 100644 --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -21,6 +21,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalValue.h" +#include "llvm/ProfileData/ProfileFuncRef.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorOr.h" @@ -111,12 +112,10 @@ static inline uint64_t SPMagic(SampleProfileFormat Format = SPF_Binary) { /// Get the proper representation of a string according to whether the /// current Format uses MD5 to represent the string. -static inline StringRef getRepInFormat(StringRef Name, bool UseMD5, - std::string &GUIDBuf) { +static inline ProfileFuncRef getRepInFormat(StringRef Name, bool UseMD5) { if (Name.empty() || !UseMD5) - return Name; - GUIDBuf = std::to_string(Function::getGUID(Name)); - return GUIDBuf; + return ProfileFuncRef(Name); + return ProfileFuncRef(Function::getGUID(Name)); } static inline uint64_t SPVersion() { return 103; } @@ -305,6 +304,10 @@ struct LineLocation { return LineOffset != O.LineOffset || Discriminator != O.Discriminator; } + uint64_t getHashCode() const { + return ((uint64_t) Discriminator << 32) | LineOffset; + } + uint32_t LineOffset; uint32_t Discriminator; }; @@ -318,14 +321,6 @@ struct LineLocationHash { raw_ostream &operator<<(raw_ostream &OS, const LineLocation &Loc); -static inline uint64_t hashFuncName(StringRef F) { - // If function name is already MD5 string, do not hash again. - uint64_t Hash; - if (F.getAsInteger(10, Hash)) - Hash = MD5Hash(F); - return Hash; -} - /// Representation of a single sample record. /// /// A sample record is represented by a positive integer value, which @@ -338,7 +333,7 @@ static inline uint64_t hashFuncName(StringRef F) { /// will be a list of one or more functions. class SampleRecord { public: - using CallTarget = std::pair<StringRef, uint64_t>; + using CallTarget = std::pair<ProfileFuncRef, uint64_t>; struct CallTargetComparator { bool operator()(const CallTarget &LHS, const CallTarget &RHS) const { if (LHS.second != RHS.second) @@ -349,7 +344,7 @@ class SampleRecord { }; using SortedCallTargetSet = std::set<CallTarget, CallTargetComparator>; - using CallTargetMap = StringMap<uint64_t>; + using CallTargetMap = std::unordered_map<ProfileFuncRef, uint64_t>; SampleRecord() = default; /// Increment the number of samples for this record by \p S. @@ -378,7 +373,7 @@ class SampleRecord { /// /// Sample counts accumulate using saturating arithmetic, to avoid wrapping /// around unsigned integers. - sampleprof_error addCalledTarget(StringRef F, uint64_t S, + sampleprof_error addCalledTarget(ProfileFuncRef F, uint64_t S, uint64_t Weight = 1) { uint64_t &TargetSamples = CallTargets[F]; bool Overflowed; @@ -390,7 +385,7 @@ class SampleRecord { /// Remove called function from the call target map. Return the target sample /// count of the called function. - uint64_t removeCalledTarget(StringRef F) { + uint64_t removeCalledTarget(ProfileFuncRef F) { uint64_t Count = 0; auto I = CallTargets.find(F); if (I != CallTargets.end()) { @@ -476,12 +471,12 @@ enum ContextAttributeMask { // Represents a context frame with function name and line location struct SampleContextFrame { - StringRef FuncName; + ProfileFuncRef FuncName; LineLocation Location; SampleContextFrame() : Location(0, 0) {} - SampleContextFrame(StringRef FuncName, LineLocation Location) + SampleContextFrame(ProfileFuncRef FuncName, LineLocation Location) : FuncName(FuncName), Location(Location) {} bool operator==(const SampleContextFrame &That) const { @@ -502,6 +497,12 @@ struct SampleContextFrame { } return OContextStr.str(); } + + uint64_t getHashCode() const { + uint64_t NameHash = FuncName.getHashCode(); + uint64_t LocId = Location.getHashCode(); + return NameHash + (LocId << 5) + LocId; + } }; static inline hash_code hash_value(const SampleContextFrame &arg) { @@ -535,6 +536,9 @@ class SampleContext { SampleContext(StringRef Name) : Name(Name), State(UnknownContext), Attributes(ContextNone) {} + SampleContext(ProfileFuncRef Name) + : Name(Name), State(UnknownContext), Attributes(ContextNone) {} + SampleContext(SampleContextFrames Context, ContextStateMask CState = RawContext) : Attributes(ContextNone) { @@ -555,7 +559,7 @@ class SampleContext { bool HasContext = ContextStr.startswith("["); if (!HasContext) { State = UnknownContext; - Name = ContextStr; + Name = ProfileFuncRef(ContextStr); } else { CSNameTable.emplace_back(); SampleContextFrameVector &Context = CSNameTable.back(); @@ -572,7 +576,7 @@ class SampleContext { ContextStr = ContextStr.substr(1, ContextStr.size() - 2); StringRef ContextRemain = ContextStr; StringRef ChildContext; - StringRef CalleeName; + ProfileFuncRef CalleeName; while (!ContextRemain.empty()) { auto ContextSplit = ContextRemain.split(" @ "); ChildContext = ContextSplit.first; @@ -585,11 +589,12 @@ class SampleContext { // Decode context string for a frame to get function name and location. // `ContextStr` is in the form of `FuncName:StartLine.Discriminator`. - static void decodeContextString(StringRef ContextStr, StringRef &FName, + static void decodeContextString(StringRef ContextStr, + ProfileFuncRef &FName, LineLocation &LineLoc) { // Get function name auto EntrySplit = ContextStr.split(':'); - FName = EntrySplit.first; + FName = ProfileFuncRef(EntrySplit.first); LineLoc = {0, 0}; if (!EntrySplit.second.empty()) { @@ -616,7 +621,7 @@ class SampleContext { void clearState(ContextStateMask S) { State &= (uint32_t)~S; } bool hasContext() const { return State != UnknownContext; } bool isBaseContext() const { return FullContext.size() == 1; } - StringRef getName() const { return Name; } + ProfileFuncRef getName() const { return Name; } SampleContextFrames getContextFrames() const { return FullContext; } static std::string getContextString(SampleContextFrames Context, @@ -641,14 +646,11 @@ class SampleContext { uint64_t getHashCode() const { if (hasContext()) return hash_value(getContextFrames()); - - // For non-context function name, use its MD5 as hash value, so that it is - // consistent with the profile map's key. - return hashFuncName(getName()); + return getName().getHashCode(); } /// Set the name of the function and clear the current context. - void setName(StringRef FunctionName) { + void setName(ProfileFuncRef FunctionName) { Name = FunctionName; FullContext = SampleContextFrames(); State = UnknownContext; @@ -713,7 +715,7 @@ class SampleContext { private: /// Mangled name of the function. - StringRef Name; + ProfileFuncRef Name; // Full context including calling context and leaf function name SampleContextFrames FullContext; // State of the associated sample profile @@ -736,7 +738,7 @@ class SampleProfileReaderItaniumRemapper; using BodySampleMap = std::map<LineLocation, SampleRecord>; // NOTE: Using a StringMap here makes parsed profiles consume around 17% more // memory, which is *very* significant for large profiles. -using FunctionSamplesMap = std::map<std::string, FunctionSamples, std::less<>>; +using FunctionSamplesMap = std::map<ProfileFuncRef, FunctionSamples>; using CallsiteSampleMap = std::map<LineLocation, FunctionSamplesMap>; using LocToLocMap = std::unordered_map<LineLocation, LineLocation, LineLocationHash>; @@ -788,14 +790,16 @@ class FunctionSamples { sampleprof_error addCalledTargetSamples(uint32_t LineOffset, uint32_t Discriminator, - StringRef FName, uint64_t Num, + ProfileFuncRef FName, + uint64_t Num, uint64_t Weight = 1) { return BodySamples[LineLocation(LineOffset, Discriminator)].addCalledTarget( FName, Num, Weight); } sampleprof_error addSampleRecord(LineLocation Location, - const SampleRecord &SampleRecord, uint64_t Weight = 1) { + const SampleRecord &SampleRecord, + uint64_t Weight = 1) { return BodySamples[Location].merge(SampleRecord, Weight); } @@ -803,7 +807,7 @@ class FunctionSamples { // the number of body samples actually decreased. uint64_t removeCalledTargetAndBodySample(uint32_t LineOffset, uint32_t Discriminator, - StringRef FName) { + ProfileFuncRef FName) { uint64_t Count = 0; auto I = BodySamples.find(LineLocation(LineOffset, Discriminator)); if (I != BodySamples.end()) { @@ -1046,16 +1050,16 @@ class FunctionSamples { }; if (isDeclaration(SymbolMap.lookup(getFuncName()))) { // Add to the import list only when it's defined out of module. - S.insert(getGUID(getName())); + S.insert(getGUID()); } // Import hot CallTargets, which may not be available in IR because full // profile annotation cannot be done until backend compilation in ThinLTO. for (const auto &BS : BodySamples) for (const auto &TS : BS.second.getCallTargets()) - if (TS.getValue() > Threshold) { - const Function *Callee = SymbolMap.lookup(getFuncName(TS.getKey())); + if (TS.second > Threshold) { + const Function *Callee = SymbolMap.lookup(getFuncName(TS.first)); if (isDeclaration(Callee)) - S.insert(getGUID(TS.getKey())); + S.insert(TS.first.getHashCode()); } for (const auto &CS : CallsiteSamples) for (const auto &NameFS : CS.second) @@ -1063,10 +1067,12 @@ class FunctionSamples { } /// Set the name of the function. - void setName(StringRef FunctionName) { Context.setName(FunctionName); } + void setName(ProfileFuncRef FunctionName) { + Context.setName(FunctionName); + } /// Return the function name. - StringRef getName() const { return Context.getName(); } + ProfileFuncRef getName() const { return Context.getName(); } /// Return the original function name. StringRef getFuncName() const { return getFuncName(getName()); } @@ -1133,12 +1139,12 @@ class FunctionSamples { /// translate \p Name in current FunctionSamples into its original name /// by looking up in the function map GUIDToFuncNameMap. /// If the original name doesn't exist in the map, return empty StringRef. - StringRef getFuncName(StringRef Name) const { + StringRef getFuncName(ProfileFuncRef Name) const { if (!UseMD5) - return Name; + return reinterpret_cast<StringRef &>(Name); assert(GUIDToFuncNameMap && "GUIDToFuncNameMap needs to be populated first"); - return GUIDToFuncNameMap->lookup(std::stoull(Name.data())); + return GUIDToFuncNameMap->lookup(Name.getHashCode()); } /// Returns the line offset to the start line of the subprogram. @@ -1154,7 +1160,9 @@ class FunctionSamples { /// Returns a unique hash code for a combination of a callsite location and /// the callee function name. - static uint64_t getCallSiteHash(StringRef CalleeName, + /// Guarantee MD5 and non-MD5 representation of the same function results in + /// the same hash. + static uint64_t getCallSiteHash(ProfileFuncRef CalleeName, const LineLocation &Callsite); /// Get the FunctionSamples of the inline instance where DIL originates @@ -1195,16 +1203,15 @@ class FunctionSamples { /// all the function symbols defined or declared in current module. DenseMap<uint64_t, StringRef> *GUIDToFuncNameMap = nullptr; - // Assume the input \p Name is a name coming from FunctionSamples itself. - // If UseMD5 is true, the name is already a GUID and we - // don't want to return the GUID of GUID. - static uint64_t getGUID(StringRef Name) { - return UseMD5 ? std::stoull(Name.data()) : Function::getGUID(Name); + /// Return the GUID of the context's name. If the context is already using + /// MD5, don't hash it again. + uint64_t getGUID() const { + return getName().getHashCode(); } // Find all the names in the current FunctionSamples including names in // all the inline instances and names of call targets. - void findAllNames(DenseSet<StringRef> &NameSet) const; + void findAllNames(DenseSet<ProfileFuncRef> &NameSet) const; bool operator==(const FunctionSamples &Other) const { return (GUIDToFuncNameMap == Other.GUIDToFuncNameMap || @@ -1221,9 +1228,6 @@ class FunctionSamples { return !(*this == Other); } - template <typename T> - const T &getKey() const; - private: /// CFG hash value for the function. uint64_t FunctionHash = 0; @@ -1287,9 +1291,8 @@ class FunctionSamples { const LocToLocMap *IRToProfileLocationMap = nullptr; }; -template <> -inline const SampleContext &FunctionSamples::getKey<SampleContext>() const { - return getContext(); +static inline ProfileFuncRef getRepInFormat(StringRef Name) { + return getRepInFormat(Name, FunctionSamples::UseMD5); } raw_ostream &operator<<(raw_ostream &OS, const FunctionSamples &FS); @@ -1398,11 +1401,6 @@ class SampleProfileMap Ctx); } - // Overloaded find() to lookup a function by name. - iterator find(StringRef Fname) { - return base_type::find(hashFuncName(Fname)); - } - size_t erase(const SampleContext &Ctx) { return HashKeyMap<std::unordered_map, SampleContext, FunctionSamples>:: erase(Ctx); @@ -1473,7 +1471,7 @@ class ProfileConverter { // profile. void convertCSProfiles(); struct FrameNode { - FrameNode(StringRef FName = StringRef(), + FrameNode(ProfileFuncRef FName = ProfileFuncRef(), FunctionSamples *FSamples = nullptr, LineLocation CallLoc = {0, 0}) : FuncName(FName), FuncSamples(FSamples), CallSiteLoc(CallLoc){}; @@ -1481,14 +1479,14 @@ class ProfileConverter { // Map line+discriminator location to child frame std::map<uint64_t, FrameNode> AllChildFrames; // Function name for current frame - StringRef FuncName; + ProfileFuncRef FuncName; // Function Samples for current frame FunctionSamples *FuncSamples; // Callsite location in parent context LineLocation CallSiteLoc; FrameNode *getOrCreateChildFrame(const LineLocation &CallSite, - StringRef CalleeName); + ProfileFuncRef CalleeName); }; static void flattenProfile(SampleProfileMap &ProfileMap, @@ -1623,7 +1621,9 @@ using namespace sampleprof; template <> struct DenseMapInfo<SampleContext> { static inline SampleContext getEmptyKey() { return SampleContext(); } - static inline SampleContext getTombstoneKey() { return SampleContext("@"); } + static inline SampleContext getTombstoneKey() { + return SampleContext(ProfileFuncRef(~1ULL)); + } static unsigned getHashValue(const SampleContext &Val) { return Val.getHashCode(); diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h index a5b1df3ef550b2b..df30722cb1b5bc3 100644 --- a/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -409,13 +409,13 @@ class SampleProfileReader { /// Return the samples collected for function \p F. FunctionSamples *getSamplesFor(StringRef Fname) { - auto It = Profiles.find(Fname); + auto It = Profiles.find(ProfileFuncRef(Fname)); if (It != Profiles.end()) return &It->second; if (Remapper) { if (auto NameInProfile = Remapper->lookUpNameInProfile(Fname)) { - auto It = Profiles.find(*NameInProfile); + auto It = Profiles.find(ProfileFuncRef(*NameInProfile)); if (It != Profiles.end()) return &It->second; } @@ -474,7 +474,7 @@ class SampleProfileReader { /// It includes all the names that have samples either in outline instance /// or inline instance. - virtual std::vector<StringRef> *getNameTable() { return nullptr; } + virtual std::vector<ProfileFuncRef> *getNameTable() { return nullptr; } virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) { return false; }; /// Return whether names in the profile are all MD5 numbers. @@ -508,10 +508,6 @@ class SampleProfileReader { /// Memory buffer holding the profile file. std::unique_ptr<MemoryBuffer> Buffer; - /// Extra name buffer holding names created on demand. - /// This should only be needed for md5 profiles. - std::unordered_set<std::string> MD5NameBuffer; - /// Profile summary information. std::unique_ptr<ProfileSummary> Summary; @@ -595,7 +591,9 @@ class SampleProfileReaderBinary : public SampleProfileReader { /// It includes all the names that have samples either in outline instance /// or inline instance. - std::vector<StringRef> *getNameTable() override { return &NameTable; } + std::vector<ProfileFuncRef> *getNameTable() override { + return &NameTable; + } protected: /// Read a numeric value of type T from the profile. @@ -637,7 +635,7 @@ class SampleProfileReaderBinary : public SampleProfileReader { std::error_code readNameTable(); /// Read a string indirectly via the name table. Optionally return the index. - ErrorOr<StringRef> readStringFromTable(size_t *RetIdx = nullptr); + ErrorOr<ProfileFuncRef> readStringFromTable(size_t *RetIdx = nullptr); /// Read a context indirectly via the CSNameTable. Optionally return the /// index. @@ -654,16 +652,7 @@ class SampleProfileReaderBinary : public SampleProfileReader { const uint8_t *End = nullptr; /// Function name table. - std::vector<StringRef> NameTable; - - /// If MD5 is used in NameTable section, the section saves uint64_t data. - /// The uint64_t data has to be converted to a string and then the string - /// will be used to initialize StringRef in NameTable. - /// Note NameTable contains StringRef so it needs another buffer to own - /// the string data. MD5StringBuf serves as the string buffer that is - /// referenced by NameTable (vector of StringRef). We make sure - /// the lifetime of MD5StringBuf is not shorter than that of NameTable. - std::vector<std::string> MD5StringBuf; + std::vector<ProfileFuncRef> NameTable; /// The starting address of fixed length MD5 name table section. const uint8_t *MD5NameMemStart = nullptr; diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h index 1f19283ea1dd0a8..3eff17e5e5f0900 100644 --- a/llvm/include/llvm/ProfileData/SampleProfWriter.h +++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h @@ -196,20 +196,20 @@ class SampleProfileWriterBinary : public SampleProfileWriter { std::error_code writeSample(const FunctionSamples &S) override; protected: - virtual MapVector<StringRef, uint32_t> &getNameTable() { return NameTable; } + virtual MapVector<ProfileFuncRef, uint32_t> &getNameTable() { return NameTable; } virtual std::error_code writeMagicIdent(SampleProfileFormat Format); virtual std::error_code writeNameTable(); std::error_code writeHeader(const SampleProfileMap &ProfileMap) override; std::error_code writeSummary(); virtual std::error_code writeContextIdx(const SampleContext &Context); - std::error_code writeNameIdx(StringRef FName); + std::error_code writeNameIdx(ProfileFuncRef FName); std::error_code writeBody(const FunctionSamples &S); - inline void stablizeNameTable(MapVector<StringRef, uint32_t> &NameTable, - std::set<StringRef> &V); + inline void stablizeNameTable(MapVector<ProfileFuncRef, uint32_t> &NameTable, + std::set<ProfileFuncRef> &V); - MapVector<StringRef, uint32_t> NameTable; + MapVector<ProfileFuncRef, uint32_t> NameTable; - void addName(StringRef FName); + void addName(ProfileFuncRef FName); virtual void addContext(const SampleContext &Context); void addNames(const FunctionSamples &S); diff --git a/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h b/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h index bc8360a80bc02bd..b575058614211e5 100644 --- a/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h +++ b/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h @@ -53,9 +53,10 @@ struct ProfiledCallGraphNode { using iterator = edges::iterator; using const_iterator = edges::const_iterator; - ProfiledCallGraphNode(StringRef FName = StringRef()) : Name(FName) {} + ProfiledCallGraphNode(ProfileFuncRef FName = ProfileFuncRef()) : Name(FName) + {} - StringRef Name; + ProfileFuncRef Name; edges Edges; }; @@ -85,7 +86,7 @@ class ProfiledCallGraph { std::queue<ContextTrieNode *> Queue; for (auto &Child : ContextTracker.getRootContext().getAllChildContext()) { ContextTrieNode *Callee = &Child.second; - addProfiledFunction(ContextTracker.getFuncNameFor(Callee)); + addProfiledFunction(Callee->getFuncName()); Queue.push(Callee); } @@ -102,7 +103,7 @@ class ProfiledCallGraph { // context-based one, which may in turn block context-based inlining. for (auto &Child : Caller->getAllChildContext()) { ContextTrieNode *Callee = &Child.second; - addProfiledFunction(ContextTracker.getFuncNameFor(Callee)); + addProfiledFunction(Callee->getFuncName()); Queue.push(Callee); // Fetch edge weight from the profile. @@ -123,8 +124,7 @@ class ProfiledCallGraph { Weight = std::max(CallsiteCount, CalleeEntryCount); } - addProfiledCall(ContextTracker.getFuncNameFor(Caller), - ContextTracker.getFuncNameFor(Callee), Weight); + addProfiledCall(Caller->getFuncName(), Callee->getFuncName(), Weight); } } @@ -137,7 +137,7 @@ class ProfiledCallGraph { iterator end() { return Root.Edges.end(); } ProfiledCallGraphNode *getEntryNode() { return &Root; } - void addProfiledFunction(StringRef Name) { + void addProfiledFunction(ProfileFuncRef Name) { if (!ProfiledFunctions.count(Name)) { // Link to synthetic root to make sure every node is reachable // from root. This does not affect SCC order. @@ -147,7 +147,7 @@ class ProfiledCallGraph { } private: - void addProfiledCall(StringRef CallerName, StringRef CalleeName, + void addProfiledCall(ProfileFuncRef CallerName, ProfileFuncRef CalleeName, uint64_t Weight = 0) { assert(ProfiledFunctions.count(CallerName)); auto CalleeIt = ProfiledFunctions.find(CalleeName); @@ -168,19 +168,19 @@ class ProfiledCallGraph { } void addProfiledCalls(const FunctionSamples &Samples) { - addProfiledFunction(Samples.getFuncName()); + addProfiledFunction(Samples.getName()); for (const auto &Sample : Samples.getBodySamples()) { for (const auto &[Target, Frequency] : Sample.second.getCallTargets()) { addProfiledFunction(Target); - addProfiledCall(Samples.getFuncName(), Target, Frequency); + addProfiledCall(Samples.getName(), Target, Frequency); } } for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) { for (const auto &InlinedSamples : CallsiteSamples.second) { addProfiledFunction(InlinedSamples.first); - addProfiledCall(Samples.getFuncName(), InlinedSamples.first, + addProfiledCall(Samples.getName(), InlinedSamples.first, InlinedSamples.second.getHeadSamplesEstimate()); addProfiledCalls(InlinedSamples.second); } @@ -206,7 +206,7 @@ class ProfiledCallGraph { } ProfiledCallGraphNode Root; - StringMap<ProfiledCallGraphNode> ProfiledFunctions; + std::unordered_map<ProfileFuncRef, ProfiledCallGraphNode> ProfiledFunctions; }; } // end namespace sampleprof diff --git a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h index 347dac1e9684db1..270dd25672b9f36 100644 --- a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h +++ b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h @@ -35,20 +35,20 @@ class Instruction; class ContextTrieNode { public: ContextTrieNode(ContextTrieNode *Parent = nullptr, - StringRef FName = StringRef(), + ProfileFuncRef FName = ProfileFuncRef(), FunctionSamples *FSamples = nullptr, LineLocation CallLoc = {0, 0}) : ParentContext(Parent), FuncName(FName), FuncSamples(FSamples), CallSiteLoc(CallLoc){}; ContextTrieNode *getChildContext(const LineLocation &CallSite, - StringRef ChildName); + ProfileFuncRef ChildName); ContextTrieNode *getHottestChildContext(const LineLocation &CallSite); ContextTrieNode *getOrCreateChildContext(const LineLocation &CallSite, - StringRef ChildName, + ProfileFuncRef ChildName, bool AllowCreate = true); - void removeChildContext(const LineLocation &CallSite, StringRef ChildName); + void removeChildContext(const LineLocation &CallSite, ProfileFuncRef ChildName); std::map<uint64_t, ContextTrieNode> &getAllChildContext(); - StringRef getFuncName() const; + ProfileFuncRef getFuncName() const; FunctionSamples *getFunctionSamples() const; void setFunctionSamples(FunctionSamples *FSamples); std::optional<uint32_t> getFunctionSize() const; @@ -68,7 +68,7 @@ class ContextTrieNode { ContextTrieNode *ParentContext; // Function name for current context - StringRef FuncName; + ProfileFuncRef FuncName; // Function Samples for current context FunctionSamples *FuncSamples; @@ -118,7 +118,8 @@ class SampleContextTracker { FunctionSamples *getBaseSamplesFor(const Function &Func, bool MergeContext = true); // Query base profile for a given function by name. - FunctionSamples *getBaseSamplesFor(StringRef Name, bool MergeContext = true); + FunctionSamples *getBaseSamplesFor(ProfileFuncRef Name, + bool MergeContext = true); // Retrieve the context trie node for given profile context ContextTrieNode *getContextFor(const SampleContext &Context); // Get real function name for a given trie node. @@ -129,7 +130,7 @@ class SampleContextTracker { void markContextSamplesInlined(const FunctionSamples *InlinedSamples); ContextTrieNode &getRootContext(); void promoteMergeContextSamplesTree(const Instruction &Inst, - StringRef CalleeName); + ProfileFuncRef CalleeName); // Create a merged conext-less profile map. void createContextLessProfileMap(SampleProfileMap &ContextLessProfiles); @@ -140,7 +141,7 @@ class SampleContextTracker { return nullptr; return I->second; } - StringMap<ContextSamplesTy> &getFuncToCtxtProfiles() { + std::unordered_map<ProfileFuncRef, ContextSamplesTy> &getFuncToCtxtProfiles() { return FuncToCtxtProfiles; } @@ -189,9 +190,9 @@ class SampleContextTracker { private: ContextTrieNode *getContextFor(const DILocation *DIL); ContextTrieNode *getCalleeContextFor(const DILocation *DIL, - StringRef CalleeName); - ContextTrieNode *getTopLevelContextNode(StringRef FName); - ContextTrieNode &addTopLevelContextNode(StringRef FName); + ProfileFuncRef CalleeName); + ContextTrieNode *getTopLevelContextNode(ProfileFuncRef FName); + ContextTrieNode &addTopLevelContextNode(ProfileFuncRef FName); ContextTrieNode &promoteMergeContextSamplesTree(ContextTrieNode &NodeToPromo); void mergeContextNode(ContextTrieNode &FromNode, ContextTrieNode &ToNode); ContextTrieNode & @@ -204,7 +205,7 @@ class SampleContextTracker { ProfileToNodeMap[FSample] = Node; } // Map from function name to context profiles (excluding base profile) - StringMap<ContextSamplesTy> FuncToCtxtProfiles; + std::unordered_map<ProfileFuncRef, ContextSamplesTy> FuncToCtxtProfiles; // Map from current FunctionSample to the belonged context trie. std::unordered_map<const FunctionSamples *, ContextTrieNode *> diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp index b14dc01be236245..a3de3c8e7400f0a 100644 --- a/llvm/lib/ProfileData/SampleProf.cpp +++ b/llvm/lib/ProfileData/SampleProf.cpp @@ -121,7 +121,7 @@ sampleprof_error SampleRecord::merge(const SampleRecord &Other, sampleprof_error Result; Result = addSamples(Other.getSamples(), Weight); for (const auto &I : Other.getCallTargets()) { - MergeResult(Result, addCalledTarget(I.first(), I.second, Weight)); + MergeResult(Result, addCalledTarget(I.first, I.second, Weight)); } return Result; } @@ -234,9 +234,9 @@ LineLocation FunctionSamples::getCallSiteIdentifier(const DILocation *DIL, } } -uint64_t FunctionSamples::getCallSiteHash(StringRef CalleeName, +uint64_t FunctionSamples::getCallSiteHash(ProfileFuncRef CalleeName, const LineLocation &Callsite) { - uint64_t NameHash = std::hash<std::string>{}(CalleeName.str()); + uint64_t NameHash = CalleeName.getHashCode(); uint64_t LocId = (((uint64_t)Callsite.LineOffset) << 32) | Callsite.Discriminator; return NameHash + (LocId << 5) + LocId; @@ -268,11 +268,11 @@ const FunctionSamples *FunctionSamples::findFunctionSamples( return FS; } -void FunctionSamples::findAllNames(DenseSet<StringRef> &NameSet) const { +void FunctionSamples::findAllNames(DenseSet<ProfileFuncRef> &NameSet) const { NameSet.insert(getName()); for (const auto &BS : BodySamples) for (const auto &TS : BS.second.getCallTargets()) - NameSet.insert(TS.getKey()); + NameSet.insert(TS.first); for (const auto &CS : CallsiteSamples) { for (const auto &NameFS : CS.second) { @@ -287,18 +287,15 @@ const FunctionSamples *FunctionSamples::findFunctionSamplesAt( SampleProfileReaderItaniumRemapper *Remapper) const { CalleeName = getCanonicalFnName(CalleeName); - std::string CalleeGUID; - CalleeName = getRepInFormat(CalleeName, UseMD5, CalleeGUID); - auto iter = CallsiteSamples.find(mapIRLocToProfileLoc(Loc)); if (iter == CallsiteSamples.end()) return nullptr; - auto FS = iter->second.find(CalleeName); + auto FS = iter->second.find(getRepInFormat(CalleeName)); if (FS != iter->second.end()) return &FS->second; if (Remapper) { if (auto NameInProfile = Remapper->lookUpNameInProfile(CalleeName)) { - auto FS = iter->second.find(*NameInProfile); + auto FS = iter->second.find(getRepInFormat(*NameInProfile)); if (FS != iter->second.end()) return &FS->second; } @@ -422,7 +419,7 @@ void ProfileSymbolList::dump(raw_ostream &OS) const { ProfileConverter::FrameNode * ProfileConverter::FrameNode::getOrCreateChildFrame(const LineLocation &CallSite, - StringRef CalleeName) { + ProfileFuncRef CalleeName) { uint64_t Hash = FunctionSamples::getCallSiteHash(CalleeName, CallSite); auto It = AllChildFrames.find(Hash); if (It != AllChildFrames.end()) { @@ -468,7 +465,7 @@ void ProfileConverter::convertCSProfiles(ProfileConverter::FrameNode &Node) { if (!ChildProfile) continue; SampleContext OrigChildContext = ChildProfile->getContext(); - hash_code OrigChildContextHash = OrigChildContext.getHashCode(); + uint64_t OrigChildContextHash = OrigChildContext.getHashCode(); // Reset the child context to be contextless. ChildProfile->getContext().setName(OrigChildContext.getName()); if (NodeProfile) { @@ -484,7 +481,7 @@ void ProfileConverter::convertCSProfiles(ProfileConverter::FrameNode &Node) { NodeProfile->removeTotalSamples(Count); } - hash_code NewChildProfileHash(0); + uint64_t NewChildProfileHash = 0; // Separate child profile to be a standalone profile, if the current parent // profile doesn't exist. This is a duplicating operation when the child // profile is already incorporated into the parent which is still useful and @@ -498,7 +495,7 @@ void ProfileConverter::convertCSProfiles(ProfileConverter::FrameNode &Node) { ProfileMap[ChildProfile->getContext()].merge(*ChildProfile); NewChildProfileHash = ChildProfile->getContext().getHashCode(); auto &SamplesMap = NodeProfile->functionSamplesAt(ChildNode.CallSiteLoc); - SamplesMap[ChildProfile->getName().str()].getContext().setAttribute( + SamplesMap[ChildProfile->getName()].getContext().setAttribute( ContextDuplicatedIntoBase); } diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp index 6d355cdc41840b4..86306c14a048da4 100644 --- a/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/llvm/lib/ProfileData/SampleProfReader.cpp @@ -91,7 +91,7 @@ static void dumpFunctionProfileJson(const FunctionSamples &S, JOS.attributeArray("calls", [&] { for (const auto &J : CallTargets) { JOS.object([&] { - JOS.attribute("function", J.first); + JOS.attribute("function", J.first.str()); JOS.attribute("samples", J.second); }); } @@ -117,7 +117,7 @@ static void dumpFunctionProfileJson(const FunctionSamples &S, }; JOS.object([&] { - JOS.attribute("name", S.getName()); + JOS.attribute("name", S.getName().str()); JOS.attribute("total", S.getTotalSamples()); if (TopLevel) JOS.attribute("head", S.getHeadSamples()); @@ -392,8 +392,8 @@ std::error_code SampleProfileReaderText::readImpl() { switch (LineTy) { case LineType::CallSiteProfile: { FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt( - LineLocation(LineOffset, Discriminator))[std::string(FName)]; - FSamples.setName(FName); + LineLocation(LineOffset, Discriminator))[ProfileFuncRef(FName)]; + FSamples.setName(ProfileFuncRef(FName)); MergeResult(Result, FSamples.addTotalSamples(NumSamples)); InlineStack.push_back(&FSamples); DepthMetadata = 0; @@ -406,7 +406,8 @@ std::error_code SampleProfileReaderText::readImpl() { FunctionSamples &FProfile = *InlineStack.back(); for (const auto &name_count : TargetCountMap) { MergeResult(Result, FProfile.addCalledTargetSamples( - LineOffset, Discriminator, name_count.first, + LineOffset, Discriminator, + ProfileFuncRef(name_count.first), name_count.second)); } MergeResult(Result, FProfile.addBodySamples(LineOffset, Discriminator, @@ -523,24 +524,21 @@ inline ErrorOr<size_t> SampleProfileReaderBinary::readStringIndex(T &Table) { return *Idx; } -ErrorOr<StringRef> +ErrorOr<ProfileFuncRef> SampleProfileReaderBinary::readStringFromTable(size_t *RetIdx) { auto Idx = readStringIndex(NameTable); if (std::error_code EC = Idx.getError()) return EC; - // Lazy loading, if the string has not been materialized from memory storing - // MD5 values, then it is default initialized with the null pointer. This can - // only happen when using fixed length MD5, that bounds check is performed - // while parsing the name table to ensure MD5NameMemStart points to an array - // with enough MD5 entries. - StringRef &SR = NameTable[*Idx]; - if (!SR.data()) { + // If using fixed length MD5, just read directly from the pointer to the name + // table. + ProfileFuncRef &SR = NameTable[*Idx]; + if (SR.empty()) { assert(MD5NameMemStart); using namespace support; uint64_t FID = endian::read<uint64_t, little, unaligned>( MD5NameMemStart + (*Idx) * sizeof(uint64_t)); - SR = MD5StringBuf.emplace_back(std::to_string(FID)); + SR = ProfileFuncRef(FID); } if (RetIdx) *RetIdx = *Idx; @@ -662,7 +660,7 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask(); FunctionSamples &CalleeProfile = FProfile.functionSamplesAt( - LineLocation(*LineOffset, DiscriminatorVal))[std::string(*FName)]; + LineLocation(*LineOffset, DiscriminatorVal))[*FName]; CalleeProfile.setName(*FName); if (std::error_code EC = readProfile(CalleeProfile)) return EC; @@ -902,12 +900,16 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() { for (const auto &NameOffset : FuncOffsetList) { const auto &FContext = NameOffset.first; auto FName = FContext.getName(); + StringRef FNameString; + if (!useMD5()) { + FNameString = reinterpret_cast<StringRef &>(FName); + } // For function in the current module, keep its farthest ancestor // context. This can be used to load itself and its child and // sibling contexts. - if ((useMD5() && FuncGuidsToUse.count(std::stoull(FName.data()))) || - (!useMD5() && (FuncsToUse.count(FName) || - (Remapper && Remapper->exist(FName))))) { + if ((useMD5() && FuncGuidsToUse.count(FName.getHashCode())) || + (!useMD5() && (FuncsToUse.count(FNameString) || + (Remapper && Remapper->exist(FNameString))))) { if (!CommonContext || !CommonContext->IsPrefixOf(FContext)) CommonContext = &FContext; } @@ -937,7 +939,8 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() { for (auto NameOffset : FuncOffsetList) { SampleContext FContext(NameOffset.first); auto FuncName = FContext.getName(); - if (!FuncsToUse.count(FuncName) && !Remapper->exist(FuncName)) + StringRef FuncNameStr = reinterpret_cast<StringRef &>(FuncName); + if (!FuncsToUse.count(FuncNameStr) && !Remapper->exist(FuncNameStr)) continue; const uint8_t *FuncProfileAddr = Start + NameOffset.second; if (std::error_code EC = readFuncProfile(FuncProfileAddr)) @@ -1069,8 +1072,6 @@ std::error_code SampleProfileReaderBinary::readNameTable() { // tables mixing string and MD5, all of them have to be normalized to use MD5, // because optimization passes can only handle either type. bool UseMD5 = useMD5(); - if (UseMD5) - MD5StringBuf.reserve(MD5StringBuf.size() + *Size); NameTable.clear(); NameTable.reserve(*Size); @@ -1089,12 +1090,12 @@ std::error_code SampleProfileReaderBinary::readNameTable() { if (std::error_code EC = Name.getError()) return EC; if (UseMD5) { - uint64_t FID = hashFuncName(*Name); + ProfileFuncRef FID(*Name); if (!ProfileIsCS) - MD5SampleContextTable.emplace_back(FID); - NameTable.emplace_back(MD5StringBuf.emplace_back(std::to_string(FID))); + MD5SampleContextTable.emplace_back(FID.getHashCode()); + NameTable.emplace_back(FID); } else - NameTable.push_back(*Name); + NameTable.push_back(ProfileFuncRef(*Name)); } if (!ProfileIsCS) MD5SampleContextStart = MD5SampleContextTable.data(); @@ -1121,7 +1122,6 @@ SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5, // index has been read before by checking whether the element in the // NameTable is empty, meanwhile readStringIndex can do the boundary // check using the size of NameTable. - MD5StringBuf.reserve(MD5StringBuf.size() + *Size); NameTable.clear(); NameTable.resize(*Size); MD5NameMemStart = Data; @@ -1137,7 +1137,6 @@ SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5, if (std::error_code EC = Size.getError()) return EC; - MD5StringBuf.reserve(MD5StringBuf.size() + *Size); NameTable.clear(); NameTable.reserve(*Size); if (!ProfileIsCS) @@ -1148,7 +1147,7 @@ SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5, return EC; if (!ProfileIsCS) support::endian::write64le(&MD5SampleContextTable[I], *FID); - NameTable.emplace_back(MD5StringBuf.emplace_back(std::to_string(*FID))); + NameTable.emplace_back(ProfileFuncRef(*FID)); } if (!ProfileIsCS) MD5SampleContextStart = MD5SampleContextTable.data(); @@ -1250,7 +1249,7 @@ SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute, CalleeProfile = const_cast<FunctionSamples *>( &FProfile->functionSamplesAt(LineLocation( *LineOffset, - *Discriminator))[std::string(FContext.getName())]); + *Discriminator))[FContext.getName()]); } if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, CalleeProfile)) @@ -1660,7 +1659,7 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile( // body, there will be identical replicated profiles for the // original function. In this case, we simply not bother updating // the profile of the original function. - FProfile = &Profiles[Name]; + FProfile = &Profiles[ProfileFuncRef(Name)]; FProfile->addHeadSamples(HeadCount); if (FProfile->getTotalSamples() > 0) Update = false; @@ -1672,9 +1671,9 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile( uint32_t LineOffset = Offset >> 16; uint32_t Discriminator = Offset & 0xffff; FProfile = &CallerProfile->functionSamplesAt( - LineLocation(LineOffset, Discriminator))[std::string(Name)]; + LineLocation(LineOffset, Discriminator))[ProfileFuncRef(Name)]; } - FProfile->setName(Name); + FProfile->setName(ProfileFuncRef(Name)); for (uint32_t I = 0; I < NumPosCounts; ++I) { uint32_t Offset; @@ -1730,7 +1729,8 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile( if (Update) FProfile->addCalledTargetSamples(LineOffset, Discriminator, - TargetName, TargetCount); + ProfileFuncRef(TargetName), + TargetCount); } } @@ -1791,11 +1791,13 @@ void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) { // We will need to remap the entire context string. assert(Remappings && "should be initialized while creating remapper"); for (auto &Sample : Reader.getProfiles()) { - DenseSet<StringRef> NamesInSample; + DenseSet<ProfileFuncRef> NamesInSample; Sample.second.findAllNames(NamesInSample); - for (auto &Name : NamesInSample) - if (auto Key = Remappings->insert(Name)) - NameMap.insert({Key, Name}); + for (auto &Name : NamesInSample) { + StringRef NameStr = reinterpret_cast<StringRef &>(Name); + if (auto Key = Remappings->insert(NameStr)) + NameMap.insert({Key, NameStr}); + } } RemappingApplied = true; diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp index f418dbe71057095..1d5c811331cc9bf 100644 --- a/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -348,7 +348,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() { return SampleProfileWriterBinary::writeNameTable(); auto &OS = *OutputStream; - std::set<StringRef> V; + std::set<ProfileFuncRef> V; stablizeNameTable(NameTable, V); // Write out the MD5 name table. We wrote unencoded MD5 so reader can @@ -357,7 +357,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() { encodeULEB128(NameTable.size(), OS); support::endian::Writer Writer(OS, support::little); for (auto N : V) - Writer.write(hashFuncName(N)); + Writer.write(N.getHashCode()); return sampleprof_error::success; } @@ -371,10 +371,14 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection( // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag // so compiler won't strip the suffix during profile matching after // seeing the flag in the profile. - for (const auto &I : NameTable) { - if (I.first.contains(FunctionSamples::UniqSuffix)) { - addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagUniqSuffix); - break; + // Original names are unavailable if using MD5, so this option has no use. + if (!UseMD5) { + std::string Dummy; + for (const auto &I : NameTable) { + if (I.first.stringRef(Dummy).contains(FunctionSamples::UniqSuffix)) { + addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagUniqSuffix); + break; + } } } @@ -632,7 +636,7 @@ SampleProfileWriterBinary::writeContextIdx(const SampleContext &Context) { return writeNameIdx(Context.getName()); } -std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName) { +std::error_code SampleProfileWriterBinary::writeNameIdx(ProfileFuncRef FName) { auto &NTable = getNameTable(); const auto &Ret = NTable.find(FName); if (Ret == NTable.end()) @@ -641,7 +645,7 @@ std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName) { return sampleprof_error::success; } -void SampleProfileWriterBinary::addName(StringRef FName) { +void SampleProfileWriterBinary::addName(ProfileFuncRef FName) { auto &NTable = getNameTable(); NTable.insert(std::make_pair(FName, 0)); } @@ -655,7 +659,7 @@ void SampleProfileWriterBinary::addNames(const FunctionSamples &S) { for (const auto &I : S.getBodySamples()) { const SampleRecord &Sample = I.second; for (const auto &J : Sample.getCallTargets()) - addName(J.first()); + addName(J.first); } // Recursively add all the names for inlined callsites. @@ -679,18 +683,18 @@ void SampleProfileWriterExtBinaryBase::addContext( } void SampleProfileWriterBinary::stablizeNameTable( - MapVector<StringRef, uint32_t> &NameTable, std::set<StringRef> &V) { + MapVector<ProfileFuncRef, uint32_t> &NameTable, std::set<ProfileFuncRef> &V) { // Sort the names to make NameTable deterministic. for (const auto &I : NameTable) V.insert(I.first); int i = 0; - for (const StringRef &N : V) + for (const ProfileFuncRef &N : V) NameTable[N] = i++; } std::error_code SampleProfileWriterBinary::writeNameTable() { auto &OS = *OutputStream; - std::set<StringRef> V; + std::set<ProfileFuncRef> V; stablizeNameTable(NameTable, V); // Write out the name table. @@ -836,7 +840,7 @@ std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) { encodeULEB128(Sample.getSamples(), OS); encodeULEB128(Sample.getCallTargets().size(), OS); for (const auto &J : Sample.getSortedCallTargets()) { - StringRef Callee = J.first; + auto Callee = J.first; uint64_t CalleeSamples = J.second; if (std::error_code EC = writeNameIdx(Callee)) return EC; diff --git a/llvm/lib/Target/X86/X86InsertPrefetch.cpp b/llvm/lib/Target/X86/X86InsertPrefetch.cpp index 29ae05bf0c94673..a7d74275670d521 100644 --- a/llvm/lib/Target/X86/X86InsertPrefetch.cpp +++ b/llvm/lib/Target/X86/X86InsertPrefetch.cpp @@ -125,7 +125,8 @@ bool X86InsertPrefetch::findPrefetchInfo(const FunctionSamples *TopSamples, // Convert serialized prefetch hints into PrefetchInfo objects, and populate // the Prefetches vector. for (const auto &S_V : *T) { - StringRef Name = S_V.getKey(); + std::string Buffer; + StringRef Name = S_V.first.stringRef(Buffer); if (Name.consume_front(SerializedPrefetchPrefix)) { int64_t D = static_cast<int64_t>(S_V.second); unsigned IID = 0; diff --git a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp index 79bee8362ef8a9c..7bc692e4332fe0f 100644 --- a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp +++ b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp @@ -29,7 +29,7 @@ using namespace sampleprof; namespace llvm { ContextTrieNode *ContextTrieNode::getChildContext(const LineLocation &CallSite, - StringRef CalleeName) { + ProfileFuncRef CalleeName) { if (CalleeName.empty()) return getHottestChildContext(CallSite); @@ -104,7 +104,7 @@ SampleContextTracker::moveContextSamples(ContextTrieNode &ToNodeParent, } void ContextTrieNode::removeChildContext(const LineLocation &CallSite, - StringRef CalleeName) { + ProfileFuncRef CalleeName) { uint64_t Hash = FunctionSamples::getCallSiteHash(CalleeName, CallSite); // Note this essentially calls dtor and destroys that child context AllChildContext.erase(Hash); @@ -114,7 +114,7 @@ std::map<uint64_t, ContextTrieNode> &ContextTrieNode::getAllChildContext() { return AllChildContext; } -StringRef ContextTrieNode::getFuncName() const { return FuncName; } +ProfileFuncRef ContextTrieNode::getFuncName() const { return FuncName; } FunctionSamples *ContextTrieNode::getFunctionSamples() const { return FuncSamples; @@ -178,7 +178,7 @@ void ContextTrieNode::dumpTree() { } ContextTrieNode *ContextTrieNode::getOrCreateChildContext( - const LineLocation &CallSite, StringRef CalleeName, bool AllowCreate) { + const LineLocation &CallSite, ProfileFuncRef CalleeName, bool AllowCreate) { uint64_t Hash = FunctionSamples::getCallSiteHash(CalleeName, CallSite); auto It = AllChildContext.find(Hash); if (It != AllChildContext.end()) { @@ -232,14 +232,12 @@ SampleContextTracker::getCalleeContextSamplesFor(const CallBase &Inst, return nullptr; CalleeName = FunctionSamples::getCanonicalFnName(CalleeName); - // Convert real function names to MD5 names, if the input profile is - // MD5-based. - std::string FGUID; - CalleeName = getRepInFormat(CalleeName, FunctionSamples::UseMD5, FGUID); + + ProfileFuncRef FName = getRepInFormat(CalleeName); // For indirect call, CalleeName will be empty, in which case the context // profile for callee with largest total samples will be returned. - ContextTrieNode *CalleeContext = getCalleeContextFor(DIL, CalleeName); + ContextTrieNode *CalleeContext = getCalleeContextFor(DIL, FName); if (CalleeContext) { FunctionSamples *FSamples = CalleeContext->getFunctionSamples(); LLVM_DEBUG(if (FSamples) { @@ -305,27 +303,23 @@ SampleContextTracker::getContextSamplesFor(const SampleContext &Context) { SampleContextTracker::ContextSamplesTy & SampleContextTracker::getAllContextSamplesFor(const Function &Func) { StringRef CanonName = FunctionSamples::getCanonicalFnName(Func); - return FuncToCtxtProfiles[CanonName]; + return FuncToCtxtProfiles[getRepInFormat(CanonName)]; } SampleContextTracker::ContextSamplesTy & SampleContextTracker::getAllContextSamplesFor(StringRef Name) { - return FuncToCtxtProfiles[Name]; + return FuncToCtxtProfiles[getRepInFormat(Name)]; } FunctionSamples *SampleContextTracker::getBaseSamplesFor(const Function &Func, bool MergeContext) { StringRef CanonName = FunctionSamples::getCanonicalFnName(Func); - return getBaseSamplesFor(CanonName, MergeContext); + return getBaseSamplesFor(getRepInFormat(CanonName), MergeContext); } -FunctionSamples *SampleContextTracker::getBaseSamplesFor(StringRef Name, +FunctionSamples *SampleContextTracker::getBaseSamplesFor(ProfileFuncRef Name, bool MergeContext) { LLVM_DEBUG(dbgs() << "Getting base profile for function: " << Name << "\n"); - // Convert real function names to MD5 names, if the input profile is - // MD5-based. - std::string FGUID; - Name = getRepInFormat(Name, FunctionSamples::UseMD5, FGUID); // Base profile is top-level node (child of root node), so try to retrieve // existing top-level node for given function first. If it exists, it could be @@ -373,7 +367,7 @@ void SampleContextTracker::markContextSamplesInlined( ContextTrieNode &SampleContextTracker::getRootContext() { return RootContext; } void SampleContextTracker::promoteMergeContextSamplesTree( - const Instruction &Inst, StringRef CalleeName) { + const Instruction &Inst, ProfileFuncRef CalleeName) { LLVM_DEBUG(dbgs() << "Promoting and merging context tree for instr: \n" << Inst << "\n"); // Get the caller context for the call instruction, we don't use callee @@ -457,10 +451,11 @@ SampleContextTracker::getContextString(ContextTrieNode *Node) const { void SampleContextTracker::dump() { RootContext.dumpTree(); } StringRef SampleContextTracker::getFuncNameFor(ContextTrieNode *Node) const { + std::string Dummy; if (!FunctionSamples::UseMD5) - return Node->getFuncName(); + return Node->getFuncName().stringRef(Dummy); assert(GUIDToFuncNameMap && "GUIDToFuncNameMap needs to be populated first"); - return GUIDToFuncNameMap->lookup(std::stoull(Node->getFuncName().data())); + return GUIDToFuncNameMap->lookup(Node->getFuncName().getHashCode()); } ContextTrieNode * @@ -470,7 +465,7 @@ SampleContextTracker::getContextFor(const SampleContext &Context) { ContextTrieNode * SampleContextTracker::getCalleeContextFor(const DILocation *DIL, - StringRef CalleeName) { + ProfileFuncRef CalleeName) { assert(DIL && "Expect non-null location"); ContextTrieNode *CallContext = getContextFor(DIL); @@ -485,7 +480,7 @@ SampleContextTracker::getCalleeContextFor(const DILocation *DIL, ContextTrieNode *SampleContextTracker::getContextFor(const DILocation *DIL) { assert(DIL && "Expect non-null location"); - SmallVector<std::pair<LineLocation, StringRef>, 10> S; + SmallVector<std::pair<LineLocation, ProfileFuncRef>, 10> S; // Use C++ linkage name if possible. const DILocation *PrevDIL = DIL; @@ -494,7 +489,8 @@ ContextTrieNode *SampleContextTracker::getContextFor(const DILocation *DIL) { if (Name.empty()) Name = PrevDIL->getScope()->getSubprogram()->getName(); S.push_back( - std::make_pair(FunctionSamples::getCallSiteIdentifier(DIL), Name)); + std::make_pair(FunctionSamples::getCallSiteIdentifier(DIL), + getRepInFormat(Name))); PrevDIL = DIL; } @@ -503,24 +499,14 @@ ContextTrieNode *SampleContextTracker::getContextFor(const DILocation *DIL) { StringRef RootName = PrevDIL->getScope()->getSubprogram()->getLinkageName(); if (RootName.empty()) RootName = PrevDIL->getScope()->getSubprogram()->getName(); - S.push_back(std::make_pair(LineLocation(0, 0), RootName)); - - // Convert real function names to MD5 names, if the input profile is - // MD5-based. - std::list<std::string> MD5Names; - if (FunctionSamples::UseMD5) { - for (auto &Location : S) { - MD5Names.emplace_back(); - getRepInFormat(Location.second, FunctionSamples::UseMD5, MD5Names.back()); - Location.second = MD5Names.back(); - } - } + S.push_back(std::make_pair(LineLocation(0, 0), + getRepInFormat(RootName))); ContextTrieNode *ContextNode = &RootContext; int I = S.size(); while (--I >= 0 && ContextNode) { LineLocation &CallSite = S[I].first; - StringRef CalleeName = S[I].second; + ProfileFuncRef CalleeName = S[I].second; ContextNode = ContextNode->getChildContext(CallSite, CalleeName); } @@ -553,12 +539,14 @@ SampleContextTracker::getOrCreateContextPath(const SampleContext &Context, return ContextNode; } -ContextTrieNode *SampleContextTracker::getTopLevelContextNode(StringRef FName) { +ContextTrieNode * +SampleContextTracker::getTopLevelContextNode(ProfileFuncRef FName) { assert(!FName.empty() && "Top level node query must provide valid name"); return RootContext.getChildContext(LineLocation(0, 0), FName); } -ContextTrieNode &SampleContextTracker::addTopLevelContextNode(StringRef FName) { +ContextTrieNode & +SampleContextTracker::addTopLevelContextNode(ProfileFuncRef FName) { assert(!getTopLevelContextNode(FName) && "Node to add must not exist"); return *RootContext.getOrCreateChildContext(LineLocation(0, 0), FName); } diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp index 67a2e167b6bf7dd..e89913e2795349d 100644 --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -424,7 +424,7 @@ struct CandidateComparer { return LCS->getBodySamples().size() > RCS->getBodySamples().size(); // Tie breaker using GUID so we have stable/deterministic inlining order - return LCS->getGUID(LCS->getName()) < RCS->getGUID(RCS->getName()); + return LCS->getGUID() < RCS->getGUID(); } }; @@ -467,7 +467,7 @@ class SampleProfileMatcher { private: FunctionSamples *getFlattenedSamplesFor(const Function &F) { StringRef CanonFName = FunctionSamples::getCanonicalFnName(F); - auto It = FlattenedProfiles.find(CanonFName); + auto It = FlattenedProfiles.find(ProfileFuncRef(CanonFName)); if (It != FlattenedProfiles.end()) return &It->second; return nullptr; @@ -475,18 +475,21 @@ class SampleProfileMatcher { void runOnFunction(const Function &F); void findIRAnchors(const Function &F, std::map<LineLocation, StringRef> &IRAnchors); - void findProfileAnchors(const FunctionSamples &FS, - std::map<LineLocation, StringSet<>> &ProfileAnchors); + void findProfileAnchors( + const FunctionSamples &FS, + std::map<LineLocation, std::unordered_set<ProfileFuncRef>> + &ProfileAnchors); void countMismatchedSamples(const FunctionSamples &FS); void countProfileMismatches( const Function &F, const FunctionSamples &FS, const std::map<LineLocation, StringRef> &IRAnchors, - const std::map<LineLocation, StringSet<>> &ProfileAnchors); + const std::map<LineLocation, std::unordered_set<ProfileFuncRef>> + &ProfileAnchors); void countProfileCallsiteMismatches( const FunctionSamples &FS, const std::map<LineLocation, StringRef> &IRAnchors, - const std::map<LineLocation, StringSet<>> &ProfileAnchors, - + const std::map<LineLocation, std::unordered_set<ProfileFuncRef>> + &ProfileAnchors, uint64_t &FuncMismatchedCallsites, uint64_t &FuncProfiledCallsites); LocToLocMap &getIRToProfileLocationMap(const Function &F) { auto Ret = FuncMappings.try_emplace( @@ -497,7 +500,8 @@ class SampleProfileMatcher { void distributeIRToProfileLocationMap(FunctionSamples &FS); void runStaleProfileMatching( const Function &F, const std::map<LineLocation, StringRef> &IRAnchors, - const std::map<LineLocation, StringSet<>> &ProfileAnchors, + const std::map<LineLocation, std::unordered_set<ProfileFuncRef>> + &ProfileAnchors, LocToLocMap &IRToProfileLocationMap); }; @@ -615,7 +619,7 @@ class SampleProfileLoader final : public SampleProfileLoaderBaseImpl<Function> { // All the Names used in FunctionSamples including outline function // names, inline instance names and call target names. - StringSet<> NamesInProfile; + std::unordered_set<ProfileFuncRef> NamesInProfile; // For symbol in profile symbol list, whether to regard their profiles // to be accurate. It is mainly decided by existance of profile symbol @@ -760,8 +764,7 @@ SampleProfileLoader::findIndirectCallFunctionSamples( assert(L && R && "Expect non-null FunctionSamples"); if (L->getHeadSamplesEstimate() != R->getHeadSamplesEstimate()) return L->getHeadSamplesEstimate() > R->getHeadSamplesEstimate(); - return FunctionSamples::getGUID(L->getName()) < - FunctionSamples::getGUID(R->getName()); + return L->getGUID() < R->getGUID(); }; if (FunctionSamples::ProfileIsCS) { @@ -1081,7 +1084,7 @@ void SampleProfileLoader::findExternalInlineCandidate( // just add the direct GUID and move on if (!Samples) { InlinedGUIDs.insert( - FunctionSamples::getGUID(CB->getCalledFunction()->getName())); + Function::getGUID(CB->getCalledFunction()->getName())); return; } // Otherwise, drop the threshold to import everything that we can @@ -1127,17 +1130,17 @@ void SampleProfileLoader::findExternalInlineCandidate( Function *Func = SymbolMap.lookup(Name); // Add to the import list only when it's defined out of module. if (!Func || Func->isDeclaration()) - InlinedGUIDs.insert(FunctionSamples::getGUID(CalleeSample->getName())); + InlinedGUIDs.insert(CalleeSample->getGUID()); // Import hot CallTargets, which may not be available in IR because full // profile annotation cannot be done until backend compilation in ThinLTO. for (const auto &BS : CalleeSample->getBodySamples()) for (const auto &TS : BS.second.getCallTargets()) - if (TS.getValue() > Threshold) { - StringRef CalleeName = CalleeSample->getFuncName(TS.getKey()); + if (TS.second > Threshold) { + StringRef CalleeName = CalleeSample->getFuncName(TS.first); const Function *Callee = SymbolMap.lookup(CalleeName); if (!Callee || Callee->isDeclaration()) - InlinedGUIDs.insert(FunctionSamples::getGUID(TS.getKey())); + InlinedGUIDs.insert(TS.first.getHashCode()); } // Import hot child context profile associted with callees. Note that this @@ -1644,7 +1647,7 @@ GetSortedValueDataFromCallTargets(const SampleRecord::CallTargetMap &M) { SmallVector<InstrProfValueData, 2> R; for (const auto &I : SampleRecord::SortCallTargets(M)) { R.emplace_back( - InstrProfValueData{FunctionSamples::getGUID(I.first), I.second}); + InstrProfValueData{I.first.getHashCode(), I.second}); } return R; } @@ -1871,7 +1874,8 @@ SampleProfileLoader::buildProfiledCallGraph(Module &M) { for (Function &F : M) { if (F.isDeclaration() || !F.hasFnAttribute("use-sample-profile")) continue; - ProfiledCG->addProfiledFunction(FunctionSamples::getCanonicalFnName(F)); + ProfiledCG->addProfiledFunction( + getRepInFormat(FunctionSamples::getCanonicalFnName(F))); } return ProfiledCG; @@ -1962,7 +1966,10 @@ SampleProfileLoader::buildFunctionOrder(Module &M, LazyCallGraph &CG) { Range = *SI; } for (auto *Node : Range) { - Function *F = SymbolMap.lookup(Node->Name); + Function *F = SymbolMap.lookup( + FunctionSamples::UseMD5 + ? GUIDToFuncNameMap.lookup(Node->Name.getHashCode()) + : reinterpret_cast<StringRef &>(Node->Name)); if (F && !F->isDeclaration() && F->hasFnAttribute("use-sample-profile")) FunctionOrderList.push_back(F); } @@ -2177,7 +2184,7 @@ void SampleProfileMatcher::findIRAnchors( } void SampleProfileMatcher::countMismatchedSamples(const FunctionSamples &FS) { - const auto *FuncDesc = ProbeManager->getDesc(FS.getName()); + const auto *FuncDesc = ProbeManager->getDesc(FS.getGUID()); // Skip the function that is external or renamed. if (!FuncDesc) return; @@ -2194,7 +2201,8 @@ void SampleProfileMatcher::countMismatchedSamples(const FunctionSamples &FS) { void SampleProfileMatcher::countProfileMismatches( const Function &F, const FunctionSamples &FS, const std::map<LineLocation, StringRef> &IRAnchors, - const std::map<LineLocation, StringSet<>> &ProfileAnchors) { + const std::map<LineLocation, std::unordered_set<ProfileFuncRef>> + &ProfileAnchors) { [[maybe_unused]] bool IsFuncHashMismatch = false; if (FunctionSamples::ProfileIsProbeBased) { TotalFuncHashSamples += FS.getTotalSamples(); @@ -2228,7 +2236,8 @@ void SampleProfileMatcher::countProfileMismatches( void SampleProfileMatcher::countProfileCallsiteMismatches( const FunctionSamples &FS, const std::map<LineLocation, StringRef> &IRAnchors, - const std::map<LineLocation, StringSet<>> &ProfileAnchors, + const std::map<LineLocation, std::unordered_set<ProfileFuncRef>> + &ProfileAnchors, uint64_t &FuncMismatchedCallsites, uint64_t &FuncProfiledCallsites) { // Check if there are any callsites in the profile that does not match to any @@ -2263,7 +2272,7 @@ void SampleProfileMatcher::countProfileCallsiteMismatches( // reported as mismatching. if (IRCalleeName == UnknownIndirectCallee) CallsiteIsMatched = true; - else if (Callees.size() == 1 && Callees.count(IRCalleeName)) + else if (Callees.size() == 1 && Callees.count(getRepInFormat(IRCalleeName))) CallsiteIsMatched = true; FuncProfiledCallsites++; @@ -2275,9 +2284,8 @@ void SampleProfileMatcher::countProfileCallsiteMismatches( } } -void SampleProfileMatcher::findProfileAnchors( - const FunctionSamples &FS, - std::map<LineLocation, StringSet<>> &ProfileAnchors) { +void SampleProfileMatcher::findProfileAnchors(const FunctionSamples &FS, + std::map<LineLocation, std::unordered_set<ProfileFuncRef>> &ProfileAnchors) { auto isInvalidLineOffset = [](uint32_t LineOffset) { return LineOffset & 0x8000; }; @@ -2287,8 +2295,9 @@ void SampleProfileMatcher::findProfileAnchors( if (isInvalidLineOffset(Loc.LineOffset)) continue; for (const auto &I : I.second.getCallTargets()) { - auto Ret = ProfileAnchors.try_emplace(Loc, StringSet<>()); - Ret.first->second.insert(I.first()); + auto Ret = ProfileAnchors.try_emplace(Loc, + std::unordered_set<ProfileFuncRef>()); + Ret.first->second.insert(I.first); } } @@ -2298,7 +2307,8 @@ void SampleProfileMatcher::findProfileAnchors( continue; const auto &CalleeMap = I.second; for (const auto &I : CalleeMap) { - auto Ret = ProfileAnchors.try_emplace(Loc, StringSet<>()); + auto Ret = ProfileAnchors.try_emplace(Loc, + std::unordered_set<ProfileFuncRef>()); Ret.first->second.insert(I.first); } } @@ -2322,21 +2332,24 @@ void SampleProfileMatcher::findProfileAnchors( // [1, 2, 3(foo), 4, 7, 8(bar), 9] // The output mapping: [2->3, 3->4, 5->7, 6->8, 7->9]. void SampleProfileMatcher::runStaleProfileMatching( - const Function &F, const std::map<LineLocation, StringRef> &IRAnchors, - const std::map<LineLocation, StringSet<>> &ProfileAnchors, + const Function &F, + const std::map<LineLocation, StringRef> &IRAnchors, + const std::map<LineLocation, std::unordered_set<ProfileFuncRef>> + &ProfileAnchors, LocToLocMap &IRToProfileLocationMap) { LLVM_DEBUG(dbgs() << "Run stale profile matching for " << F.getName() << "\n"); assert(IRToProfileLocationMap.empty() && "Run stale profile matching only once per function"); - StringMap<std::set<LineLocation>> CalleeToCallsitesMap; + std::unordered_map<ProfileFuncRef, std::set<LineLocation>> + CalleeToCallsitesMap; for (const auto &I : ProfileAnchors) { const auto &Loc = I.first; const auto &Callees = I.second; // Filter out possible indirect calls, use direct callee name as anchor. if (Callees.size() == 1) { - StringRef CalleeName = Callees.begin()->first(); + ProfileFuncRef CalleeName = *Callees.begin(); const auto &Candidates = CalleeToCallsitesMap.try_emplace( CalleeName, std::set<LineLocation>()); Candidates.first->second.insert(Loc); @@ -2355,11 +2368,12 @@ void SampleProfileMatcher::runStaleProfileMatching( for (const auto &IR : IRAnchors) { const auto &Loc = IR.first; - StringRef CalleeName = IR.second; + auto CalleeName = IR.second; bool IsMatchedAnchor = false; // Match the anchor location in lexical order. if (!CalleeName.empty()) { - auto CandidateAnchors = CalleeToCallsitesMap.find(CalleeName); + auto CandidateAnchors = CalleeToCallsitesMap.find( + getRepInFormat(CalleeName)); if (CandidateAnchors != CalleeToCallsitesMap.end() && !CandidateAnchors->second.empty()) { auto CI = CandidateAnchors->second.begin(); @@ -2420,7 +2434,7 @@ void SampleProfileMatcher::runOnFunction(const Function &F) { findIRAnchors(F, IRAnchors); // Anchors for profile. It's a map from callsite location to a set of callee // name. - std::map<LineLocation, StringSet<>> ProfileAnchors; + std::map<LineLocation, std::unordered_set<ProfileFuncRef>> ProfileAnchors; findProfileAnchors(*FSFlattened, ProfileAnchors); // Detect profile mismatch for profile staleness metrics report. @@ -2499,7 +2513,7 @@ void SampleProfileMatcher::runOnModule() { void SampleProfileMatcher::distributeIRToProfileLocationMap( FunctionSamples &FS) { - const auto ProfileMappings = FuncMappings.find(FS.getName()); + const auto ProfileMappings = FuncMappings.find(FS.getFuncName()); if (ProfileMappings != FuncMappings.end()) { FS.setIRToProfileLocationMap(&(ProfileMappings->second)); } @@ -2625,7 +2639,7 @@ bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM) // but not cold accumulatively...), so the outline function showing up as // cold in sampled binary will actually not be cold after current build. StringRef CanonName = FunctionSamples::getCanonicalFnName(F); - if (NamesInProfile.count(CanonName)) + if (NamesInProfile.count(getRepInFormat(CanonName))) initialEntryCount = -1; } diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index fdb2c1405f1237f..32718482296f46b 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -199,6 +199,12 @@ class SymbolRemapper { StringRef New = RemappingTable.lookup(Name); return New.empty() ? Name : New; } + + ProfileFuncRef operator()(ProfileFuncRef Name) { + std::string Buffer; + StringRef New = RemappingTable.lookup(Name.stringRef(Buffer)); + return New.empty() ? Name : ProfileFuncRef(New); + } }; } @@ -716,7 +722,8 @@ adjustInstrProfile(std::unique_ptr<WriterContext> &WC, auto BuildMaxSampleMapImpl = [&](const FunctionSamples &FS, const StringRef &RootName, auto &BuildImpl) -> void { - const StringRef &Name = FS.getName(); + std::string Buffer; + const StringRef &Name = FS.getName().stringRef(Buffer); const StringRef *NewRootName = &RootName; uint64_t EntrySample = FS.getHeadSamplesEstimate(); uint64_t MaxBodySample = FS.getMaxCountInside(/* SkipCallSite*/ true); @@ -770,7 +777,8 @@ adjustInstrProfile(std::unique_ptr<WriterContext> &WC, for (auto &PD : Reader->getProfiles()) { sampleprof::FunctionSamples &FS = PD.second; - BuildMaxSampleMap(FS, FS.getName()); + std::string Buffer; + BuildMaxSampleMap(FS, FS.getName().stringRef(Buffer)); } ProfileSummary InstrPS = *IPBuilder.getSummary(); @@ -896,7 +904,7 @@ remapSamples(const sampleprof::FunctionSamples &Samples, for (const auto &Target : BodySample.second.getCallTargets()) { Result.addCalledTargetSamples(BodySample.first.LineOffset, MaskedDiscriminator, - Remapper(Target.first()), Target.second); + Remapper(Target.first), Target.second); } } for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) { @@ -906,7 +914,7 @@ remapSamples(const sampleprof::FunctionSamples &Samples, sampleprof::FunctionSamples Remapped = remapSamples(Callsite.second, Remapper, Error); MergeResult(Error, - Target[std::string(Remapped.getName())].merge(Remapped)); + Target[Remapped.getName()].merge(Remapped)); } } return Result; diff --git a/llvm/tools/llvm-profgen/CSPreInliner.cpp b/llvm/tools/llvm-profgen/CSPreInliner.cpp index 9421118a3bb1b50..01796f41d188a02 100644 --- a/llvm/tools/llvm-profgen/CSPreInliner.cpp +++ b/llvm/tools/llvm-profgen/CSPreInliner.cpp @@ -80,8 +80,8 @@ CSPreInliner::CSPreInliner(SampleContextTracker &Tracker, ProfileInlineLimitMax = 50000; } -std::vector<StringRef> CSPreInliner::buildTopDownOrder() { - std::vector<StringRef> Order; +std::vector<ProfileFuncRef> CSPreInliner::buildTopDownOrder() { + std::vector<ProfileFuncRef> Order; // Trim cold edges to get a more stable call graph. This allows for a more // stable top-down order which in turns helps the stablity of the generated // profile from run to run. @@ -202,7 +202,7 @@ bool CSPreInliner::shouldInline(ProfiledInlineCandidate &Candidate) { return (Candidate.SizeCost < SampleThreshold); } -void CSPreInliner::processFunction(const StringRef Name) { +void CSPreInliner::processFunction(const ProfileFuncRef Name) { FunctionSamples *FSamples = ContextTracker.getBaseSamplesFor(Name); if (!FSamples) return; @@ -303,7 +303,7 @@ void CSPreInliner::run() { // It also helps better compress context profile to control profile // size, as we now only need context profile for functions going to // be inlined. - for (StringRef FuncName : buildTopDownOrder()) { + for (ProfileFuncRef FuncName : buildTopDownOrder()) { processFunction(FuncName); } diff --git a/llvm/tools/llvm-profgen/CSPreInliner.h b/llvm/tools/llvm-profgen/CSPreInliner.h index 4d848aafdab914d..85e297cddfe8544 100644 --- a/llvm/tools/llvm-profgen/CSPreInliner.h +++ b/llvm/tools/llvm-profgen/CSPreInliner.h @@ -57,8 +57,7 @@ struct ProfiledCandidateComparer { // Tie breaker using GUID so we have stable/deterministic inlining order assert(LHS.CalleeSamples && RHS.CalleeSamples && "Expect non-null FunctionSamples"); - return LHS.CalleeSamples->getGUID(LHS.CalleeSamples->getName()) < - RHS.CalleeSamples->getGUID(RHS.CalleeSamples->getName()); + return LHS.CalleeSamples->getGUID() < RHS.CalleeSamples->getGUID(); } }; @@ -81,8 +80,8 @@ class CSPreInliner { private: bool getInlineCandidates(ProfiledCandidateQueue &CQueue, const FunctionSamples *FCallerContextSamples); - std::vector<StringRef> buildTopDownOrder(); - void processFunction(StringRef Name); + std::vector<ProfileFuncRef> buildTopDownOrder(); + void processFunction(ProfileFuncRef Name); bool shouldInline(ProfiledInlineCandidate &Candidate); uint32_t getFuncSize(const ContextTrieNode *ContextNode); bool UseContextCost; diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp index d307ab465676296..f5f2bc0d800e83a 100644 --- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp +++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp @@ -586,20 +586,21 @@ void ProfileGenerator::populateBoundarySamplesWithProbesForAllFunctions( FunctionProfile.addCalledTargetSamples( FrameVec.back().Location.LineOffset, FrameVec.back().Location.Discriminator, - CalleeName, Count); + ProfileFuncRef(CalleeName), Count); } } } FunctionSamples &ProfileGenerator::getLeafProfileAndAddTotalSamples( const SampleContextFrameVector &FrameVec, uint64_t Count) { + std::string Buffer; // Get top level profile FunctionSamples *FunctionProfile = - &getTopLevelFunctionProfile(FrameVec[0].FuncName); + &getTopLevelFunctionProfile(FrameVec[0].FuncName.stringRef(Buffer)); FunctionProfile->addTotalSamples(Count); if (Binary->usePseudoProbes()) { const auto *FuncDesc = Binary->getFuncDescForGUID( - Function::getGUID(FunctionProfile->getName())); + FunctionProfile->getName().getHashCode()); FunctionProfile->setFunctionHash(FuncDesc->FuncHash); } @@ -619,7 +620,7 @@ FunctionSamples &ProfileGenerator::getLeafProfileAndAddTotalSamples( FunctionProfile->addTotalSamples(Count); if (Binary->usePseudoProbes()) { const auto *FuncDesc = Binary->getFuncDescForGUID( - Function::getGUID(FunctionProfile->getName())); + FunctionProfile->getName().getHashCode()); FunctionProfile->setFunctionHash(FuncDesc->FuncHash); } } @@ -716,7 +717,7 @@ void ProfileGenerator::populateBoundarySamplesForAllFunctions( FunctionProfile.addCalledTargetSamples( FrameVec.back().Location.LineOffset, getBaseDiscriminator(FrameVec.back().Location.Discriminator), - CalleeName, Count); + ProfileFuncRef(CalleeName), Count); } // Add head samples for callee. FunctionSamples &CalleeProfile = getTopLevelFunctionProfile(CalleeName); @@ -899,7 +900,8 @@ void CSProfileGenerator::populateBoundarySamplesForFunction( if (LeafLoc) { CallerNode->getFunctionSamples()->addCalledTargetSamples( LeafLoc->Location.LineOffset, - getBaseDiscriminator(LeafLoc->Location.Discriminator), CalleeName, + getBaseDiscriminator(LeafLoc->Location.Discriminator), + ProfileFuncRef(CalleeName), Count); // Record head sample for called target(callee) CalleeCallSite = LeafLoc->Location; @@ -907,7 +909,8 @@ void CSProfileGenerator::populateBoundarySamplesForFunction( } ContextTrieNode *CalleeNode = - CallerNode->getOrCreateChildContext(CalleeCallSite, CalleeName); + CallerNode->getOrCreateChildContext(CalleeCallSite, + ProfileFuncRef(CalleeName)); FunctionSamples *CalleeProfile = getOrCreateFunctionSamples(CalleeNode); CalleeProfile->addHeadSamples(Count); } @@ -1212,7 +1215,7 @@ void CSProfileGenerator::populateBoundarySamplesWithProbes( continue; FunctionProfile.addCalledTargetSamples(CallProbe->getIndex(), CallProbe->getDiscriminator(), - CalleeName, Count); + ProfileFuncRef(CalleeName), Count); } } diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.cpp b/llvm/tools/llvm-profgen/ProfiledBinary.cpp index 5fb51d3ac9eba78..95cb1f91bd68fe0 100644 --- a/llvm/tools/llvm-profgen/ProfiledBinary.cpp +++ b/llvm/tools/llvm-profgen/ProfiledBinary.cpp @@ -77,7 +77,7 @@ void BinarySizeContextTracker::addInstructionForContext( ContextTrieNode *CurNode = &RootContext; bool IsLeaf = true; for (const auto &Callsite : reverse(Context)) { - StringRef CallerName = Callsite.FuncName; + ProfileFuncRef CallerName = Callsite.FuncName; LineLocation CallsiteLoc = IsLeaf ? LineLocation(0, 0) : Callsite.Location; CurNode = CurNode->getOrCreateChildContext(CallsiteLoc, CallerName); IsLeaf = false; @@ -145,7 +145,8 @@ void BinarySizeContextTracker::trackInlineesOptimizedAway( StringRef CallerName = ProbeFrame.first; LineLocation CallsiteLoc(ProbeFrame.second, 0); SizeContext = - SizeContext->getOrCreateChildContext(CallsiteLoc, CallerName); + SizeContext->getOrCreateChildContext(CallsiteLoc, + ProfileFuncRef(CallerName)); } // Add 0 size to make known. SizeContext->addFunctionSize(0); @@ -831,6 +832,14 @@ void ProfiledBinary::loadSymbolsFromDWARF(ObjectFile &Obj) { if (BinaryFunctions.empty()) WithColor::warning() << "Loading of DWARF info completed, but no binary " "functions have been retrieved.\n"; + + + // Populate the hash binary function map for MD5 function name lookup. This + // is done after BinaryFunctions are finalized. + for (auto &BinaryFunction : BinaryFunctions) { + HashBinaryFunctions[MD5Hash(StringRef(BinaryFunction.first))] = + &BinaryFunction.second; + } } void ProfiledBinary::populateSymbolListFromDWARF( @@ -882,7 +891,7 @@ SampleContextFrameVector ProfiledBinary::symbolize(const InstructionPointer &IP, LineLocation Line(LineOffset, Discriminator); auto It = NameStrings.insert(FunctionName.str()); - CallStack.emplace_back(*It.first, Line); + CallStack.emplace_back(ProfileFuncRef(*It.first), Line); } return CallStack; diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.h b/llvm/tools/llvm-profgen/ProfiledBinary.h index a6d78c661cc1c31..298ec2781b38727 100644 --- a/llvm/tools/llvm-profgen/ProfiledBinary.h +++ b/llvm/tools/llvm-profgen/ProfiledBinary.h @@ -220,6 +220,10 @@ class ProfiledBinary { // A map of mapping function name to BinaryFunction info. std::unordered_map<std::string, BinaryFunction> BinaryFunctions; + // Lookup BinaryFunctions using the function name's MD5 hash. Needed if the + // profile is using MD5. + std::unordered_map<uint64_t, BinaryFunction *> HashBinaryFunctions; + // A list of binary functions that have samples. std::unordered_set<const BinaryFunction *> ProfiledFunctions; @@ -477,11 +481,17 @@ class ProfiledBinary { ProfiledFunctions = Funcs; } - BinaryFunction *getBinaryFunction(StringRef FName) { - auto I = BinaryFunctions.find(FName.str()); - if (I == BinaryFunctions.end()) + BinaryFunction *getBinaryFunction(ProfileFuncRef FName) { + if (FName.isStringRef()) { + auto I = BinaryFunctions.find(FName.str()); + if (I == BinaryFunctions.end()) + return nullptr; + return &I->second; + } + auto I = HashBinaryFunctions.find(FName.getHashCode()); + if (I == HashBinaryFunctions.end()) return nullptr; - return &I->second; + return I->second; } uint32_t getFuncSizeForContext(const ContextTrieNode *ContextNode) { @@ -556,7 +566,7 @@ class ProfiledBinary { InlineContextStack.clear(); continue; } - InlineContextStack.emplace_back(Callsite.first, + InlineContextStack.emplace_back(ProfileFuncRef(Callsite.first), LineLocation(Callsite.second, 0)); } } diff --git a/llvm/unittests/ProfileData/SampleProfTest.cpp b/llvm/unittests/ProfileData/SampleProfTest.cpp index e01025440e4a56a..a13869d901bc2eb 100644 --- a/llvm/unittests/ProfileData/SampleProfTest.cpp +++ b/llvm/unittests/ProfileData/SampleProfTest.cpp @@ -145,7 +145,7 @@ struct SampleProfTest : ::testing::Test { StringRef FooName("_Z3fooi"); FunctionSamples FooSamples; - FooSamples.setName(FooName); + FooSamples.setName(ProfileFuncRef(FooName)); FooSamples.addTotalSamples(7711); FooSamples.addHeadSamples(610); FooSamples.addBodySamples(1, 0, 610); @@ -155,43 +155,43 @@ struct SampleProfTest : ::testing::Test { FooSamples.addBodySamples(10, 0, 605); // Add inline instance with name "_Z3gooi". - StringRef GooName("_Z3gooi"); + ProfileFuncRef GooName(StringRef("_Z3gooi")); auto &GooSamples = - FooSamples.functionSamplesAt(LineLocation(7, 0))[GooName.str()]; + FooSamples.functionSamplesAt(LineLocation(7, 0))[GooName]; GooSamples.setName(GooName); GooSamples.addTotalSamples(502); GooSamples.addBodySamples(3, 0, 502); // Add inline instance with name "_Z3hooi". - StringRef HooName("_Z3hooi"); + ProfileFuncRef HooName(StringRef("_Z3hooi")); auto &HooSamples = - GooSamples.functionSamplesAt(LineLocation(9, 0))[HooName.str()]; + GooSamples.functionSamplesAt(LineLocation(9, 0))[HooName]; HooSamples.setName(HooName); HooSamples.addTotalSamples(317); HooSamples.addBodySamples(4, 0, 317); StringRef BarName("_Z3bari"); FunctionSamples BarSamples; - BarSamples.setName(BarName); + BarSamples.setName(ProfileFuncRef(BarName)); BarSamples.addTotalSamples(20301); BarSamples.addHeadSamples(1437); BarSamples.addBodySamples(1, 0, 1437); // Test how reader/writer handles unmangled names. StringRef MconstructName("_M_construct<char *>"); StringRef StringviewName("string_view<std::allocator<char> >"); - BarSamples.addCalledTargetSamples(1, 0, MconstructName, 1000); - BarSamples.addCalledTargetSamples(1, 0, StringviewName, 437); + BarSamples.addCalledTargetSamples(1, 0, ProfileFuncRef(MconstructName), 1000); + BarSamples.addCalledTargetSamples(1, 0, ProfileFuncRef(StringviewName), 437); StringRef BazName("_Z3bazi"); FunctionSamples BazSamples; - BazSamples.setName(BazName); + BazSamples.setName(ProfileFuncRef(BazName)); BazSamples.addTotalSamples(12557); BazSamples.addHeadSamples(1257); BazSamples.addBodySamples(1, 0, 12557); StringRef BooName("_Z3booi"); FunctionSamples BooSamples; - BooSamples.setName(BooName); + BooSamples.setName(ProfileFuncRef(BooName)); BooSamples.addTotalSamples(1232); BooSamples.addHeadSamples(1); BooSamples.addBodySamples(1, 0, 1232); @@ -210,8 +210,8 @@ struct SampleProfTest : ::testing::Test { if (Remap) { FooName = "_Z4fauxi"; BarName = "_Z3barl"; - GooName = "_Z3gool"; - HooName = "_Z3hool"; + GooName = ProfileFuncRef(StringRef("_Z3gool")); + HooName = ProfileFuncRef(StringRef("_Z3hool")); } M.getOrInsertFunction(FooName, fn_type); @@ -245,16 +245,17 @@ struct SampleProfTest : ::testing::Test { FunctionSamples *ReadFooSamples = Reader->getSamplesFor(FooName); ASSERT_TRUE(ReadFooSamples != nullptr); if (!UseMD5) { - ASSERT_EQ("_Z3fooi", ReadFooSamples->getName()); + ASSERT_EQ("_Z3fooi", ReadFooSamples->getName().str()); } ASSERT_EQ(7711u, ReadFooSamples->getTotalSamples()); ASSERT_EQ(610u, ReadFooSamples->getHeadSamples()); + std::string Buffer; // Try to find a FunctionSamples with GooName at given callsites containing // inline instance for GooName. Test the correct FunctionSamples can be // found with Remapper support. const FunctionSamples *ReadGooSamples = - ReadFooSamples->findFunctionSamplesAt(LineLocation(7, 0), GooName, + ReadFooSamples->findFunctionSamplesAt(LineLocation(7, 0), GooName.stringRef(Buffer), Reader->getRemapper()); ASSERT_TRUE(ReadGooSamples != nullptr); ASSERT_EQ(502u, ReadGooSamples->getTotalSamples()); @@ -263,7 +264,7 @@ struct SampleProfTest : ::testing::Test { // no inline instance for GooName. Test no FunctionSamples will be // found with Remapper support. const FunctionSamples *ReadGooSamplesAgain = - ReadFooSamples->findFunctionSamplesAt(LineLocation(9, 0), GooName, + ReadFooSamples->findFunctionSamplesAt(LineLocation(9, 0), GooName.stringRef(Buffer), Reader->getRemapper()); ASSERT_TRUE(ReadGooSamplesAgain == nullptr); @@ -272,7 +273,7 @@ struct SampleProfTest : ::testing::Test { // inline instance for HooName. Test the correct FunctionSamples can be // found with Remapper support. const FunctionSamples *ReadHooSamples = - ReadGooSamples->findFunctionSamplesAt(LineLocation(9, 0), HooName, + ReadGooSamples->findFunctionSamplesAt(LineLocation(9, 0), HooName.stringRef(Buffer), Reader->getRemapper()); ASSERT_TRUE(ReadHooSamples != nullptr); ASSERT_EQ(317u, ReadHooSamples->getTotalSamples()); @@ -280,7 +281,7 @@ struct SampleProfTest : ::testing::Test { FunctionSamples *ReadBarSamples = Reader->getSamplesFor(BarName); ASSERT_TRUE(ReadBarSamples != nullptr); if (!UseMD5) { - ASSERT_EQ("_Z3bari", ReadBarSamples->getName()); + ASSERT_EQ("_Z3bari", ReadBarSamples->getName().str()); } ASSERT_EQ(20301u, ReadBarSamples->getTotalSamples()); ASSERT_EQ(1437u, ReadBarSamples->getHeadSamples()); @@ -305,12 +306,8 @@ struct SampleProfTest : ::testing::Test { ASSERT_TRUE(ReadBooSamples != nullptr); ASSERT_EQ(1232u, ReadBooSamples->getTotalSamples()); - std::string MconstructGUID; - StringRef MconstructRep = - getRepInFormat(MconstructName, UseMD5, MconstructGUID); - std::string StringviewGUID; - StringRef StringviewRep = - getRepInFormat(StringviewName, UseMD5, StringviewGUID); + ProfileFuncRef MconstructRep = getRepInFormat(MconstructName, UseMD5); + ProfileFuncRef StringviewRep = getRepInFormat(StringviewName, UseMD5); ASSERT_EQ(1000u, CTMap.get()[MconstructRep]); ASSERT_EQ(437u, CTMap.get()[StringviewRep]); @@ -333,7 +330,7 @@ struct SampleProfTest : ::testing::Test { uint64_t TotalSamples, uint64_t HeadSamples) { StringRef Name(Fname); FunctionSamples FcnSamples; - FcnSamples.setName(Name); + FcnSamples.setName(ProfileFuncRef(Name)); FcnSamples.addTotalSamples(TotalSamples); FcnSamples.addHeadSamples(HeadSamples); FcnSamples.addBodySamples(1, 0, HeadSamples); >From 11ea8142388cdf768f536142715e43ab830cef97 Mon Sep 17 00:00:00 2001 From: William Huang <williamjhu...@google.com> Date: Wed, 20 Sep 2023 20:30:30 +0000 Subject: [PATCH 2/2] Change ProfileFuncRef member name to be more descriptive --- .../include/llvm/ProfileData/ProfileFuncRef.h | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/llvm/include/llvm/ProfileData/ProfileFuncRef.h b/llvm/include/llvm/ProfileData/ProfileFuncRef.h index 60be6232372b8c6..37a12bf27d745d3 100644 --- a/llvm/include/llvm/ProfileData/ProfileFuncRef.h +++ b/llvm/include/llvm/ProfileData/ProfileFuncRef.h @@ -40,7 +40,7 @@ class ProfileFuncRef { const char *Data = nullptr; /// Use uint64_t instead of size_t so that it can also hold a MD5 value. - uint64_t Length = 0; + uint64_t LengthOrHashCode = 0; /// Extension to memcmp to handle hash code representation. If both are hash /// values, Lhs and Rhs are both null, function returns 0 (and needs an extra @@ -61,11 +61,11 @@ class ProfileFuncRef { /// Constructor from a StringRef. explicit ProfileFuncRef(StringRef Str) - : Data(Str.data()), Length(Str.size()) {} + : Data(Str.data()), LengthOrHashCode(Str.size()) {} /// Constructor from a hash code. explicit ProfileFuncRef(uint64_t HashCode) - : Data(nullptr), Length(HashCode) { + : Data(nullptr), LengthOrHashCode(HashCode) { assert(HashCode != 0); } @@ -74,10 +74,10 @@ class ProfileFuncRef { /// if so, convert the numerical string to a hash code first. We assume that /// no function name (from a profile) can be a pure number. explicit ProfileFuncRef(const std::string &Str) - : Data(Str.data()), Length(Str.size()) { + : Data(Str.data()), LengthOrHashCode(Str.size()) { // Only need to check for base 10 digits, fail faster if otherwise. if (Str.length() > 0 && isdigit(Str[0]) && - !StringRef(Str).getAsInteger(10, Length)) + !StringRef(Str).getAsInteger(10, LengthOrHashCode)) Data = nullptr; } @@ -86,8 +86,8 @@ class ProfileFuncRef { /// sufficient. A hash code ProfileFuncName is considered not equal to a /// StringRef ProfileFuncName regardless of actual contents. bool equals(const ProfileFuncRef &Other) const { - return Length == Other.Length && - compareMemory(Data, Other.Data, Length) == 0; + return LengthOrHashCode == Other.LengthOrHashCode && + compareMemory(Data, Other.Data, LengthOrHashCode) == 0; } /// Total order comparison. If both ProfileFuncName are StringRef, this is the @@ -95,20 +95,21 @@ class ProfileFuncRef { /// greater than the hash code ProfileFuncName. Otherwise this is the the /// same as comparing their int values. int compare(const ProfileFuncRef &Other) const { - auto Res = compareMemory(Data, Other.Data, std::min(Length, Other.Length)); + auto Res = compareMemory( + Data, Other.Data, std::min(LengthOrHashCode, Other.LengthOrHashCode)); if (Res != 0) return Res; - if (Length == Other.Length) + if (LengthOrHashCode == Other.LengthOrHashCode) return 0; - return Length < Other.Length ? -1 : 1; + return LengthOrHashCode < Other.LengthOrHashCode ? -1 : 1; } /// Convert to a string, usually for output purpose. std::string str() const { if (Data) - return std::string(Data, Length); - if (Length != 0) - return std::to_string(Length); + return std::string(Data, LengthOrHashCode); + if (LengthOrHashCode != 0) + return std::to_string(LengthOrHashCode); return std::string(); } @@ -117,9 +118,9 @@ class ProfileFuncRef { /// the buffer. If this object represents a StringRef, the buffer is not used. StringRef stringRef(std::string &Buffer) const { if (Data) - return StringRef(Data, Length); - if (Length != 0) { - Buffer = std::to_string(Length); + return StringRef(Data, LengthOrHashCode); + if (LengthOrHashCode != 0) { + Buffer = std::to_string(LengthOrHashCode); return Buffer; } return StringRef(); @@ -134,11 +135,11 @@ class ProfileFuncRef { /// form has the same hash code. uint64_t getHashCode() const { if (Data) - return MD5Hash(StringRef(Data, Length)); - return Length; + return MD5Hash(StringRef(Data, LengthOrHashCode)); + return LengthOrHashCode; } - bool empty() const { return Length == 0; } + bool empty() const { return LengthOrHashCode == 0; } /// Check if this object represents a StringRef, or a hash code. bool isStringRef() const { return Data != nullptr; } @@ -170,9 +171,9 @@ inline bool operator>=(const ProfileFuncRef &LHS, const ProfileFuncRef &RHS) { inline raw_ostream &operator<<(raw_ostream &OS, const ProfileFuncRef &Obj) { if (Obj.Data) - return OS << StringRef(Obj.Data, Obj.Length); - if (Obj.Length != 0) - return OS << Obj.Length; + return OS << StringRef(Obj.Data, Obj.LengthOrHashCode); + if (Obj.LengthOrHashCode != 0) + return OS << Obj.LengthOrHashCode; return OS; } _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
