================ @@ -0,0 +1,2261 @@ +//===-- NumericalStabilitySanitizer.cpp -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of NumericalStabilitySanitizer. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h" + +#include <cstdint> +#include <unordered_map> + +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Analysis/CaptureTracking.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/ValueTracking.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/InitializePasses.h" +#include "llvm/ProfileData/InstrProf.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/Regex.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Utils/EscapeEnumerator.h" +#include "llvm/Transforms/Utils/Local.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" + +using namespace llvm; + +#define DEBUG_TYPE "nsan" + +STATISTIC(NumInstrumentedFTLoads, + "Number of instrumented floating-point loads"); + +STATISTIC(NumInstrumentedFTCalls, + "Number of instrumented floating-point calls"); +STATISTIC(NumInstrumentedFTRets, + "Number of instrumented floating-point returns"); +STATISTIC(NumInstrumentedFTStores, + "Number of instrumented floating-point stores"); +STATISTIC(NumInstrumentedNonFTStores, + "Number of instrumented non floating-point stores"); +STATISTIC( + NumInstrumentedNonFTMemcpyStores, + "Number of instrumented non floating-point stores with memcpy semantics"); +STATISTIC(NumInstrumentedFCmp, "Number of instrumented fcmps"); + +// Using smaller shadow types types can help improve speed. For example, `dlq` +// is 3x slower to 5x faster in opt mode and 2-6x faster in dbg mode compared to +// `dqq`. +static cl::opt<std::string> ClShadowMapping( + "nsan-shadow-type-mapping", cl::init("dqq"), + cl::desc("One shadow type id for each of `float`, `double`, `long double`. " + "`d`,`l`,`q`,`e` mean double, x86_fp80, fp128 (quad) and " + "ppc_fp128 (extended double) respectively. The default is to " + "shadow `float` as `double`, and `double` and `x86_fp80` as " + "`fp128`"), + cl::Hidden); + +static cl::opt<bool> + ClInstrumentFCmp("nsan-instrument-fcmp", cl::init(true), + cl::desc("Instrument floating-point comparisons"), + cl::Hidden); + +static cl::opt<std::string> ClCheckFunctionsFilter( + "check-functions-filter", + cl::desc("Only emit checks for arguments of functions " + "whose names match the given regular expression"), + cl::value_desc("regex")); + +static cl::opt<bool> ClTruncateFCmpEq( + "nsan-truncate-fcmp-eq", cl::init(true), + cl::desc( + "This flag controls the behaviour of fcmp equality comparisons:" + "For equality comparisons such as `x == 0.0f`, we can perform the " + "shadow check in the shadow (`x_shadow == 0.0) == (x == 0.0f)`) or app " + " domain (`(trunc(x_shadow) == 0.0f) == (x == 0.0f)`). This helps " + "catch the case when `x_shadow` is accurate enough (and therefore " + "close enough to zero) so that `trunc(x_shadow)` is zero even though " + "both `x` and `x_shadow` are not. "), + cl::Hidden); + +// When there is external, uninstrumented code writing to memory, the shadow +// memory can get out of sync with the application memory. Enabling this flag +// emits consistency checks for loads to catch this situation. +// When everything is instrumented, this is not strictly necessary because any +// load should have a corresponding store, but can help debug cases when the +// framework did a bad job at tracking shadow memory modifications by failing on +// load rather than store. +// FIXME: provide a way to resume computations from the FT value when the load +// is inconsistent. This ensures that further computations are not polluted. +static cl::opt<bool> ClCheckLoads("nsan-check-loads", cl::init(false), + cl::desc("Check floating-point load"), + cl::Hidden); + +static cl::opt<bool> ClCheckStores("nsan-check-stores", cl::init(true), + cl::desc("Check floating-point stores"), + cl::Hidden); + +static cl::opt<bool> ClCheckRet("nsan-check-ret", cl::init(true), + cl::desc("Check floating-point return values"), + cl::Hidden); + +static const char *const kNsanModuleCtorName = "nsan.module_ctor"; +static const char *const kNsanInitName = "__nsan_init"; + +// The following values must be kept in sync with the runtime. +static constexpr const int kShadowScale = 2; +static constexpr const int kMaxVectorWidth = 8; +static constexpr const int kMaxNumArgs = 128; +static constexpr const int kMaxShadowTypeSizeBytes = 16; // fp128 + +namespace { + +// Defines the characteristics (type id, type, and floating-point semantics) +// attached for all possible shadow types. +class ShadowTypeConfig { +public: + static std::unique_ptr<ShadowTypeConfig> fromNsanTypeId(char TypeId); + // The floating-point semantics of the shadow type. + virtual const fltSemantics &semantics() const = 0; + + // The LLVM Type corresponding to the shadow type. + virtual Type *getType(LLVMContext &Context) const = 0; + + // The nsan type id of the shadow type (`d`, `l`, `q`, ...). + virtual char getNsanTypeId() const = 0; + + virtual ~ShadowTypeConfig() {} +}; + +template <char NsanTypeId> +class ShadowTypeConfigImpl : public ShadowTypeConfig { +public: + char getNsanTypeId() const override { return NsanTypeId; } + static constexpr const char kNsanTypeId = NsanTypeId; +}; + +// `double` (`d`) shadow type. +class F64ShadowConfig : public ShadowTypeConfigImpl<'d'> { + const fltSemantics &semantics() const override { + return APFloat::IEEEdouble(); + } + Type *getType(LLVMContext &Context) const override { + return Type::getDoubleTy(Context); + } +}; + +// `x86_fp80` (`l`) shadow type: X86 long double. +class F80ShadowConfig : public ShadowTypeConfigImpl<'l'> { + const fltSemantics &semantics() const override { + return APFloat::x87DoubleExtended(); + } + Type *getType(LLVMContext &Context) const override { + return Type::getX86_FP80Ty(Context); + } +}; + +// `fp128` (`q`) shadow type. +class F128ShadowConfig : public ShadowTypeConfigImpl<'q'> { + const fltSemantics &semantics() const override { return APFloat::IEEEquad(); } + Type *getType(LLVMContext &Context) const override { + return Type::getFP128Ty(Context); + } +}; + +// `ppc_fp128` (`e`) shadow type: IBM extended double with 106 bits of mantissa. +class PPC128ShadowConfig : public ShadowTypeConfigImpl<'e'> { + const fltSemantics &semantics() const override { + return APFloat::PPCDoubleDouble(); + } + Type *getType(LLVMContext &Context) const override { + return Type::getPPC_FP128Ty(Context); + } +}; + +// Creates a ShadowTypeConfig given its type id. +std::unique_ptr<ShadowTypeConfig> +ShadowTypeConfig::fromNsanTypeId(const char TypeId) { + switch (TypeId) { + case F64ShadowConfig::kNsanTypeId: + return std::make_unique<F64ShadowConfig>(); + case F80ShadowConfig::kNsanTypeId: + return std::make_unique<F80ShadowConfig>(); + case F128ShadowConfig::kNsanTypeId: + return std::make_unique<F128ShadowConfig>(); + case PPC128ShadowConfig::kNsanTypeId: + return std::make_unique<PPC128ShadowConfig>(); + } + errs() << "nsan: invalid shadow type id'" << TypeId << "'\n"; + return nullptr; +} + +// An enum corresponding to shadow value types. Used as indices in arrays, so +// not an `enum class`. +enum FTValueType { kFloat, kDouble, kLongDouble, kNumValueTypes }; + +static FTValueType semanticsToFTValueType(const fltSemantics &Sem) { + if (&Sem == &APFloat::IEEEsingle()) { + return kFloat; + } else if (&Sem == &APFloat::IEEEdouble()) { + return kDouble; + } else if (&Sem == &APFloat::x87DoubleExtended()) { + return kLongDouble; + } + llvm_unreachable("semantics are not one of the handled types"); +} + +// If `FT` corresponds to a primitive FTValueType, return it. +static std::optional<FTValueType> ftValueTypeFromType(Type *FT) { + if (FT->isFloatTy()) + return kFloat; + if (FT->isDoubleTy()) + return kDouble; + if (FT->isX86_FP80Ty()) + return kLongDouble; + return {}; +} + +// Returns the LLVM type for an FTValueType. +static Type *typeFromFTValueType(FTValueType VT, LLVMContext &Context) { + switch (VT) { + case kFloat: + return Type::getFloatTy(Context); + case kDouble: + return Type::getDoubleTy(Context); + case kLongDouble: + return Type::getX86_FP80Ty(Context); + case kNumValueTypes: + return nullptr; + } +} + +// Returns the type name for an FTValueType. +static const char *typeNameFromFTValueType(FTValueType VT) { + switch (VT) { + case kFloat: + return "float"; + case kDouble: + return "double"; + case kLongDouble: + return "longdouble"; + case kNumValueTypes: + return nullptr; + } +} + +// A specific mapping configuration of application type to shadow type for nsan +// (see -nsan-shadow-mapping flag). +class MappingConfig { +public: + bool initialize(LLVMContext *C) { + if (ClShadowMapping.size() != 3) { + errs() << "Invalid nsan mapping: " << ClShadowMapping << "\n"; + } + Context = C; + unsigned ShadowTypeSizeBits[kNumValueTypes]; + for (int VT = 0; VT < kNumValueTypes; ++VT) { + auto Config = ShadowTypeConfig::fromNsanTypeId(ClShadowMapping[VT]); + if (Config == nullptr) + return false; + const unsigned AppTypeSize = + typeFromFTValueType(static_cast<FTValueType>(VT), *C) + ->getScalarSizeInBits(); + const unsigned ShadowTypeSize = + Config->getType(*C)->getScalarSizeInBits(); + // Check that the shadow type size is at most kShadowScale times the + // application type size, so that shadow memory compoutations are valid. + if (ShadowTypeSize > kShadowScale * AppTypeSize) { + errs() << "Invalid nsan mapping f" << AppTypeSize << "->f" + << ShadowTypeSize << ": The shadow type size should be at most " + << kShadowScale << " times the application type size\n"; + return false; + } + ShadowTypeSizeBits[VT] = ShadowTypeSize; + Configs[VT] = std::move(Config); + } + + // Check that the mapping is monotonous. This is required because if one + // does an fpextend of `float->long double` in application code, nsan is + // going to do an fpextend of `shadow(float) -> shadow(long double)` in + // shadow code. This will fail in `qql` mode, since nsan would be + // fpextending `f128->long`, which is invalid. + // FIXME: Relax this. + if (ShadowTypeSizeBits[kFloat] > ShadowTypeSizeBits[kDouble] || + ShadowTypeSizeBits[kDouble] > ShadowTypeSizeBits[kLongDouble]) { + errs() << "Invalid nsan mapping: { float->f" << ShadowTypeSizeBits[kFloat] + << "; double->f" << ShadowTypeSizeBits[kDouble] + << "; long double->f" << ShadowTypeSizeBits[kLongDouble] << " }\n"; + return false; + } + return true; + } + + const ShadowTypeConfig &byValueType(FTValueType VT) const { + assert(VT < FTValueType::kNumValueTypes && "invalid value type"); + return *Configs[VT]; + } + + const ShadowTypeConfig &bySemantics(const fltSemantics &Sem) const { + return byValueType(semanticsToFTValueType(Sem)); + } + + // Returns the extended shadow type for a given application type. + Type *getExtendedFPType(Type *FT) const { + if (const auto VT = ftValueTypeFromType(FT)) + return Configs[*VT]->getType(*Context); + if (FT->isVectorTy()) { + auto *VecTy = cast<VectorType>(FT); + Type *ExtendedScalar = getExtendedFPType(VecTy->getElementType()); + return ExtendedScalar + ? VectorType::get(ExtendedScalar, VecTy->getElementCount()) + : nullptr; + } + return nullptr; + } + +private: + LLVMContext *Context = nullptr; + std::unique_ptr<ShadowTypeConfig> Configs[FTValueType::kNumValueTypes]; +}; + +// The memory extents of a type specifies how many elements of a given +// FTValueType needs to be stored when storing this type. +struct MemoryExtents { + FTValueType ValueType; + uint64_t NumElts; +}; +static MemoryExtents getMemoryExtentsOrDie(Type *FT) { + if (const auto VT = ftValueTypeFromType(FT)) + return {*VT, 1}; + if (FT->isVectorTy()) { + auto *VecTy = cast<VectorType>(FT); + const auto ScalarExtents = getMemoryExtentsOrDie(VecTy->getElementType()); + return {ScalarExtents.ValueType, + ScalarExtents.NumElts * VecTy->getElementCount().getFixedValue()}; + } + llvm_unreachable("invalid value type"); +} + +// The location of a check. Passed as parameters to runtime checking functions. +class CheckLoc { +public: + // Creates a location that references an application memory location. + static CheckLoc makeStore(Value *Address) { + CheckLoc Result(kStore); + Result.Address = Address; + return Result; + } + static CheckLoc makeLoad(Value *Address) { + CheckLoc Result(kLoad); + Result.Address = Address; + return Result; + } + + // Creates a location that references an argument, given by id. + static CheckLoc makeArg(int ArgId) { + CheckLoc Result(kArg); + Result.ArgId = ArgId; + return Result; + } + + // Creates a location that references the return value of a function. + static CheckLoc makeRet() { return CheckLoc(kRet); } + + // Creates a location that references a vector insert. + static CheckLoc makeInsert() { return CheckLoc(kInsert); } + + // Returns the CheckType of location this refers to, as an integer-typed LLVM + // IR value. + Value *getType(LLVMContext &C) const { + return ConstantInt::get(Type::getInt32Ty(C), static_cast<int>(CheckTy)); + } + + // Returns a CheckType-specific value representing details of the location + // (e.g. application address for loads or stores), as an `IntptrTy`-typed LLVM + // IR value. + Value *getValue(Type *IntptrTy, IRBuilder<> &Builder) const { + switch (CheckTy) { + case kUnknown: + llvm_unreachable("unknown type"); + case kRet: + case kInsert: + return ConstantInt::get(IntptrTy, 0); + case kArg: + return ConstantInt::get(IntptrTy, ArgId); + case kLoad: + case kStore: + return Builder.CreatePtrToInt(Address, IntptrTy); + } + } + +private: + // Must be kept in sync with the runtime. + enum CheckType { + kUnknown = 0, + kRet, + kArg, + kLoad, + kStore, + kInsert, + }; + explicit CheckLoc(CheckType CheckTy) : CheckTy(CheckTy) {} + + const CheckType CheckTy; + Value *Address = nullptr; + int ArgId = -1; +}; + +// A map of LLVM IR values to shadow LLVM IR values. +class ValueToShadowMap { +public: + explicit ValueToShadowMap(MappingConfig *Config) : Config(Config) {} + + // Sets the shadow value for a value. Asserts that the value does not already + // have a value. + void setShadow(Value *V, Value *Shadow) { + assert(V); + assert(Shadow); + const bool Inserted = Map.emplace(V, Shadow).second; +#ifdef LLVM_ENABLE_DUMP + if (!Inserted) { + if (const auto *const I = dyn_cast<Instruction>(V)) + I->getParent()->getParent()->dump(); + errs() << "duplicate shadow (" << V << "): "; + V->dump(); + } +#endif + assert(Inserted && "duplicate shadow"); + (void)Inserted; + } + + // Returns true if the value already has a shadow (including if the value is a + // constant). If true, calling getShadow() is valid. + bool hasShadow(Value *V) const { + return isa<Constant>(V) || (Map.find(V) != Map.end()); + } + + // Returns the shadow value for a given value. Asserts that the value has + // a shadow value. Lazily creates shadows for constant values. + Value *getShadow(Value *V) const { + assert(V); + if (Constant *C = dyn_cast<Constant>(V)) + return getShadowConstant(C); + const auto ShadowValIt = Map.find(V); + assert(ShadowValIt != Map.end() && "shadow val does not exist"); + assert(ShadowValIt->second && "shadow val is null"); + return ShadowValIt->second; + } + + bool empty() const { return Map.empty(); } + +private: + // Extends a constant application value to its shadow counterpart. + APFloat extendConstantFP(APFloat CV) const { + bool LosesInfo = false; + CV.convert(Config->bySemantics(CV.getSemantics()).semantics(), + APFloatBase::rmTowardZero, &LosesInfo); + return CV; + } + + // Returns the shadow constant for the given application constant. + Constant *getShadowConstant(Constant *C) const { + if (UndefValue *U = dyn_cast<UndefValue>(C)) { + return UndefValue::get(Config->getExtendedFPType(U->getType())); + } + if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) { + // Floating-point constants. + return ConstantFP::get(Config->getExtendedFPType(CFP->getType()), + extendConstantFP(CFP->getValueAPF())); + } + // Vector, array, or aggregate constants. + if (C->getType()->isVectorTy()) { + SmallVector<Constant *, 8> Elements; + for (int I = 0, E = cast<VectorType>(C->getType()) + ->getElementCount() + .getFixedValue(); + I < E; ++I) + Elements.push_back(getShadowConstant(C->getAggregateElement(I))); + return ConstantVector::get(Elements); + } + llvm_unreachable("unimplemented"); + } + + MappingConfig *const Config; + std::unordered_map<Value *, Value *> Map; +}; + +/// Instantiating NumericalStabilitySanitizer inserts the nsan runtime library +/// API function declarations into the module if they don't exist already. +/// Instantiating ensures the __nsan_init function is in the list of global +/// constructors for the module. +class NumericalStabilitySanitizer { +public: + bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI); + +private: + void initialize(Module &M); + bool instrumentMemIntrinsic(MemIntrinsic *MI); + void maybeAddSuffixForNsanInterface(CallBase *CI); + bool addrPointsToConstantData(Value *Addr); + void maybeCreateShadowValue(Instruction &Root, const TargetLibraryInfo &TLI, + ValueToShadowMap &Map); + Value *createShadowValueWithOperandsAvailable(Instruction &Inst, + const TargetLibraryInfo &TLI, + const ValueToShadowMap &Map); + PHINode *maybeCreateShadowPhi(PHINode &Phi, const TargetLibraryInfo &TLI); + void createShadowArguments(Function &F, const TargetLibraryInfo &TLI, + ValueToShadowMap &Map); + + void populateShadowStack(CallBase &CI, const TargetLibraryInfo &TLI, + const ValueToShadowMap &Map); + + void propagateShadowValues(Instruction &Inst, const TargetLibraryInfo &TLI, + const ValueToShadowMap &Map); + Value *emitCheck(Value *V, Value *ShadowV, IRBuilder<> &Builder, + CheckLoc Loc); + Value *emitCheckInternal(Value *V, Value *ShadowV, IRBuilder<> &Builder, + CheckLoc Loc); + void emitFCmpCheck(FCmpInst &FCmp, const ValueToShadowMap &Map); + Value *getCalleeAddress(CallBase &Call, IRBuilder<> &Builder) const; + + // Value creation handlers. + Value *handleLoad(LoadInst &Load, Type *VT, Type *ExtendedVT); + Value *handleTrunc(FPTruncInst &Trunc, Type *VT, Type *ExtendedVT, + const ValueToShadowMap &Map); + Value *handleExt(FPExtInst &Ext, Type *VT, Type *ExtendedVT, + const ValueToShadowMap &Map); + Value *handleCallBase(CallBase &Call, Type *VT, Type *ExtendedVT, + const TargetLibraryInfo &TLI, + const ValueToShadowMap &Map, IRBuilder<> &Builder); + Value *maybeHandleKnownCallBase(CallBase &Call, Type *VT, Type *ExtendedVT, + const TargetLibraryInfo &TLI, + const ValueToShadowMap &Map, + IRBuilder<> &Builder); + + // Value propagation handlers. + void propagateFTStore(StoreInst &Store, Type *VT, Type *ExtendedVT, + const ValueToShadowMap &Map); + void propagateNonFTStore(StoreInst &Store, Type *VT, + const ValueToShadowMap &Map); + + MappingConfig Config; + LLVMContext *Context = nullptr; + IntegerType *IntptrTy = nullptr; + FunctionCallee NsanGetShadowPtrForStore[FTValueType::kNumValueTypes]; + FunctionCallee NsanGetShadowPtrForLoad[FTValueType::kNumValueTypes]; + FunctionCallee NsanCheckValue[FTValueType::kNumValueTypes]; + FunctionCallee NsanFCmpFail[FTValueType::kNumValueTypes]; + FunctionCallee NsanCopyValues; + FunctionCallee NsanSetValueUnknown; + FunctionCallee NsanGetRawShadowTypePtr; + FunctionCallee NsanGetRawShadowPtr; + GlobalValue *NsanShadowRetTag; + + Type *NsanShadowRetType; + GlobalValue *NsanShadowRetPtr; + + GlobalValue *NsanShadowArgsTag; + + Type *NsanShadowArgsType; + GlobalValue *NsanShadowArgsPtr; + + std::optional<Regex> CheckFunctionsFilter; +}; + +void insertModuleCtor(Module &M) { + getOrCreateSanitizerCtorAndInitFunctions( + M, kNsanModuleCtorName, kNsanInitName, /*InitArgTypes=*/{}, + /*InitArgs=*/{}, + // This callback is invoked when the functions are created the first + // time. Hook them into the global ctors list in that case: + [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, Ctor, 0); }); +} + +} // end anonymous namespace + +PreservedAnalyses +NumericalStabilitySanitizerPass::run(Function &F, + FunctionAnalysisManager &FAM) { + NumericalStabilitySanitizer Nsan; + if (Nsan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F))) + return PreservedAnalyses::none(); + return PreservedAnalyses::all(); +} + +PreservedAnalyses +NumericalStabilitySanitizerPass::run(Module &M, ModuleAnalysisManager &MAM) { + insertModuleCtor(M); + return PreservedAnalyses::none(); +} + +static GlobalValue *createThreadLocalGV(const char *Name, Module &M, Type *Ty) { + return dyn_cast<GlobalValue>(M.getOrInsertGlobal(Name, Ty, [&M, Ty, Name] { + return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage, + nullptr, Name, nullptr, + GlobalVariable::InitialExecTLSModel); + })); +} + +void NumericalStabilitySanitizer::initialize(Module &M) { + const DataLayout &DL = M.getDataLayout(); + Context = &M.getContext(); + IntptrTy = DL.getIntPtrType(*Context); + Type *PtrTy = PointerType::getUnqual(*Context); + Type *Int32Ty = Type::getInt32Ty(*Context); + Type *Int1Ty = Type::getInt1Ty(*Context); + Type *VoidTy = Type::getVoidTy(*Context); + + AttributeList Attr; + Attr = Attr.addFnAttribute(*Context, Attribute::NoUnwind); + // Initialize the runtime values (functions and global variables). + for (int I = 0; I < kNumValueTypes; ++I) { + const FTValueType VT = static_cast<FTValueType>(I); + const char *const VTName = typeNameFromFTValueType(VT); + Type *const VTTy = typeFromFTValueType(VT, *Context); + + // Load/store. + const std::string GetterPrefix = + std::string("__nsan_get_shadow_ptr_for_") + VTName; + NsanGetShadowPtrForStore[VT] = M.getOrInsertFunction( + GetterPrefix + "_store", Attr, PtrTy, PtrTy, IntptrTy); + NsanGetShadowPtrForLoad[VT] = M.getOrInsertFunction( + GetterPrefix + "_load", Attr, PtrTy, PtrTy, IntptrTy); + + // Check. + const auto &ShadowConfig = Config.byValueType(VT); + Type *ShadowTy = ShadowConfig.getType(*Context); + NsanCheckValue[VT] = + M.getOrInsertFunction(std::string("__nsan_internal_check_") + VTName + + "_" + ShadowConfig.getNsanTypeId(), + Attr, Int32Ty, VTTy, ShadowTy, Int32Ty, IntptrTy); + NsanFCmpFail[VT] = M.getOrInsertFunction( + std::string("__nsan_fcmp_fail_") + VTName + "_" + + ShadowConfig.getNsanTypeId(), + Attr, VoidTy, VTTy, VTTy, ShadowTy, ShadowTy, Int32Ty, Int1Ty, Int1Ty); + } + + NsanCopyValues = M.getOrInsertFunction("__nsan_copy_values", Attr, VoidTy, + PtrTy, PtrTy, IntptrTy); + NsanSetValueUnknown = M.getOrInsertFunction("__nsan_set_value_unknown", Attr, + VoidTy, PtrTy, IntptrTy); + + // FIXME: Add attributes nofree, nosync, readnone, readonly, + NsanGetRawShadowTypePtr = M.getOrInsertFunction( + "__nsan_internal_get_raw_shadow_type_ptr", Attr, PtrTy, PtrTy); + NsanGetRawShadowPtr = M.getOrInsertFunction( + "__nsan_internal_get_raw_shadow_ptr", Attr, PtrTy, PtrTy); + + NsanShadowRetTag = createThreadLocalGV("__nsan_shadow_ret_tag", M, IntptrTy); + + NsanShadowRetType = ArrayType::get(Type::getInt8Ty(*Context), + kMaxVectorWidth * kMaxShadowTypeSizeBytes); + NsanShadowRetPtr = + createThreadLocalGV("__nsan_shadow_ret_ptr", M, NsanShadowRetType); + + NsanShadowArgsTag = + createThreadLocalGV("__nsan_shadow_args_tag", M, IntptrTy); + + NsanShadowArgsType = + ArrayType::get(Type::getInt8Ty(*Context), + kMaxVectorWidth * kMaxNumArgs * kMaxShadowTypeSizeBytes); + + NsanShadowArgsPtr = + createThreadLocalGV("__nsan_shadow_args_ptr", M, NsanShadowArgsType); + + if (!ClCheckFunctionsFilter.empty()) { + Regex R = Regex(ClCheckFunctionsFilter); + std::string RegexError; + assert(R.isValid(RegexError)); + CheckFunctionsFilter = std::move(R); + } +} + +// Returns true if the given LLVM Value points to constant data (typically, a +// global variable reference). +bool NumericalStabilitySanitizer::addrPointsToConstantData(Value *Addr) { + // If this is a GEP, just analyze its pointer operand. + if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr)) + Addr = GEP->getPointerOperand(); + + if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) { + return GV->isConstant(); + } + return false; +} + +// This instruments the function entry to create shadow arguments. +// Pseudocode: +// if (this_fn_ptr == __nsan_shadow_args_tag) { +// s(arg0) = LOAD<sizeof(arg0)>(__nsan_shadow_args); +// s(arg1) = LOAD<sizeof(arg1)>(__nsan_shadow_args + sizeof(arg0)); +// ... +// __nsan_shadow_args_tag = 0; +// } else { +// s(arg0) = fext(arg0); +// s(arg1) = fext(arg1); +// ... +// } +void NumericalStabilitySanitizer::createShadowArguments( + Function &F, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) { + assert(!F.getIntrinsicID() && "found a definition of an intrinsic"); + + // Do not bother if there are no FP args. + if (all_of(F.args(), [this](const Argument &Arg) { + return Config.getExtendedFPType(Arg.getType()) == nullptr; + })) + return; + + const DataLayout &DL = F.getParent()->getDataLayout(); + IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHI()); + // The function has shadow args if the shadow args tag matches the function + // address. + Value *HasShadowArgs = Builder.CreateICmpEQ( + Builder.CreateLoad(IntptrTy, NsanShadowArgsTag, /*isVolatile=*/false), + Builder.CreatePtrToInt(&F, IntptrTy)); + + unsigned ShadowArgsOffsetBytes = 0; + for (Argument &Arg : F.args()) { + Type *const VT = Arg.getType(); + Type *const ExtendedVT = Config.getExtendedFPType(VT); + if (ExtendedVT == nullptr) + continue; // Not an FT value. + Value *Shadow = Builder.CreateSelect( + HasShadowArgs, + Builder.CreateAlignedLoad( + ExtendedVT, + Builder.CreateConstGEP2_64(NsanShadowArgsType, NsanShadowArgsPtr, 0, + ShadowArgsOffsetBytes), + Align(1), /*isVolatile=*/false), + Builder.CreateCast(Instruction::FPExt, &Arg, ExtendedVT)); + Map.setShadow(&Arg, Shadow); + TypeSize SlotSize = DL.getTypeStoreSize(ExtendedVT); + assert(!SlotSize.isScalable() && "unsupported"); + ShadowArgsOffsetBytes += SlotSize.getFixedValue(); + } + Builder.CreateStore(ConstantInt::get(IntptrTy, 0), NsanShadowArgsTag); +} + +// Returns true if the instrumentation should emit code to check arguments +// before a function call. +static bool shouldCheckArgs(CallBase &CI, const TargetLibraryInfo &TLI, + const std::optional<Regex> &CheckFunctionsFilter) { + + Function *Fn = CI.getCalledFunction(); + + if (CheckFunctionsFilter) { + // Skip checking args of indirect calls. + if (Fn == nullptr) + return false; + if (CheckFunctionsFilter->match(Fn->getName())) + return true; + return false; + } + + if (Fn == nullptr) + return true; // Always check args of indirect calls. + + // Never check nsan functions, the user called them for a reason. + if (Fn->getName().starts_with("__nsan_")) + return false; + + const auto ID = Fn->getIntrinsicID(); + LibFunc LFunc = LibFunc::NumLibFuncs; + // Always check args of unknown functions. + if (ID == Intrinsic::ID() && !TLI.getLibFunc(*Fn, LFunc)) + return true; + + // Do not check args of an `fabs` call that is used for a comparison. + // This is typically used for `fabs(a-b) < tolerance`, where what matters is + // the result of the comparison, which is already caught be the fcmp checks. + if (ID == Intrinsic::fabs || LFunc == LibFunc_fabsf || + LFunc == LibFunc_fabs || LFunc == LibFunc_fabsl) + for (const auto &U : CI.users()) + if (isa<CmpInst>(U)) + return false; + + return true; // Default is check. +} + +// Populates the shadow call stack (which contains shadow values for every +// floating-point parameter to the function). +void NumericalStabilitySanitizer::populateShadowStack( + CallBase &CI, const TargetLibraryInfo &TLI, const ValueToShadowMap &Map) { + // Do not create a shadow stack for inline asm. + if (CI.isInlineAsm()) + return; + + // Do not bother if there are no FP args. + if (all_of(CI.operands(), [this](const Value *Arg) { + return Config.getExtendedFPType(Arg->getType()) == nullptr; + })) + return; + + IRBuilder<> Builder(&CI); + SmallVector<Value *, 8> ArgShadows; + const bool ShouldCheckArgs = shouldCheckArgs(CI, TLI, CheckFunctionsFilter); + int ArgId = -1; + for (Value *Arg : CI.operands()) { + ++ArgId; + if (Config.getExtendedFPType(Arg->getType()) == nullptr) + continue; // Not an FT value. + Value *ArgShadow = Map.getShadow(Arg); + ArgShadows.push_back(ShouldCheckArgs ? emitCheck(Arg, ArgShadow, Builder, + CheckLoc::makeArg(ArgId)) + : ArgShadow); + } + + // Do not create shadow stacks for intrinsics/known lib funcs. + if (Function *Fn = CI.getCalledFunction()) { + LibFunc LFunc; + if (Fn->getIntrinsicID() || TLI.getLibFunc(*Fn, LFunc)) ---------------- arsenm wrote:
check isIntrinsic or getIntrinsicID == Intrinsic::not_intrinsic https://github.com/llvm/llvm-project/pull/85916 _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
