yonghong-song updated this revision to Diff 223233.
yonghong-song edited the summary of this revision.
yonghong-song added a comment.
using 64bit buf always, handling non bitfield members, etc.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D67980/new/
https://reviews.llvm.org/D67980
Files:
clang/include/clang/Basic/BuiltinsBPF.def
clang/include/clang/Basic/DiagnosticSemaKinds.td
clang/include/clang/Basic/TargetBuiltins.h
clang/include/clang/Sema/Sema.h
clang/include/clang/module.modulemap
clang/lib/Basic/Targets/BPF.cpp
clang/lib/Basic/Targets/BPF.h
clang/lib/CodeGen/CGBuiltin.cpp
clang/lib/CodeGen/CGExpr.cpp
clang/lib/CodeGen/CodeGenFunction.h
clang/lib/Sema/SemaChecking.cpp
llvm/include/llvm/IR/IntrinsicsBPF.td
llvm/lib/Target/BPF/BPF.h
llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
llvm/lib/Target/BPF/BPFCORE.h
llvm/lib/Target/BPF/BPFTargetMachine.cpp
llvm/lib/Target/BPF/BTF.h
llvm/lib/Target/BPF/BTFDebug.cpp
llvm/lib/Target/BPF/BTFDebug.h
Index: llvm/lib/Target/BPF/BTFDebug.h
===================================================================
--- llvm/lib/Target/BPF/BTFDebug.h
+++ llvm/lib/Target/BPF/BTFDebug.h
@@ -224,10 +224,11 @@
};
/// Represent one offset relocation.
-struct BTFOffsetReloc {
+struct BTFFieldReloc {
const MCSymbol *Label; ///< MCSymbol identifying insn for the reloc
uint32_t TypeID; ///< Type ID
uint32_t OffsetNameOff; ///< The string to traverse types
+ uint32_t RelocKind; ///< What to patch the instruction
};
/// Represent one extern relocation.
@@ -249,12 +250,12 @@
std::unordered_map<const DIType *, uint32_t> DIToIdMap;
std::map<uint32_t, std::vector<BTFFuncInfo>> FuncInfoTable;
std::map<uint32_t, std::vector<BTFLineInfo>> LineInfoTable;
- std::map<uint32_t, std::vector<BTFOffsetReloc>> OffsetRelocTable;
+ std::map<uint32_t, std::vector<BTFFieldReloc>> FieldRelocTable;
std::map<uint32_t, std::vector<BTFExternReloc>> ExternRelocTable;
StringMap<std::vector<std::string>> FileContent;
std::map<std::string, std::unique_ptr<BTFKindDataSec>> DataSecEntries;
std::vector<BTFTypeStruct *> StructTypes;
- std::map<std::string, int64_t> AccessOffsets;
+ std::map<std::string, uint32_t> PatchImms;
std::map<StringRef, std::pair<bool, std::vector<BTFTypeDerived *>>>
FixupDerivedTypes;
@@ -300,7 +301,7 @@
void processGlobals(bool ProcessingMapDef);
/// Generate one offset relocation record.
- void generateOffsetReloc(const MachineInstr *MI, const MCSymbol *ORSym,
+ void generateFieldReloc(const MachineInstr *MI, const MCSymbol *ORSym,
DIType *RootTy, StringRef AccessPattern);
/// Populating unprocessed struct type.
Index: llvm/lib/Target/BPF/BTFDebug.cpp
===================================================================
--- llvm/lib/Target/BPF/BTFDebug.cpp
+++ llvm/lib/Target/BPF/BTFDebug.cpp
@@ -754,7 +754,7 @@
void BTFDebug::emitBTFExtSection() {
// Do not emit section if empty FuncInfoTable and LineInfoTable.
if (!FuncInfoTable.size() && !LineInfoTable.size() &&
- !OffsetRelocTable.size() && !ExternRelocTable.size())
+ !FieldRelocTable.size() && !ExternRelocTable.size())
return;
MCContext &Ctx = OS.getContext();
@@ -766,8 +766,8 @@
// Account for FuncInfo/LineInfo record size as well.
uint32_t FuncLen = 4, LineLen = 4;
- // Do not account for optional OffsetReloc/ExternReloc.
- uint32_t OffsetRelocLen = 0, ExternRelocLen = 0;
+ // Do not account for optional FieldReloc/ExternReloc.
+ uint32_t FieldRelocLen = 0, ExternRelocLen = 0;
for (const auto &FuncSec : FuncInfoTable) {
FuncLen += BTF::SecFuncInfoSize;
FuncLen += FuncSec.second.size() * BTF::BPFFuncInfoSize;
@@ -776,17 +776,17 @@
LineLen += BTF::SecLineInfoSize;
LineLen += LineSec.second.size() * BTF::BPFLineInfoSize;
}
- for (const auto &OffsetRelocSec : OffsetRelocTable) {
- OffsetRelocLen += BTF::SecOffsetRelocSize;
- OffsetRelocLen += OffsetRelocSec.second.size() * BTF::BPFOffsetRelocSize;
+ for (const auto &FieldRelocSec : FieldRelocTable) {
+ FieldRelocLen += BTF::SecFieldRelocSize;
+ FieldRelocLen += FieldRelocSec.second.size() * BTF::BPFFieldRelocSize;
}
for (const auto &ExternRelocSec : ExternRelocTable) {
ExternRelocLen += BTF::SecExternRelocSize;
ExternRelocLen += ExternRelocSec.second.size() * BTF::BPFExternRelocSize;
}
- if (OffsetRelocLen)
- OffsetRelocLen += 4;
+ if (FieldRelocLen)
+ FieldRelocLen += 4;
if (ExternRelocLen)
ExternRelocLen += 4;
@@ -795,8 +795,8 @@
OS.EmitIntValue(FuncLen, 4);
OS.EmitIntValue(LineLen, 4);
OS.EmitIntValue(FuncLen + LineLen, 4);
- OS.EmitIntValue(OffsetRelocLen, 4);
- OS.EmitIntValue(FuncLen + LineLen + OffsetRelocLen, 4);
+ OS.EmitIntValue(FieldRelocLen, 4);
+ OS.EmitIntValue(FuncLen + LineLen + FieldRelocLen, 4);
OS.EmitIntValue(ExternRelocLen, 4);
// Emit func_info table.
@@ -832,18 +832,19 @@
}
// Emit offset reloc table.
- if (OffsetRelocLen) {
- OS.AddComment("OffsetReloc");
- OS.EmitIntValue(BTF::BPFOffsetRelocSize, 4);
- for (const auto &OffsetRelocSec : OffsetRelocTable) {
+ if (FieldRelocLen) {
+ OS.AddComment("FieldReloc");
+ OS.EmitIntValue(BTF::BPFFieldRelocSize, 4);
+ for (const auto &FieldRelocSec : FieldRelocTable) {
OS.AddComment("Offset reloc section string offset=" +
- std::to_string(OffsetRelocSec.first));
- OS.EmitIntValue(OffsetRelocSec.first, 4);
- OS.EmitIntValue(OffsetRelocSec.second.size(), 4);
- for (const auto &OffsetRelocInfo : OffsetRelocSec.second) {
- Asm->EmitLabelReference(OffsetRelocInfo.Label, 4);
- OS.EmitIntValue(OffsetRelocInfo.TypeID, 4);
- OS.EmitIntValue(OffsetRelocInfo.OffsetNameOff, 4);
+ std::to_string(FieldRelocSec.first));
+ OS.EmitIntValue(FieldRelocSec.first, 4);
+ OS.EmitIntValue(FieldRelocSec.second.size(), 4);
+ for (const auto &FieldRelocInfo : FieldRelocSec.second) {
+ Asm->EmitLabelReference(FieldRelocInfo.Label, 4);
+ OS.EmitIntValue(FieldRelocInfo.TypeID, 4);
+ OS.EmitIntValue(FieldRelocInfo.OffsetNameOff, 4);
+ OS.EmitIntValue(FieldRelocInfo.RelocKind, 4);
}
}
}
@@ -959,22 +960,26 @@
}
/// Generate a struct member offset relocation.
-void BTFDebug::generateOffsetReloc(const MachineInstr *MI,
+void BTFDebug::generateFieldReloc(const MachineInstr *MI,
const MCSymbol *ORSym, DIType *RootTy,
StringRef AccessPattern) {
unsigned RootId = populateStructType(RootTy);
size_t FirstDollar = AccessPattern.find_first_of('$');
size_t FirstColon = AccessPattern.find_first_of(':');
+ size_t SecondColon = AccessPattern.find_first_of(':', FirstColon + 1);
StringRef IndexPattern = AccessPattern.substr(FirstDollar + 1);
- StringRef OffsetStr = AccessPattern.substr(FirstColon + 1,
- FirstDollar - FirstColon);
-
- BTFOffsetReloc OffsetReloc;
- OffsetReloc.Label = ORSym;
- OffsetReloc.OffsetNameOff = addString(IndexPattern);
- OffsetReloc.TypeID = RootId;
- AccessOffsets[AccessPattern.str()] = std::stoi(OffsetStr);
- OffsetRelocTable[SecNameOff].push_back(OffsetReloc);
+ StringRef RelocKindStr = AccessPattern.substr(FirstColon + 1,
+ SecondColon - FirstColon);
+ StringRef PatchImmStr = AccessPattern.substr(SecondColon + 1,
+ FirstDollar - SecondColon);
+
+ BTFFieldReloc FieldReloc;
+ FieldReloc.Label = ORSym;
+ FieldReloc.OffsetNameOff = addString(IndexPattern);
+ FieldReloc.TypeID = RootId;
+ FieldReloc.RelocKind = std::stoull(RelocKindStr);
+ PatchImms[AccessPattern.str()] = std::stoul(PatchImmStr);
+ FieldRelocTable[SecNameOff].push_back(FieldReloc);
}
void BTFDebug::processLDimm64(const MachineInstr *MI) {
@@ -982,7 +987,7 @@
// will generate an .BTF.ext record.
//
// If the insn is "r2 = LD_imm64 @__BTF_...",
- // add this insn into the .BTF.ext OffsetReloc subsection.
+ // add this insn into the .BTF.ext FieldReloc subsection.
// Relocation looks like:
// . SecName:
// . InstOffset
@@ -1013,7 +1018,7 @@
MDNode *MDN = GVar->getMetadata(LLVMContext::MD_preserve_access_index);
DIType *Ty = dyn_cast<DIType>(MDN);
- generateOffsetReloc(MI, ORSym, Ty, GVar->getName());
+ generateFieldReloc(MI, ORSym, Ty, GVar->getName());
} else if (GVar && !GVar->hasInitializer() && GVar->hasExternalLinkage() &&
GVar->getSection() == BPFCoreSharedInfo::PatchableExtSecName) {
MCSymbol *ORSym = OS.getContext().createTempSymbol();
@@ -1154,8 +1159,8 @@
const GlobalValue *GVal = MO.getGlobal();
auto *GVar = dyn_cast<GlobalVariable>(GVal);
if (GVar && GVar->hasAttribute(BPFCoreSharedInfo::AmaAttr)) {
- // Emit "mov ri, <imm>" for abstract member accesses.
- int64_t Imm = AccessOffsets[GVar->getName().str()];
+ // Emit "mov ri, <imm>" for patched immediate.
+ uint32_t Imm = PatchImms[GVar->getName().str()];
OutMI.setOpcode(BPF::MOV_ri);
OutMI.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
OutMI.addOperand(MCOperand::createImm(Imm));
Index: llvm/lib/Target/BPF/BTF.h
===================================================================
--- llvm/lib/Target/BPF/BTF.h
+++ llvm/lib/Target/BPF/BTF.h
@@ -17,7 +17,7 @@
///
/// The binary layout for .BTF.ext section:
/// struct ExtHeader
-/// FuncInfo, LineInfo, OffsetReloc and ExternReloc subsections
+/// FuncInfo, LineInfo, FieldReloc and ExternReloc subsections
/// The FuncInfo subsection is defined as below:
/// BTFFuncInfo Size
/// struct SecFuncInfo for ELF section #1
@@ -32,12 +32,12 @@
/// struct SecLineInfo for ELF section #2
/// A number of struct BPFLineInfo for ELF section #2
/// ...
-/// The OffsetReloc subsection is defined as below:
-/// BPFOffsetReloc Size
-/// struct SecOffsetReloc for ELF section #1
-/// A number of struct BPFOffsetReloc for ELF section #1
-/// struct SecOffsetReloc for ELF section #2
-/// A number of struct BPFOffsetReloc for ELF section #2
+/// The FieldReloc subsection is defined as below:
+/// BPFFieldReloc Size
+/// struct SecFieldReloc for ELF section #1
+/// A number of struct BPFFieldReloc for ELF section #1
+/// struct SecFieldReloc for ELF section #2
+/// A number of struct BPFFieldReloc for ELF section #2
/// ...
/// The ExternReloc subsection is defined as below:
/// BPFExternReloc Size
@@ -72,11 +72,11 @@
BTFDataSecVarSize = 12,
SecFuncInfoSize = 8,
SecLineInfoSize = 8,
- SecOffsetRelocSize = 8,
+ SecFieldRelocSize = 8,
SecExternRelocSize = 8,
BPFFuncInfoSize = 8,
BPFLineInfoSize = 16,
- BPFOffsetRelocSize = 12,
+ BPFFieldRelocSize = 16,
BPFExternRelocSize = 8,
};
@@ -213,8 +213,8 @@
uint32_t FuncInfoLen; ///< Length of func info section
uint32_t LineInfoOff; ///< Offset of line info section
uint32_t LineInfoLen; ///< Length of line info section
- uint32_t OffsetRelocOff; ///< Offset of offset reloc section
- uint32_t OffsetRelocLen; ///< Length of offset reloc section
+ uint32_t FieldRelocOff; ///< Offset of offset reloc section
+ uint32_t FieldRelocLen; ///< Length of offset reloc section
uint32_t ExternRelocOff; ///< Offset of extern reloc section
uint32_t ExternRelocLen; ///< Length of extern reloc section
};
@@ -247,16 +247,17 @@
};
/// Specifying one offset relocation.
-struct BPFOffsetReloc {
+struct BPFFieldReloc {
uint32_t InsnOffset; ///< Byte offset in this section
uint32_t TypeID; ///< TypeID for the relocation
uint32_t OffsetNameOff; ///< The string to traverse types
+ uint32_t RelocKind; ///< What to patch the instruction
};
/// Specifying offset relocation's in one section.
-struct SecOffsetReloc {
+struct SecFieldReloc {
uint32_t SecNameOff; ///< Section name index in the .BTF string table
- uint32_t NumOffsetReloc; ///< Number of offset reloc's in this section
+ uint32_t NumFieldReloc; ///< Number of offset reloc's in this section
};
/// Specifying one offset relocation.
Index: llvm/lib/Target/BPF/BPFTargetMachine.cpp
===================================================================
--- llvm/lib/Target/BPF/BPFTargetMachine.cpp
+++ llvm/lib/Target/BPF/BPFTargetMachine.cpp
@@ -94,7 +94,7 @@
void BPFPassConfig::addIRPasses() {
- addPass(createBPFAbstractMemberAccess());
+ addPass(createBPFAbstractMemberAccess(&getBPFTargetMachine()));
TargetPassConfig::addIRPasses();
}
Index: llvm/lib/Target/BPF/BPFCORE.h
===================================================================
--- llvm/lib/Target/BPF/BPFCORE.h
+++ llvm/lib/Target/BPF/BPFCORE.h
@@ -13,6 +13,17 @@
class BPFCoreSharedInfo {
public:
+ enum OffsetRelocKind : uint32_t {
+ FIELD_ACCESS_OFFSET = 0,
+ FIELD_EXISTENCE,
+ FIELD_SIGNEDNESS,
+ FIELD_BYTE_SIZE,
+ FIELD_BYTE_OFFSET,
+ FIELD_LSHIFT_64BIT_BUF,
+ FIELD_RSHIFT_AFTER_LSHIFT_64BIT_BUF,
+
+ MAX_FIELD_RELOC_KIND,
+ };
/// The attribute attached to globals representing a member offset
static const std::string AmaAttr;
/// The section name to identify a patchable external global
Index: llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
===================================================================
--- llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
+++ llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
@@ -50,6 +50,28 @@
// addr = preserve_struct_access_index(base, gep_index, di_index)
// !llvm.preserve.access.index <struct_ditype>
//
+// Bitfield member access needs special attention. User cannot take the
+// address of a bitfield acceess. To facilitate kernel verifier
+// for easy bitfield code optimization, a new clang intrinsic is introduced:
+// uint32_t __builtin_preserve_field_info(member_access, info_kind)
+// In IR, a chain with two (or more) intrinsic calls will be generated:
+// ...
+// addr = preserve_struct_access_index(base, 1, 1) !struct s
+// uint32_t result = bpf_preserve_field_info(addr, info_kind)
+//
+// Suppose the info_kind is FIELD_SIGNEDNESS,
+// The above two IR intrinsics will be replaced with
+// a relocatable insn:
+// signness = /* signness of member_access */
+// and signness can be changed by bpf loader based on the
+// types on the host.
+//
+// User can also test whether a field exists or not with
+// uint32_t result = bpf_preserve_field_info(member_access, FIELD_EXISTENCE)
+// The field will be always available (result = 1) during initial
+// compilation, but bpf loader can patch with the correct value
+// on the target host where the member_access may or may not be available
+//
//===----------------------------------------------------------------------===//
#include "BPF.h"
@@ -88,7 +110,11 @@
public:
static char ID;
- BPFAbstractMemberAccess() : ModulePass(ID) {}
+ TargetMachine *TM;
+ // Add optional BPFTargetMachine parameter so that BPF backend can add the phase
+ // with target machine to find out the endianness. The default constructor (without
+ // parameters) is used by the pass manager for managing purposes.
+ BPFAbstractMemberAccess(BPFTargetMachine *TM = nullptr) : ModulePass(ID), TM(TM) {}
struct CallInfo {
uint32_t Kind;
@@ -102,13 +128,14 @@
BPFPreserveArrayAI = 1,
BPFPreserveUnionAI = 2,
BPFPreserveStructAI = 3,
+ BPFPreserveFieldInfoAI = 4,
};
std::map<std::string, GlobalVariable *> GEPGlobals;
// A map to link preserve_*_access_index instrinsic calls.
std::map<CallInst *, std::pair<CallInst *, CallInfo>> AIChain;
// A map to hold all the base preserve_*_access_index instrinsic calls.
- // The base call is not an input of any other preserve_*_access_index
+ // The base call is not an input of any other preserve_*
// intrinsics.
std::map<CallInst *, CallInfo> BaseAICalls;
@@ -127,6 +154,8 @@
bool removePreserveAccessIndexIntrinsic(Module &M);
void replaceWithGEP(std::vector<CallInst *> &CallList,
uint32_t NumOfZerosIndex, uint32_t DIIndex);
+ uint32_t GetFieldInfo(uint32_t InfoKind, DICompositeType *CTy,
+ uint32_t AccessIndex);
Value *computeBaseAndAccessKey(CallInst *Call, CallInfo &CInfo,
std::string &AccessKey, MDNode *&BaseMeta);
@@ -139,8 +168,8 @@
INITIALIZE_PASS(BPFAbstractMemberAccess, DEBUG_TYPE,
"abstracting struct/union member accessees", false, false)
-ModulePass *llvm::createBPFAbstractMemberAccess() {
- return new BPFAbstractMemberAccess();
+ModulePass *llvm::createBPFAbstractMemberAccess(BPFTargetMachine *TM) {
+ return new BPFAbstractMemberAccess(TM);
}
bool BPFAbstractMemberAccess::runOnModule(Module &M) {
@@ -231,6 +260,17 @@
CInfo.Base = Call->getArgOperand(0);
return true;
}
+ if (GV->getName().startswith("llvm.bpf.preserve.field.info")) {
+ CInfo.Kind = BPFPreserveFieldInfoAI;
+ CInfo.Metadata = nullptr;
+ // Check validity of info_kind as clang did not check this.
+ uint64_t InfoKind = getConstant(Call->getArgOperand(1));
+ if (InfoKind == BPFCoreSharedInfo::FIELD_ACCESS_OFFSET ||
+ InfoKind >= BPFCoreSharedInfo::MAX_FIELD_RELOC_KIND)
+ report_fatal_error("Incorrect info_kind for llvm.bpf.preserve.field.info intrinsic");
+ CInfo.AccessIndex = InfoKind;
+ return true;
+ }
return false;
}
@@ -306,6 +346,9 @@
bool BPFAbstractMemberAccess::IsValidAIChain(const MDNode *ParentType,
uint32_t ParentAI,
const MDNode *ChildType) {
+ if (!ChildType)
+ return true; // preserve_field_info, no type comparison needed.
+
const DIType *PType = stripQualifiers(cast<DIType>(ParentType));
const DIType *CType = stripQualifiers(cast<DIType>(ChildType));
@@ -463,7 +506,97 @@
return CV->getValue().getZExtValue();
}
-/// Compute the base of the whole preserve_*_access_index chains, i.e., the base
+uint32_t BPFAbstractMemberAccess::GetFieldInfo(uint32_t InfoKind,
+ DICompositeType *CTy,
+ uint32_t AccessIndex) {
+ if (InfoKind == BPFCoreSharedInfo::FIELD_EXISTENCE)
+ return 1;
+
+ uint32_t Tag = CTy->getTag();
+ assert(Tag == dwarf::DW_TAG_structure_type || Tag == dwarf::DW_TAG_union_type);
+ auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]);
+
+ if (InfoKind == BPFCoreSharedInfo::FIELD_SIGNEDNESS) {
+ const DIType *BaseTy = stripQualifiers(MemberTy->getBaseType());
+ // Only basic types and enum types have signedness.
+ const auto *BTy = dyn_cast<DIBasicType>(BaseTy);
+ while (!BTy) {
+ const auto *CompTy = dyn_cast<DICompositeType>(BaseTy);
+ // Report an error if the member expression does not have signedness.
+ if (!CompTy || CompTy->getTag() != dwarf::DW_TAG_enumeration_type)
+ report_fatal_error("Invalid member expression for llvm.bpf.preserve.field.info");
+ BaseTy = stripQualifiers(CompTy->getBaseType());
+ BTy = dyn_cast<DIBasicType>(BaseTy);
+ }
+ uint32_t Encoding = BTy->getEncoding();
+ return (Encoding == dwarf::DW_ATE_signed || Encoding == dwarf::DW_ATE_signed_char);
+ }
+
+ if (InfoKind == BPFCoreSharedInfo::FIELD_BYTE_SIZE) {
+ uint32_t SizeInBits = MemberTy->getSizeInBits();
+ if (!MemberTy->isBitField())
+ return SizeInBits >> 3;
+
+ uint32_t OffsetInBits = MemberTy->getOffsetInBits();
+ uint32_t Start = OffsetInBits >> 3;
+ uint32_t End = (OffsetInBits + SizeInBits - 1) >> 3;
+ return End - Start + 1;
+ }
+
+ if (InfoKind == BPFCoreSharedInfo::FIELD_BYTE_OFFSET)
+ return MemberTy->getOffsetInBits() >> 3;
+
+ if (InfoKind == BPFCoreSharedInfo::FIELD_LSHIFT_64BIT_BUF) {
+ uint32_t SizeInBits = MemberTy->getSizeInBits();
+ const Triple &Triple = TM->getTargetTriple();
+ if (!MemberTy->isBitField()) {
+ if (SizeInBits > 64)
+ report_fatal_error("too big member size for llvm.bpf.preserve.field.info");
+
+ /* endianness */
+ if (Triple.getArch() == Triple::bpfeb)
+ return 0;
+ else
+ return 64 - SizeInBits;
+ }
+
+ uint32_t OffsetInBits = MemberTy->getOffsetInBits();
+ uint32_t Start = OffsetInBits >> 3;
+ uint32_t End = (OffsetInBits + SizeInBits - 1) >> 3;
+ uint32_t NumOfBytes = End - Start + 1;
+ if (NumOfBytes > 8)
+ report_fatal_error("too big member size for llvm.bpf.preserve.field.info");
+
+ /* endianness */
+ if (Triple.getArch() == Triple::bpfeb)
+ return OffsetInBits - (Start << 3);
+ else
+ return ((Start + 8) << 3) - OffsetInBits - SizeInBits;
+ }
+
+ if (InfoKind == BPFCoreSharedInfo::FIELD_RSHIFT_AFTER_LSHIFT_64BIT_BUF) {
+ uint32_t SizeInBits = MemberTy->getSizeInBits();
+ if (!MemberTy->isBitField()) {
+ if (SizeInBits > 64)
+ report_fatal_error("too big member size for llvm.bpf.preserve.field.info");
+
+ return 64 - SizeInBits;
+ }
+
+ uint32_t OffsetInBits = MemberTy->getOffsetInBits();
+ uint32_t Start = OffsetInBits >> 3;
+ uint32_t End = (OffsetInBits + SizeInBits - 1) >> 3;
+ uint32_t NumOfBytes = End - Start + 1;
+ if (NumOfBytes > 8)
+ report_fatal_error("too big member size for llvm.bpf.preserve.field.info");
+
+ return 64 - SizeInBits;
+ }
+
+ llvm_unreachable("Unknown llvm.bpf.preserve.field.info info kind");
+}
+
+/// Compute the base of the whole preserve_* intrinsics chains, i.e., the base
/// pointer of the first preserve_*_access_index call, and construct the access
/// string, which will be the name of a global variable.
Value *BPFAbstractMemberAccess::computeBaseAndAccessKey(CallInst *Call,
@@ -492,7 +625,8 @@
// int a[10][20]; ... __builtin_preserve_access_index(&a[2][3]) ...
// we will skip them.
uint32_t FirstIndex = 0;
- uint32_t AccessOffset = 0;
+ uint32_t PatchImm = 0; // AccessOffset or the requested field info
+ uint32_t InfoKind = BPFCoreSharedInfo::FIELD_ACCESS_OFFSET;
while (CallStack.size()) {
auto StackElem = CallStack.top();
Call = StackElem.first;
@@ -507,10 +641,13 @@
// struct or union type
TypeName = Ty->getName();
TypeMeta = Ty;
- AccessOffset += FirstIndex * Ty->getSizeInBits() >> 3;
+ PatchImm += FirstIndex * (Ty->getSizeInBits() >> 3);
break;
}
+ assert(CInfo.Kind == BPFPreserveFieldInfoAI &&
+ "Reaching preserve_field_info and cannot find top type name");
+
// Array entries will always be consumed for accumulative initial index.
CallStack.pop();
@@ -555,7 +692,7 @@
else
TypeName = CTy->getName();
TypeMeta = CTy;
- AccessOffset += FirstIndex * CTy->getSizeInBits() >> 3;
+ PatchImm += FirstIndex * (CTy->getSizeInBits() >> 3);
break;
}
}
@@ -569,6 +706,20 @@
CInfo = StackElem.second;
CallStack.pop();
+ if (CInfo.Kind == BPFPreserveFieldInfoAI)
+ break;
+
+ // If the next Call (the top of the stack) is a BPFPreserveFieldInfoAI,
+ // the action will be extracting field info.
+ if (CallStack.size()) {
+ auto StackElem2 = CallStack.top();
+ CallInfo CInfo2 = StackElem2.second;
+ if (CInfo2.Kind == BPFPreserveFieldInfoAI) {
+ InfoKind = CInfo2.AccessIndex;
+ assert(CallStack.size() == 1);
+ }
+ }
+
// Access Index
uint64_t AccessIndex = CInfo.AccessIndex;
AccessKey += ":" + std::to_string(AccessIndex);
@@ -577,19 +728,24 @@
// At this stage, it cannot be pointer type.
auto *CTy = cast<DICompositeType>(stripQualifiers(cast<DIType>(MDN)));
uint32_t Tag = CTy->getTag();
- if (Tag == dwarf::DW_TAG_structure_type) {
- auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]);
- AccessOffset += MemberTy->getOffsetInBits() >> 3;
- } else if (Tag == dwarf::DW_TAG_array_type) {
- auto *EltTy = stripQualifiers(CTy->getBaseType());
- AccessOffset += AccessIndex * calcArraySize(CTy, 1) *
- EltTy->getSizeInBits() >> 3;
+ if (InfoKind == BPFCoreSharedInfo::FIELD_ACCESS_OFFSET) {
+ if (Tag == dwarf::DW_TAG_structure_type) {
+ auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]);
+ PatchImm += MemberTy->getOffsetInBits() >> 3;
+ } else if (Tag == dwarf::DW_TAG_array_type) {
+ auto *EltTy = stripQualifiers(CTy->getBaseType());
+ PatchImm += AccessIndex * calcArraySize(CTy, 1) *
+ (EltTy->getSizeInBits() >> 3);
+ }
+ } else {
+ PatchImm = GetFieldInfo(InfoKind, CTy, AccessIndex);
}
}
- // Access key is the type name + access string, uniquely identifying
- // one kernel memory access.
- AccessKey = TypeName + ":" + std::to_string(AccessOffset) + "$" + AccessKey;
+ // Access key is the type name + reloc type + patched imm + access string,
+ // uniquely identifying one relocation.
+ AccessKey = TypeName + ":" + std::to_string(InfoKind) + ":" +
+ std::to_string(PatchImm) + "$" + AccessKey;
return Base;
}
@@ -605,21 +761,11 @@
if (!Base)
return false;
- // Do the transformation
- // For any original GEP Call and Base %2 like
- // %4 = bitcast %struct.net_device** %dev1 to i64*
- // it is transformed to:
- // %6 = load sk_buff:50:$0:0:0:2:0
- // %7 = bitcast %struct.sk_buff* %2 to i8*
- // %8 = getelementptr i8, i8* %7, %6
- // %9 = bitcast i8* %8 to i64*
- // using %9 instead of %4
- // The original Call inst is removed.
BasicBlock *BB = Call->getParent();
GlobalVariable *GV;
if (GEPGlobals.find(AccessKey) == GEPGlobals.end()) {
- GV = new GlobalVariable(M, Type::getInt64Ty(BB->getContext()), false,
+ GV = new GlobalVariable(M, Type::getInt32Ty(BB->getContext()), false,
GlobalVariable::ExternalLinkage, NULL, AccessKey);
GV->addAttribute(BPFCoreSharedInfo::AmaAttr);
GV->setMetadata(LLVMContext::MD_preserve_access_index, TypeMeta);
@@ -628,8 +774,27 @@
GV = GEPGlobals[AccessKey];
}
+ if (CInfo.Kind == BPFPreserveFieldInfoAI) {
+ // Load the global variable which represents the returned field info.
+ auto *LDInst = new LoadInst(Type::getInt32Ty(BB->getContext()), GV);
+ BB->getInstList().insert(Call->getIterator(), LDInst);
+ Call->replaceAllUsesWith(LDInst);
+ Call->eraseFromParent();
+ return true;
+ }
+
+ // For any original GEP Call and Base %2 like
+ // %4 = bitcast %struct.net_device** %dev1 to i64*
+ // it is transformed to:
+ // %6 = load sk_buff:50:$0:0:0:2:0
+ // %7 = bitcast %struct.sk_buff* %2 to i8*
+ // %8 = getelementptr i8, i8* %7, %6
+ // %9 = bitcast i8* %8 to i64*
+ // using %9 instead of %4
+ // The original Call inst is removed.
+
// Load the global variable.
- auto *LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV);
+ auto *LDInst = new LoadInst(Type::getInt32Ty(BB->getContext()), GV);
BB->getInstList().insert(Call->getIterator(), LDInst);
// Generate a BitCast
Index: llvm/lib/Target/BPF/BPF.h
===================================================================
--- llvm/lib/Target/BPF/BPF.h
+++ llvm/lib/Target/BPF/BPF.h
@@ -15,7 +15,7 @@
namespace llvm {
class BPFTargetMachine;
-ModulePass *createBPFAbstractMemberAccess();
+ModulePass *createBPFAbstractMemberAccess(BPFTargetMachine *TM);
FunctionPass *createBPFISelDag(BPFTargetMachine &TM);
FunctionPass *createBPFMISimplifyPatchablePass();
Index: llvm/include/llvm/IR/IntrinsicsBPF.td
===================================================================
--- llvm/include/llvm/IR/IntrinsicsBPF.td
+++ llvm/include/llvm/IR/IntrinsicsBPF.td
@@ -20,4 +20,7 @@
Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>;
def int_bpf_pseudo : GCCBuiltin<"__builtin_bpf_pseudo">,
Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty]>;
+ def int_bpf_preserve_field_info : GCCBuiltin<"__builtin_bpf_preserve_field_info">,
+ Intrinsic<[llvm_i32_ty], [llvm_anyptr_ty, llvm_i64_ty],
+ [IntrNoMem, ImmArg<1>]>;
}
Index: clang/lib/Sema/SemaChecking.cpp
===================================================================
--- clang/lib/Sema/SemaChecking.cpp
+++ clang/lib/Sema/SemaChecking.cpp
@@ -1540,6 +1540,11 @@
if (CheckAArch64BuiltinFunctionCall(BuiltinID, TheCall))
return ExprError();
break;
+ case llvm::Triple::bpfeb:
+ case llvm::Triple::bpfel:
+ if (CheckBPFBuiltinFunctionCall(BuiltinID, TheCall))
+ return ExprError();
+ break;
case llvm::Triple::hexagon:
if (CheckHexagonBuiltinFunctionCall(BuiltinID, TheCall))
return ExprError();
@@ -1940,6 +1945,36 @@
return SemaBuiltinConstantArgRange(TheCall, i, l, u + l);
}
+bool Sema::CheckBPFBuiltinFunctionCall(unsigned BuiltinID,
+ CallExpr *TheCall) {
+ assert(BuiltinID == BPF::BI__builtin_preserve_field_info &&
+ "unexpected ARM builtin");
+
+ if (checkArgCount(*this, TheCall, 2))
+ return true;
+
+ // The first argument needs to be a record field access
+ Expr *Arg = TheCall->getArg(0);
+ if (Arg->getType()->getAsPlaceholderType() ||
+ (Arg->IgnoreParens()->getObjectKind() != OK_BitField &&
+ !dyn_cast<MemberExpr>(Arg->IgnoreParens()))) {
+ Diag(Arg->getBeginLoc(), diag::err_preserve_field_info_not_field)
+ << 1 << Arg->getSourceRange();
+ return true;
+ }
+
+ // The second argument needs to be a constant int
+ llvm::APSInt Value;
+ if (!TheCall->getArg(1)->isIntegerConstantExpr(Value, Context)) {
+ Diag(Arg->getBeginLoc(), diag::err_preserve_field_info_not_const)
+ << 2 << Arg->getSourceRange();
+ return true;
+ }
+
+ TheCall->setType(Context.UnsignedIntTy);
+ return false;
+}
+
bool Sema::CheckHexagonBuiltinCpu(unsigned BuiltinID, CallExpr *TheCall) {
struct BuiltinAndString {
unsigned BuiltinID;
Index: clang/lib/CodeGen/CodeGenFunction.h
===================================================================
--- clang/lib/CodeGen/CodeGenFunction.h
+++ clang/lib/CodeGen/CodeGenFunction.h
@@ -3760,6 +3760,7 @@
llvm::Value *vectorWrapScalar16(llvm::Value *Op);
llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E,
llvm::Triple::ArchType Arch);
+ llvm::Value *EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
llvm::Value *BuildVector(ArrayRef<llvm::Value*> Ops);
llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E);
Index: clang/lib/CodeGen/CGExpr.cpp
===================================================================
--- clang/lib/CodeGen/CGExpr.cpp
+++ clang/lib/CodeGen/CGExpr.cpp
@@ -3990,9 +3990,19 @@
const CGBitFieldInfo &Info = RL.getBitFieldInfo(field);
Address Addr = base.getAddress();
unsigned Idx = RL.getLLVMFieldNo(field);
- if (Idx != 0)
- // For structs, we GEP to the field that the record layout suggests.
- Addr = Builder.CreateStructGEP(Addr, Idx, field->getName());
+ if (!IsInPreservedAIRegion) {
+ if (Idx != 0)
+ // For structs, we GEP to the field that the record layout suggests.
+ Addr = Builder.CreateStructGEP(Addr, Idx, field->getName());
+ } else {
+ const RecordDecl *rec = field->getParent();
+ llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType(
+ getContext().getRecordType(rec), rec->getLocation());
+ Addr = Builder.CreatePreserveStructAccessIndex(Addr, Idx,
+ getDebugInfoFIndex(rec, field->getFieldIndex()),
+ DbgInfo);
+ }
+
// Get the access type.
llvm::Type *FieldIntTy =
llvm::Type::getIntNTy(getLLVMContext(), Info.StorageSize);
Index: clang/lib/CodeGen/CGBuiltin.cpp
===================================================================
--- clang/lib/CodeGen/CGBuiltin.cpp
+++ clang/lib/CodeGen/CGBuiltin.cpp
@@ -4242,6 +4242,9 @@
case llvm::Triple::aarch64:
case llvm::Triple::aarch64_be:
return CGF->EmitAArch64BuiltinExpr(BuiltinID, E, Arch);
+ case llvm::Triple::bpfeb:
+ case llvm::Triple::bpfel:
+ return CGF->EmitBPFBuiltinExpr(BuiltinID, E);
case llvm::Triple::x86:
case llvm::Triple::x86_64:
return CGF->EmitX86BuiltinExpr(BuiltinID, E);
@@ -9300,6 +9303,37 @@
}
}
+Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID,
+ const CallExpr *E) {
+ assert(BuiltinID == BPF::BI__builtin_preserve_field_info &&
+ "unexpected ARM builtin");
+
+ const Expr *Arg = E->getArg(0);
+ bool IsBitField = Arg->IgnoreParens()->getObjectKind() == OK_BitField;
+
+ if (!getDebugInfo()) {
+ CGM.Error(E->getExprLoc(), "using builtin_preserve_field_info() without -g");
+ return IsBitField ? EmitLValue(Arg).getBitFieldPointer()
+ : EmitLValue(Arg).getPointer();
+ }
+
+ // Enable underlying preserve_*_access_index() generation.
+ bool OldIsInPreservedAIRegion = IsInPreservedAIRegion;
+ IsInPreservedAIRegion = true;
+ Value *FieldAddr = IsBitField ? EmitLValue(Arg).getBitFieldPointer()
+ : EmitLValue(Arg).getPointer();
+ IsInPreservedAIRegion = OldIsInPreservedAIRegion;
+
+ ConstantInt *C = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
+ Value *InfoKind = ConstantInt::get(Int64Ty, C->getSExtValue());
+
+ // Built the IR for the preserve_field_info intrinsic.
+ llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getDeclaration(
+ &CGM.getModule(), llvm::Intrinsic::bpf_preserve_field_info,
+ {FieldAddr->getType()});
+ return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
+}
+
llvm::Value *CodeGenFunction::
BuildVector(ArrayRef<llvm::Value*> Ops) {
assert((Ops.size() & (Ops.size() - 1)) == 0 &&
Index: clang/lib/Basic/Targets/BPF.h
===================================================================
--- clang/lib/Basic/Targets/BPF.h
+++ clang/lib/Basic/Targets/BPF.h
@@ -22,6 +22,8 @@
namespace targets {
class LLVM_LIBRARY_VISIBILITY BPFTargetInfo : public TargetInfo {
+ static const Builtin::Info BuiltinInfo[];
+
public:
BPFTargetInfo(const llvm::Triple &Triple, const TargetOptions &)
: TargetInfo(Triple) {
@@ -54,7 +56,7 @@
Features[Name] = Enabled;
}
- ArrayRef<Builtin::Info> getTargetBuiltins() const override { return None; }
+ ArrayRef<Builtin::Info> getTargetBuiltins() const override;
const char *getClobbers() const override { return ""; }
Index: clang/lib/Basic/Targets/BPF.cpp
===================================================================
--- clang/lib/Basic/Targets/BPF.cpp
+++ clang/lib/Basic/Targets/BPF.cpp
@@ -13,11 +13,18 @@
#include "BPF.h"
#include "Targets.h"
#include "clang/Basic/MacroBuilder.h"
+#include "clang/Basic/TargetBuiltins.h"
#include "llvm/ADT/StringRef.h"
using namespace clang;
using namespace clang::targets;
+const Builtin::Info BPFTargetInfo::BuiltinInfo[] = {
+#define BUILTIN(ID, TYPE, ATTRS) \
+ {#ID, TYPE, ATTRS, nullptr, ALL_LANGUAGES, nullptr},
+#include "clang/Basic/BuiltinsBPF.def"
+};
+
void BPFTargetInfo::getTargetDefines(const LangOptions &Opts,
MacroBuilder &Builder) const {
Builder.defineMacro("__bpf__");
@@ -34,3 +41,8 @@
void BPFTargetInfo::fillValidCPUList(SmallVectorImpl<StringRef> &Values) const {
Values.append(std::begin(ValidCPUNames), std::end(ValidCPUNames));
}
+
+ArrayRef<Builtin::Info> BPFTargetInfo::getTargetBuiltins() const {
+ return llvm::makeArrayRef(BuiltinInfo, clang::BPF::LastTSBuiltin -
+ Builtin::FirstTSBuiltin);
+}
Index: clang/include/clang/module.modulemap
===================================================================
--- clang/include/clang/module.modulemap
+++ clang/include/clang/module.modulemap
@@ -34,6 +34,7 @@
textual header "Basic/BuiltinsAArch64.def"
textual header "Basic/BuiltinsAMDGPU.def"
textual header "Basic/BuiltinsARM.def"
+ textual header "Basic/BuiltinsBPF.def"
textual header "Basic/Builtins.def"
textual header "Basic/BuiltinsHexagon.def"
textual header "Basic/BuiltinsLe64.def"
Index: clang/include/clang/Sema/Sema.h
===================================================================
--- clang/include/clang/Sema/Sema.h
+++ clang/include/clang/Sema/Sema.h
@@ -11050,6 +11050,7 @@
bool CheckARMBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall);
bool CheckAArch64BuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall);
+ bool CheckBPFBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall);
bool CheckHexagonBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall);
bool CheckHexagonBuiltinCpu(unsigned BuiltinID, CallExpr *TheCall);
bool CheckHexagonBuiltinArgument(unsigned BuiltinID, CallExpr *TheCall);
Index: clang/include/clang/Basic/TargetBuiltins.h
===================================================================
--- clang/include/clang/Basic/TargetBuiltins.h
+++ clang/include/clang/Basic/TargetBuiltins.h
@@ -52,6 +52,16 @@
};
}
+ /// BPF builtins
+ namespace BPF {
+ enum {
+ LastTIBuiltin = clang::Builtin::FirstTSBuiltin - 1,
+ #define BUILTIN(ID, TYPE, ATTRS) BI##ID,
+ #include "clang/Basic/BuiltinsBPF.def"
+ LastTSBuiltin
+ };
+ }
+
/// PPC builtins
namespace PPC {
enum {
Index: clang/include/clang/Basic/DiagnosticSemaKinds.td
===================================================================
--- clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -9950,6 +9950,11 @@
"%select{non-pointer|function pointer|void pointer}0 argument to "
"'__builtin_launder' is not allowed">;
+def err_preserve_field_info_not_field : Error<
+ "__builtin_preserve_field_info argument %0 not a field access">;
+def err_preserve_field_info_not_const: Error<
+ "__builtin_preserve_field_info argument %0 not a constant">;
+
def err_bit_cast_non_trivially_copyable : Error<
"__builtin_bit_cast %select{source|destination}0 type must be trivially copyable">;
def err_bit_cast_type_size_mismatch : Error<
Index: clang/include/clang/Basic/BuiltinsBPF.def
===================================================================
--- /dev/null
+++ clang/include/clang/Basic/BuiltinsBPF.def
@@ -0,0 +1,24 @@
+//===--- BuiltinsBPF.def - BPF Builtin function database --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the BPF-specific builtin function database. Users of
+// this file must define the BUILTIN macro to make use of this information.
+//
+//===----------------------------------------------------------------------===//
+
+// The format of this database matches clang/Basic/Builtins.def.
+
+#if defined(BUILTIN) && !defined(TARGET_BUILTIN)
+# define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BUILTIN(ID, TYPE, ATTRS)
+#endif
+
+// Get record field information.
+TARGET_BUILTIN(__builtin_preserve_field_info, "Ui.", "t", "")
+
+#undef BUILTIN
+#undef TARGET_BUILTIN
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
