samitolvanen created this revision.
Herald added subscribers: jobnoorman, luke, VincentWu, ormris, vkmr,
frasercrmck, evandro, luismarques, apazos, sameer.abuasal, pengfei, s.egerton,
Jim, benna, psnobl, jocewei, PkmX, the_o, brucehoult, MartinMosbeck, rogfer01,
edward-jones, zzheng, jrtc27, shiva0217, kito-cheng, niosHD, sabuasal,
simoncook, johnrusso, rbar, asb, hiraditya, arichardson.
Herald added a project: All.
samitolvanen requested review of this revision.
Herald added subscribers: llvm-commits, cfe-commits, pcwang-thead, eopXD,
MaskRay.
Herald added projects: clang, LLVM.
Similarly to the target-specific lowering added in D119296
<https://reviews.llvm.org/D119296>, implement
KCFI operand bundle lowering for RV64. Compared to the generic
lowering pass, this ensures we always emit a predictable instruction
sequence for indirect call checks, and that the checks are emitted
immediately before the call.
Note that unlike the generic pass, this patch emits an `ebreak`
instruction for error handling to match the Linux kernel's `BUG()`
implementation. Just like for X86, we also emit trap locations to a
`.kcfi_traps` section as we cannot embed additional information to the
trap instruction itself.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D148385
Files:
clang/lib/CodeGen/BackendUtil.cpp
llvm/lib/Target/RISCV/CMakeLists.txt
llvm/lib/Target/RISCV/RISCV.h
llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
llvm/lib/Target/RISCV/RISCVISelLowering.cpp
llvm/lib/Target/RISCV/RISCVISelLowering.h
llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
llvm/lib/Target/RISCV/RISCVInstrInfo.h
llvm/lib/Target/RISCV/RISCVInstrInfo.td
llvm/lib/Target/RISCV/RISCVKCFI.cpp
llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
llvm/test/CodeGen/RISCV/O0-pipeline.ll
llvm/test/CodeGen/RISCV/O3-pipeline.ll
llvm/test/CodeGen/RISCV/kcfi-patchable-function-prefix.ll
llvm/test/CodeGen/RISCV/kcfi.ll
llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
Index: llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
===================================================================
--- llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
+++ llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
@@ -78,6 +78,7 @@
"RISCVInsertNTLHInsts.cpp",
"RISCVInsertVSETVLI.cpp",
"RISCVInstrInfo.cpp",
+ "RISCVKCFI.cpp",
"RISCVMCInstLower.cpp",
"RISCVMachineFunctionInfo.cpp",
"RISCVMacroFusion.cpp",
Index: llvm/test/CodeGen/RISCV/kcfi.ll
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/RISCV/kcfi.ll
@@ -0,0 +1,73 @@
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs -riscv-no-aliases < %s | FileCheck %s --check-prefix=ASM
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs -stop-after=finalize-isel < %s | FileCheck %s --check-prefixes=MIR,ISEL
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs -stop-after=riscv-kcfi < %s | FileCheck %s --check-prefixes=MIR,KCFI
+
+; ASM: .word 12345678
+define void @f1(ptr noundef %x) !kcfi_type !1 {
+; ASM-LABEL: f1:
+; ASM: # %bb.0:
+; ASM: lw t1, -4(a0)
+; ASM-NEXT: lui t2, 3014
+; ASM-NEXT: addiw t2, t2, 334
+; ASM-NEXT: beq t1, t2, .Ltmp0
+; ASM-NEXT: .Ltmp1:
+; ASM-NEXT: ebreak
+; ASM-NEXT: .section .kcfi_traps,"ao",@progbits,.text
+; ASM-NEXT: .Ltmp2:
+; ASM-NEXT: .word .Ltmp1-.Ltmp2
+; ASM-NEXT: .text
+; ASM-NEXT: .Ltmp0:
+; ASM-NEXT: jalr ra, 0(a0)
+
+; MIR-LABEL: name: f1
+; MIR: body:
+
+; ISEL: PseudoCALLIndirect %0,{{.*}} cfi-type 12345678
+
+; KCFI: BUNDLE{{.*}} {
+; KCFI-NEXT: KCFI_CHECK $x10, 12345678,{{.*}}
+; KCFI-NEXT: PseudoCALLIndirect killed $x10,{{.*}}
+; KCFI-NEXT: }
+
+ call void %x() [ "kcfi"(i32 12345678) ]
+ ret void
+}
+
+; ASM-NOT: .word:
+define void @f2(ptr noundef %x) #0 {
+; ASM-LABEL: f2:
+; ASM: # %bb.0:
+; ASM-NEXT: addi zero, zero, 0
+; ASM-NEXT: addi zero, zero, 0
+; ASM: lw t1, -4(a0)
+; ASM-NEXT: lui t2, 3014
+; ASM-NEXT: addiw t2, t2, 334
+; ASM-NEXT: beq t1, t2, .Ltmp3
+; ASM-NEXT: .Ltmp4:
+; ASM-NEXT: ebreak
+; ASM-NEXT: .section .kcfi_traps,"ao",@progbits,.text
+; ASM-NEXT: .Ltmp5:
+; ASM-NEXT: .word .Ltmp4-.Ltmp5
+; ASM-NEXT: .text
+; ASM-NEXT: .Ltmp3:
+; ASM-NEXT: jalr zero, 0(a0)
+
+; MIR-LABEL: name: f2
+; MIR: body:
+
+; ISEL: PseudoTAILIndirect %0,{{.*}} cfi-type 12345678
+
+; KCFI: BUNDLE{{.*}} {
+; KCFI-NEXT: KCFI_CHECK $x10, 12345678,{{.*}}
+; KCFI-NEXT: PseudoTAILIndirect killed $x10,{{.*}}
+; KCFI-NEXT: }
+
+ tail call void %x() [ "kcfi"(i32 12345678) ]
+ ret void
+}
+
+attributes #0 = { "patchable-function-entry"="2" }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 4, !"kcfi", i32 1}
+!1 = !{i32 12345678}
Index: llvm/test/CodeGen/RISCV/kcfi-patchable-function-prefix.ll
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/RISCV/kcfi-patchable-function-prefix.ll
@@ -0,0 +1,54 @@
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NOC
+; RUN: llc -mtriple=riscv64 -mattr=+c -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,C
+
+; NOC: .p2align 2
+; C: .p2align 1
+; CHECK-NOT: nop
+; CHECK: .word 12345678
+; CHECK-LABEL: f1:
+define void @f1(ptr noundef %x) !kcfi_type !1 {
+; CHECK: lw t1, -4(a0)
+ call void %x() [ "kcfi"(i32 12345678) ]
+ ret void
+}
+
+; NOC: .p2align 2
+; C: .p2align 1
+; CHECK-NOT: .word
+; CHECK-NOT: nop
+; CHECK-LABEL: f2:
+define void @f2(ptr noundef %x) {
+; CHECK: lw t1, -4(a0)
+ call void %x() [ "kcfi"(i32 12345678) ]
+ ret void
+}
+
+; NOC: .p2align 2
+; C: .p2align 1
+; CHECK: .word 12345678
+; CHECK-COUNT-11: nop
+; CHECK-LABEL: f3:
+define void @f3(ptr noundef %x) #0 !kcfi_type !1 {
+; NOC: lw t1, -48(a0)
+; C: lw t1, -26(a0)
+ call void %x() [ "kcfi"(i32 12345678) ]
+ ret void
+}
+
+; NOC: .p2align 2
+; C: .p2align 1
+; CHECK-NOT: .word
+; CHECK-COUNT-11: nop
+; CHECK-LABEL: f4:
+define void @f4(ptr noundef %x) #0 {
+; NOC: lw t1, -48(a0)
+; C: lw t1, -26(a0)
+ call void %x() [ "kcfi"(i32 12345678) ]
+ ret void
+}
+
+attributes #0 = { "patchable-function-prefix"="11" }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 4, !"kcfi", i32 1}
+!1 = !{i32 12345678}
Index: llvm/test/CodeGen/RISCV/O3-pipeline.ll
===================================================================
--- llvm/test/CodeGen/RISCV/O3-pipeline.ll
+++ llvm/test/CodeGen/RISCV/O3-pipeline.ll
@@ -154,6 +154,7 @@
; CHECK-NEXT: Tail Duplication
; CHECK-NEXT: Machine Copy Propagation Pass
; CHECK-NEXT: Post-RA pseudo instruction expansion pass
+; RV64-NEXT: Insert KCFI indirect call checks
; CHECK-NEXT: MachineDominator Tree Construction
; CHECK-NEXT: Machine Natural Loop Construction
; CHECK-NEXT: Post RA top-down list latency scheduler
@@ -179,6 +180,7 @@
; CHECK-NEXT: RISC-V pseudo instruction expansion pass
; CHECK-NEXT: RISC-V insert NTLH instruction pass
; CHECK-NEXT: RISC-V atomic pseudo instruction expansion pass
+; RV64-NEXT: Unpack machine instruction bundles
; CHECK-NEXT: Lazy Machine Block Frequency Analysis
; CHECK-NEXT: Machine Optimization Remark Emitter
; CHECK-NEXT: RISC-V Assembly Printer
Index: llvm/test/CodeGen/RISCV/O0-pipeline.ll
===================================================================
--- llvm/test/CodeGen/RISCV/O0-pipeline.ll
+++ llvm/test/CodeGen/RISCV/O0-pipeline.ll
@@ -3,7 +3,7 @@
; RUN: FileCheck %s --check-prefixes=CHECK
; RUN: llc -mtriple=riscv64 -O0 -debug-pass=Structure < %s -o /dev/null 2>&1 | \
; RUN: grep -v "Verify generated machine code" | \
-; RUN: FileCheck %s --check-prefixes=CHECK
+; RUN: FileCheck %s --check-prefixes=CHECK,RV64
; REQUIRES: asserts
@@ -50,6 +50,7 @@
; CHECK-NEXT: Machine Optimization Remark Emitter
; CHECK-NEXT: Prologue/Epilogue Insertion & Frame Finalization
; CHECK-NEXT: Post-RA pseudo instruction expansion pass
+; RV64-NEXT: Insert KCFI indirect call checks
; CHECK-NEXT: Analyze Machine Code For Garbage Collection
; CHECK-NEXT: Insert fentry calls
; CHECK-NEXT: Insert XRay ops
@@ -66,6 +67,7 @@
; CHECK-NEXT: RISC-V pseudo instruction expansion pass
; CHECK-NEXT: RISC-V insert NTLH instruction pass
; CHECK-NEXT: RISC-V atomic pseudo instruction expansion pass
+; RV64-NEXT: Unpack machine instruction bundles
; CHECK-NEXT: Lazy Machine Block Frequency Analysis
; CHECK-NEXT: Machine Optimization Remark Emitter
; CHECK-NEXT: RISC-V Assembly Printer
Index: llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
===================================================================
--- llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -87,6 +87,7 @@
initializeRISCVInsertVSETVLIPass(*PR);
initializeRISCVDAGToDAGISelPass(*PR);
initializeRISCVInitUndefPass(*PR);
+ initializeRISCVKCFIPass(*PR);
}
static StringRef computeDataLayout(const Triple &TT) {
@@ -332,7 +333,11 @@
return false;
}
-void RISCVPassConfig::addPreSched2() {}
+void RISCVPassConfig::addPreSched2() {
+ // Emit KCFI checks for indirect calls.
+ if (TM->getTargetTriple().isRISCV64())
+ addPass(createRISCVKCFIPass());
+}
void RISCVPassConfig::addPreEmitPass() {
addPass(&BranchRelaxationPassID);
@@ -355,6 +360,13 @@
// possibility for other passes to break the requirements for forward
// progress in the LR/SC block.
addPass(createRISCVExpandAtomicPseudoPass());
+
+ // KCFI indirect call checks are lowered to a bundle.
+ if (TM->getTargetTriple().isRISCV64()) {
+ addPass(createUnpackMachineBundles([&](const MachineFunction &MF) {
+ return MF.getFunction().getParent()->getModuleFlag("kcfi");
+ }));
+ }
}
void RISCVPassConfig::addMachineSSAOptimization() {
Index: llvm/lib/Target/RISCV/RISCVKCFI.cpp
===================================================================
--- /dev/null
+++ llvm/lib/Target/RISCV/RISCVKCFI.cpp
@@ -0,0 +1,115 @@
+//===---- RISCVKCFI.cpp - Implements KCFI ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements KCFI indirect call checking.
+//
+//===----------------------------------------------------------------------===//
+
+#include "RISCV.h"
+#include "RISCVInstrInfo.h"
+#include "RISCVSubtarget.h"
+#include "RISCVTargetMachine.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineInstrBundle.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "riscv-kcfi"
+#define RISCV_KCFI_PASS_NAME "Insert KCFI indirect call checks"
+
+STATISTIC(NumKCFIChecksAdded, "Number of indirect call checks added");
+
+namespace {
+class RISCVKCFI : public MachineFunctionPass {
+public:
+ static char ID;
+
+ RISCVKCFI() : MachineFunctionPass(ID) {}
+
+ StringRef getPassName() const override { return RISCV_KCFI_PASS_NAME; }
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+private:
+ /// Machine instruction info used throughout the class.
+ const RISCVInstrInfo *TII = nullptr;
+
+ /// Emits a KCFI check before an indirect call.
+ /// \returns true if the check was added and false otherwise.
+ bool emitCheck(MachineBasicBlock &MBB,
+ MachineBasicBlock::instr_iterator I) const;
+};
+
+char RISCVKCFI::ID = 0;
+} // end anonymous namespace
+
+INITIALIZE_PASS(RISCVKCFI, DEBUG_TYPE, RISCV_KCFI_PASS_NAME, false, false)
+
+FunctionPass *llvm::createRISCVKCFIPass() { return new RISCVKCFI(); }
+
+bool RISCVKCFI::emitCheck(MachineBasicBlock &MBB,
+ MachineBasicBlock::instr_iterator MBBI) const {
+ assert(TII && "Target instruction info was not initialized");
+
+ // If the call instruction is bundled, we can only emit a check safely if
+ // it's the first instruction in the bundle.
+ if (MBBI->isBundled() && !std::prev(MBBI)->isBundle())
+ report_fatal_error("Cannot emit a KCFI check for a bundled call");
+
+ switch (MBBI->getOpcode()) {
+ case RISCV::PseudoCALLIndirect:
+ case RISCV::PseudoTAILIndirect:
+ break;
+ default:
+ llvm_unreachable("Unexpected CFI call opcode");
+ }
+
+ MachineOperand &Target = MBBI->getOperand(0);
+ assert(Target.isReg() && "Invalid target operand for an indirect call");
+ Target.setIsRenamable(false);
+
+ MachineInstr *Check =
+ BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(RISCV::KCFI_CHECK))
+ .addReg(Target.getReg())
+ .addImm(MBBI->getCFIType())
+ .getInstr();
+ MBBI->setCFIType(*MBB.getParent(), 0);
+
+ // If not already bundled, bundle the check and the call to prevent
+ // further changes.
+ if (!MBBI->isBundled())
+ finalizeBundle(MBB, Check->getIterator(), std::next(MBBI->getIterator()));
+
+ ++NumKCFIChecksAdded;
+ return true;
+}
+
+bool RISCVKCFI::runOnMachineFunction(MachineFunction &MF) {
+ const Module *M = MF.getMMI().getModule();
+ if (!M->getModuleFlag("kcfi"))
+ return false;
+
+ const auto &SubTarget = MF.getSubtarget<RISCVSubtarget>();
+ TII = SubTarget.getInstrInfo();
+ if (!SubTarget.is64Bit())
+ return false;
+
+ bool Changed = false;
+ for (MachineBasicBlock &MBB : MF) {
+ for (MachineBasicBlock::instr_iterator MII = MBB.instr_begin(),
+ MIE = MBB.instr_end();
+ MII != MIE; ++MII) {
+ if (MII->isCall() && MII->getCFIType())
+ Changed |= emitCheck(MBB, MII);
+ }
+ }
+
+ return Changed;
+}
Index: llvm/lib/Target/RISCV/RISCVInstrInfo.td
===================================================================
--- llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -1891,6 +1891,14 @@
[(int_hwasan_check_memaccess_shortgranules X5, GPRJALR:$ptr,
(i32 timm:$accessinfo))]>;
+// This gets lowered into a 20-byte instruction sequence (at most)
+let hasSideEffects = 0, mayLoad = 1, mayStore = 0,
+ Predicates = [IsRV64], Defs = [ X6, X7, X28, X29, X30, X31 ], Size = 20 in {
+def KCFI_CHECK : Pseudo<
+ (outs), (ins GPRJALR:$ptr, i32imm:$type), []>, Sched<[]>;
+}
+
+
/// Simple optimization
def : Pat<(add GPR:$rs1, (AddiPair:$rs2)),
(ADDI (ADDI GPR:$rs1, (AddiPairImmLarge AddiPair:$rs2)),
Index: llvm/lib/Target/RISCV/RISCVInstrInfo.h
===================================================================
--- llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -229,6 +229,9 @@
protected:
const RISCVSubtarget &STI;
+
+private:
+ unsigned getInstBundleLength(const MachineInstr &MI) const;
};
namespace RISCV {
Index: llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
===================================================================
--- llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -1257,6 +1257,9 @@
}
}
+ if (Opcode == TargetOpcode::BUNDLE)
+ return getInstBundleLength(MI);
+
if (MI.getParent() && MI.getParent()->getParent()) {
if (isCompressibleInst(MI, STI))
return 2;
@@ -1264,6 +1267,17 @@
return get(Opcode).getSize();
}
+unsigned RISCVInstrInfo::getInstBundleLength(const MachineInstr &MI) const {
+ unsigned Size = 0;
+ MachineBasicBlock::const_instr_iterator I = MI.getIterator();
+ MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
+ while (++I != E && I->isInsideBundle()) {
+ assert(!I->isBundle() && "No nested bundle!");
+ Size += getInstSizeInBytes(*I);
+ }
+ return Size;
+}
+
bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
const unsigned Opcode = MI.getOpcode();
switch (Opcode) {
Index: llvm/lib/Target/RISCV/RISCVISelLowering.h
===================================================================
--- llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -702,6 +702,10 @@
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI,
unsigned Factor) const override;
+ bool supportKCFIBundles() const override {
+ return getTargetMachine().getTargetTriple().isRISCV64();
+ }
+
private:
/// RISCVCCAssignFn - This target-specific function extends the default
/// CCValAssign with additional information used to lower RISC-V calling
Index: llvm/lib/Target/RISCV/RISCVISelLowering.cpp
===================================================================
--- llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -13921,6 +13921,7 @@
SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
SDValue Chain = CLI.Chain;
SDValue Callee = CLI.Callee;
+ bool IsCFICall = CLI.CB && CLI.CB->isIndirectCall() && CLI.CFIType;
bool &IsTailCall = CLI.IsTailCall;
CallingConv::ID CallConv = CLI.CallConv;
bool IsVarArg = CLI.IsVarArg;
@@ -14166,10 +14167,15 @@
if (IsTailCall) {
MF.getFrameInfo().setHasTailCall();
- return DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
+ SDValue Ret = DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
+ if (IsCFICall)
+ Ret.getNode()->setCFIType(CLI.CFIType->getZExtValue());
+ return Ret;
}
Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
+ if (IsCFICall)
+ Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
Glue = Chain.getValue(1);
Index: llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
===================================================================
--- llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
+++ llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
@@ -18,6 +18,7 @@
#include "RISCVMachineFunctionInfo.h"
#include "RISCVTargetMachine.h"
#include "TargetInfo/RISCVTargetInfo.h"
+#include "llvm/ADT/APInt.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGen/AsmPrinter.h"
@@ -71,6 +72,7 @@
typedef std::tuple<unsigned, uint32_t> HwasanMemaccessTuple;
std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
+ void LowerKCFI_CHECK(const MachineInstr &MI);
void EmitHwasanMemaccessSymbols(Module &M);
// Wrapper needed for tblgenned pseudo lowering.
@@ -113,6 +115,9 @@
case RISCV::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
LowerHWASAN_CHECK_MEMACCESS(*MI);
return;
+ case RISCV::KCFI_CHECK:
+ LowerKCFI_CHECK(*MI);
+ return;
case RISCV::PseudoRVVInitUndefM1:
case RISCV::PseudoRVVInitUndefM2:
case RISCV::PseudoRVVInitUndefM4:
@@ -268,6 +273,93 @@
EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr));
}
+void RISCVAsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
+ assert(STI->is64Bit() && "KCFI_CHECK requires RV64");
+
+ Register AddrReg = MI.getOperand(0).getReg();
+ assert(std::next(MI.getIterator())->isCall() &&
+ "KCFI_CHECK not followed by a call instruction");
+ assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
+ "KCFI_CHECK call target doesn't match call operand");
+
+ // Temporary registers for comparing comparing the hashes. If a register is
+ // used for the call target, or reserved by the user, we can clobber another
+ // temporary register as the check is immediately followed by the call. The
+ // check defaults to X6/X7, but can fall back to X28-X31 if needed.
+ unsigned ScratchRegs[] = {RISCV::X6, RISCV::X7};
+ unsigned NextReg = RISCV::X28;
+ auto isRegAvailable = [&](unsigned Reg) {
+ return (Reg != AddrReg && !STI->isRegisterReservedByUser(Reg));
+ };
+ for (auto &Reg : ScratchRegs) {
+ if (isRegAvailable(Reg))
+ continue;
+ while (!isRegAvailable(NextReg))
+ ++NextReg;
+ Reg = NextReg++;
+ if (Reg > RISCV::X31)
+ report_fatal_error("Unable to find scratch registers for KCFI_CHECK");
+ }
+ assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
+ "Invalid scratch registers for KCFI_CHECK");
+
+ if (AddrReg == RISCV::X0) {
+ // Checking X0 makes no sense. Instead of emitting a load, zero
+ // ScratchRegs[0].
+ EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::ADDI)
+ .addReg(ScratchRegs[0])
+ .addReg(RISCV::X0)
+ .addReg(RISCV::X0)
+ .addImm(0));
+ } else {
+ // Adjust the offset for patchable-function-prefix. This assumes that
+ // patchable-function-prefix is the same for all functions.
+ int NopSize =
+ STI->getInstrInfo()->getNop().getOpcode() == RISCV::C_NOP ? 2 : 4;
+ int64_t PrefixNops = 0;
+ (void)MI.getMF()
+ ->getFunction()
+ .getFnAttribute("patchable-function-prefix")
+ .getValueAsString()
+ .getAsInteger(10, PrefixNops);
+
+ // Load the target function type hash.
+ EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::LW)
+ .addReg(ScratchRegs[0])
+ .addReg(AddrReg)
+ .addImm(-(PrefixNops * NopSize + 4)));
+ }
+
+ // Load the expected type hash.
+ const int64_t Type = MI.getOperand(1).getImm();
+ const int64_t Hi20 = ((Type + 0x800) >> 12) & 0xFFFFF;
+ const int64_t Lo12 = SignExtend64<12>(Type);
+ if (Hi20)
+ EmitToStreamer(
+ *OutStreamer,
+ MCInstBuilder(RISCV::LUI).addReg(ScratchRegs[1]).addImm(Hi20));
+ if (Lo12 || Hi20 == 0)
+ EmitToStreamer(*OutStreamer,
+ MCInstBuilder(Hi20 ? RISCV::ADDIW : RISCV::ADDI)
+ .addReg(ScratchRegs[1])
+ .addReg(ScratchRegs[1])
+ .addImm(Lo12));
+
+ // Compare the hashes and trap if there's a mismatch.
+ MCSymbol *Pass = OutContext.createTempSymbol();
+ EmitToStreamer(*OutStreamer,
+ MCInstBuilder(RISCV::BEQ)
+ .addReg(ScratchRegs[0])
+ .addReg(ScratchRegs[1])
+ .addExpr(MCSymbolRefExpr::create(Pass, OutContext)));
+
+ MCSymbol *Trap = OutContext.createTempSymbol();
+ OutStreamer->emitLabel(Trap);
+ EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::EBREAK));
+ emitKCFITrapEntry(*MI.getMF(), Trap);
+ OutStreamer->emitLabel(Pass);
+}
+
void RISCVAsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
if (HwasanMemaccessSymbols.empty())
return;
Index: llvm/lib/Target/RISCV/RISCV.h
===================================================================
--- llvm/lib/Target/RISCV/RISCV.h
+++ llvm/lib/Target/RISCV/RISCV.h
@@ -75,6 +75,9 @@
void initializeRISCVInitUndefPass(PassRegistry &);
extern char &RISCVInitUndefID;
+FunctionPass *createRISCVKCFIPass();
+void initializeRISCVKCFIPass(PassRegistry &);
+
InstructionSelector *createRISCVInstructionSelector(const RISCVTargetMachine &,
RISCVSubtarget &,
RISCVRegisterBankInfo &);
Index: llvm/lib/Target/RISCV/CMakeLists.txt
===================================================================
--- llvm/lib/Target/RISCV/CMakeLists.txt
+++ llvm/lib/Target/RISCV/CMakeLists.txt
@@ -31,6 +31,7 @@
RISCVInstrInfo.cpp
RISCVISelDAGToDAG.cpp
RISCVISelLowering.cpp
+ RISCVKCFI.cpp
RISCVMachineFunctionInfo.cpp
RISCVMacroFusion.cpp
RISCVMCInstLower.cpp
Index: clang/lib/CodeGen/BackendUtil.cpp
===================================================================
--- clang/lib/CodeGen/BackendUtil.cpp
+++ clang/lib/CodeGen/BackendUtil.cpp
@@ -628,7 +628,7 @@
PassBuilder &PB) {
// If the back-end supports KCFI operand bundle lowering, skip KCFIPass.
if (TargetTriple.getArch() == llvm::Triple::x86_64 ||
- TargetTriple.isAArch64(64))
+ TargetTriple.isAArch64(64) || TargetTriple.isRISCV64())
return;
// Ensure we lower KCFI operand bundles with -O0.
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
