kuhar created this revision.
kuhar added reviewers: rsmith, pcc, Prazek, sanjoy.
kuhar added a project: clang.
Herald added a subscriber: eraman.
Herald added a reviewer: grosser.
Traditionally, to force some inlining decisions one has to annotate function
declarations with `__attribute__((always_inline))` or
`__attribute__((noinline))`. One problem with these attributes is that they
affect every call site. Always inlining or forbidding inlining may not be
desirable in every context, and a workaround for that is creating a few copies
of functions, each with a different inline attribute. Furthermore, it's not
always feasible (in every project) to modify library code and introduce new
function attributes there.
This patch introduces a new way of forcing inlining decisions on a per-callsite
basis. This allows for more fine-grained control over inlining, without
creating any duplicate functions. The two new intrinsics for controlling
inlining are:
- `__builtin_always_inline(Foo())` -- inlines the function `Foo` at the
callsite, if possible. Internally, this applies the `alwaysinline` attribute to
the generated call instruction.
- `__builtin_no_inline(Foo())` -- forbids the function `Foo` to be inlined at
the callsite. Internally, this applies the `noinline` attribute to the
generated call instruction.
The inline intrinsics support function, function pointer, member function,
member function pointer, virutal function, and operator calls. Support for
constructor calls (`CXXTemporaryExpr`) should also be possible, but is not the
part of this patch.
Repository:
rC Clang
https://reviews.llvm.org/D51200
Files:
include/clang/Basic/Builtins.def
include/clang/Basic/DiagnosticSemaKinds.td
include/clang/CodeGen/CGFunctionInfo.h
lib/CodeGen/CGBuiltin.cpp
lib/CodeGen/CGCall.cpp
lib/CodeGen/CGExpr.cpp
lib/CodeGen/CGExprCXX.cpp
lib/CodeGen/CodeGenFunction.h
lib/CodeGen/CodeGenTypes.h
lib/Sema/SemaChecking.cpp
test/CodeGenCXX/inline_builtin_call.cpp
test/Sema/inline-builtins.c
test/SemaCXX/inline-builtins.cpp
Index: test/SemaCXX/inline-builtins.cpp
===================================================================
--- /dev/null
+++ test/SemaCXX/inline-builtins.cpp
@@ -0,0 +1,83 @@
+// RUN: %clang_cc1 -std=c++17 -fsyntax-only -verify %s
+
+struct S {
+ S();
+ void foo();
+ void bar(int);
+ int baz(float, char);
+ static void lol();
+
+ virtual void virt();
+ void operator++();
+ friend S operator+(const S &, const S &);
+};
+
+S operator"" _s(unsigned long long);
+
+S &getS();
+
+void test_positive() {
+ S &s = getS();
+ __builtin_always_inline(s.foo());
+ __builtin_no_inline(s.foo());
+
+ __builtin_always_inline(s.bar(1));
+ __builtin_no_inline(s.bar(1));
+
+ int a = __builtin_always_inline(s.baz(3.14, 'a'));
+ int b = __builtin_no_inline(s.baz(3.14, 'a'));
+
+ __builtin_always_inline(s.lol());
+ __builtin_no_inline(s.lol());
+
+ __builtin_always_inline(S::lol());
+ __builtin_no_inline(S::lol());
+
+ __builtin_always_inline(s.virt());
+ __builtin_no_inline(s.virt());
+
+ __builtin_always_inline(++s);
+ __builtin_no_inline(++s);
+
+ __builtin_always_inline(s.operator++());
+ __builtin_no_inline(s.operator++());
+
+ S s1;
+ __builtin_always_inline(s + s1);
+ __builtin_no_inline(s + s1);
+
+ auto mptr = &S::foo;
+ __builtin_always_inline((s.*mptr)());
+ __builtin_no_inline((s.*mptr)());
+
+ __builtin_always_inline(s.~S());
+ __builtin_no_inline(s.~S());
+
+ __builtin_always_inline([] {}());
+ __builtin_no_inline([] {}());
+
+ auto lam = [&a, b](S &ss) { return ++a + b; };
+ int c = __builtin_always_inline(lam(s1));
+ int d = __builtin_no_inline(lam(s1));
+}
+
+void test_errors() {
+ using I = int;
+ I e = 1;
+ __builtin_always_inline(e.~I()); // expected-error {{argument to __builtin_always_inline must not be a pseudo-destructor call}}
+ __builtin_no_inline(e.~I()); // expected-error {{argument to __builtin_no_inline must not be a pseudo-destructor call}}
+
+ S s2 = __builtin_always_inline(1_s); // expected-error {{argument to __builtin_always_inline must be a function, member function, or operator call}}
+ s2 = __builtin_no_inline(1_s); // expected-error {{argument to __builtin_no_inline must be a function, member function, or operator call}}
+
+ // TODO: This should be handled as well.
+ S s3 = __builtin_always_inline(S()); // expected-error {{argument to __builtin_always_inline must be a function, member function, or operator call}}
+ S s4 = __builtin_no_inline(S()); // expected-error {{argument to __builtin_no_inline must be a function, member function, or operator call}}
+}
+
+constexpr int f1() { return 1; }
+
+// TODO: It should be possible to make the inline intrinsics constexpr-friendly.
+constexpr int f2(int x) { // expected-error {{constexpr function never produces a constant expression}}
+ return x + __builtin_always_inline(f1()); // expected-note {{subexpression not valid in a constant expression}}
+}
Index: test/Sema/inline-builtins.c
===================================================================
--- /dev/null
+++ test/Sema/inline-builtins.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -fsyntax-only -fblocks -verify %s
+
+void foo();
+
+void test() {
+ void (*bar)() = &foo;
+ __builtin_always_inline(foo());
+ __builtin_no_inline(foo());
+
+ __builtin_always_inline(bar());
+ __builtin_no_inline(bar());
+ __builtin_always_inline((*bar)());
+ __builtin_no_inline((*bar)());
+
+ __builtin_always_inline(^{}()); // expected-error {{argument to __builtin_always_inline must not be a block call}}
+ __builtin_no_inline(^{}()); // expected-error {{argument to __builtin_no_inline must not be a block call}}
+
+ __builtin_always_inline(__builtin_always_inline(foo())); // expected-error {{argument to __builtin_always_inline must not be a builtin call}}
+ __builtin_no_inline(__builtin_no_inline(foo())); // expected-error {{argument to __builtin_no_inline must not be a builtin call}}
+}
Index: test/CodeGenCXX/inline_builtin_call.cpp
===================================================================
--- /dev/null
+++ test/CodeGenCXX/inline_builtin_call.cpp
@@ -0,0 +1,117 @@
+// RUN: %clang_cc1 -std=c++17 -O0 -disable-O0-optnone -triple x86_64-unknown-linux -emit-llvm -o - %s | FileCheck %s
+
+struct S {
+ S() {}
+ void foo();
+ void bar(int);
+ int baz(float, char);
+ static void lol();
+
+ virtual void virt(){};
+ void operator++();
+ friend S operator+(const S &, const S &);
+};
+
+void test_always() {
+ S s;
+
+ // CHECK: call void @_ZN1S3fooEv({{.*}}) [[ATTR_ALWAYS:#[0-9]+]]
+ __builtin_always_inline(s.foo());
+
+ // CHECK-NOT: call void @_ZN1S3fooEv({{.*}}) [[ATTR_ALWAYS]]
+ // CHECK: call void @_ZN1S3fooEv({{.*}})
+ s.foo();
+
+ // CHECK: call void @_ZN1S3barEi({{.*}}) [[ATTR_ALWAYS]]
+ __builtin_always_inline(s.bar(42));
+
+ // CHECK: call i32 @_ZN1S3bazEfc({{.*}}) [[ATTR_ALWAYS]]
+ __builtin_always_inline(s.baz(3.14f, 'x'));
+
+ // CHECK: call i32 @_ZN1S3bazEfc({{.*}}) [[ATTR_ALWAYS]]
+ // CHECK-NOT: call void @_ZN1S3barEi({{.*}}) [[ATTR_ALWAYS]]
+ // CHECK: call void @_ZN1S3barEi({{.*}})
+ s.bar(__builtin_always_inline(s.baz(3.14f, 'x')));
+
+ S s2;
+
+ // CHECK-NOT: call void @_ZN1SppEv({{.*}}) [[ATTR_ALWAYS]]
+ // CHECK: call void @_ZN1SppEv({{.*}})
+ ++s2;
+ // CHECK: call void @_ZN1SppEv({{.*}}) [[ATTR_ALWAYS]]
+ __builtin_always_inline(++s2);
+
+ // CHECK: call void @_ZplRK1SS1_({{.*}}) [[ATTR_ALWAYS]]
+ __builtin_always_inline(s + s2);
+
+ // CHECK: call void @_ZN1S3lolEv({{.*}}) [[ATTR_ALWAYS]]
+ __builtin_always_inline(s.lol());
+
+ // CHECK: call void @_ZN1S3lolEv({{.*}}) [[ATTR_ALWAYS]]
+ __builtin_always_inline(S::lol());
+}
+
+void test_no_inline() {
+ S s;
+
+ // CHECK: call void @_ZN1S3fooEv({{.*}}) [[ATTR_NO:#[0-9]+]]
+ __builtin_no_inline(s.foo());
+
+ // CHECK-NOT: call void @_ZN1S3fooEv({{.*}}) [[ATTR_NO]]
+ // CHECK: call void @_ZN1S3fooEv({{.*}})
+ s.foo();
+
+ // CHECK: call void @_ZN1S3barEi({{.*}}) [[ATTR_NO]]
+ __builtin_no_inline(s.bar(42));
+
+ // CHECK: call i32 @_ZN1S3bazEfc({{.*}}) [[ATTR_NO]]
+ __builtin_no_inline(s.baz(3.14f, 'x'));
+
+ // CHECK: call i32 @_ZN1S3bazEfc({{.*}}) [[ATTR_NO]]
+ // CHECK-NOT: call void @_ZN1S3barEi({{.*}}) [[ATTR_NO]]
+ // CHECK: call void @_ZN1S3barEi({{.*}})
+ s.bar(__builtin_no_inline(s.baz(3.14f, 'x')));
+
+ S s2;
+
+ // CHECK-NOT: call void @_ZN1SppEv({{.*}}) [[ATTR_NO]]
+ // CHECK: call void @_ZN1SppEv({{.*}})
+ ++s2;
+ // CHECK: call void @_ZN1SppEv({{.*}}) [[ATTR_NO]]
+ __builtin_no_inline(++s2);
+
+ // CHECK: call void @_ZplRK1SS1_({{.*}}) [[ATTR_NO]]
+ __builtin_no_inline(s + s2);
+
+ // CHECK: call void @_ZN1S3lolEv({{.*}}) [[ATTR_NO]]
+ __builtin_no_inline(s.lol());
+
+ // CHECK: call void @_ZN1S3lolEv({{.*}}) [[ATTR_NO]]
+ __builtin_no_inline(S::lol());
+}
+
+S &getS();
+
+void test_indirect() {
+ auto fptr = &test_always;
+ // CHECK: call void %{{.*}} [[ATTR_ALWAYS]]
+ __builtin_always_inline(fptr());
+ // CHECK: call void %{{.*}} [[ATTR_NO]]
+ __builtin_no_inline(fptr());
+
+ // CHECK: call dereferenceable(8) %struct.S* @_Z4getSv()
+ S &s = getS();
+ // CHECK: call void %{{.*}} [[ATTR_ALWAYS]]
+ __builtin_always_inline(s.virt());
+ // CHECK: call void %{{.*}} [[ATTR_NO]]
+ __builtin_no_inline(s.virt());
+
+ auto mptr = &S::foo;
+ // CHECK: call void %{{.*}} [[ATTR_ALWAYS]]
+ __builtin_always_inline((s.*mptr)());
+ // CHECK: call void %{{.*}} [[ATTR_NO]]
+ __builtin_no_inline((s.*mptr)());
+}
+
+// CHECK: attributes [[ATTR_ALWAYS]] = { alwaysinline }
+// CHECK: attributes [[ATTR_NO]] = { noinline }
Index: lib/Sema/SemaChecking.cpp
===================================================================
--- lib/Sema/SemaChecking.cpp
+++ lib/Sema/SemaChecking.cpp
@@ -327,6 +327,61 @@
return false;
}
+static bool SemaBuiltinCallWithInline(Sema &S, CallExpr *BuiltinCall) {
+ if (checkArgCount(S, BuiltinCall, 1))
+ return true;
+
+ SourceLocation BuiltinLoc = BuiltinCall->getBeginLoc();
+ Expr *Builtin = BuiltinCall->getCallee()->IgnoreImpCasts();
+ Expr *Call = BuiltinCall->getArg(0);
+
+ const auto CallStmtClass = Call->getStmtClass();
+ if (CallStmtClass != Stmt::CallExprClass &&
+ CallStmtClass != Stmt::CXXMemberCallExprClass &&
+ CallStmtClass != Stmt::CXXOperatorCallExprClass) {
+ S.Diag(BuiltinLoc, diag::err_argument_to_inline_intrinsic_not_call)
+ << Builtin << Call->getSourceRange();
+ return true;
+ }
+
+ auto *CE = cast<CallExpr>(Call);
+ const Decl *TargetDecl = CE->getCalleeDecl();
+ if (const auto *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl))
+ if (FD->getBuiltinID()) {
+ S.Diag(BuiltinLoc, diag::err_argument_to_inline_intrinsic_builtin_call)
+ << Builtin << Call->getSourceRange();
+ return true;
+ }
+
+ if (CE->getCallee()->getType()->isBlockPointerType()) {
+ S.Diag(BuiltinLoc, diag::err_argument_to_inline_intrinsic_block_call)
+ << Builtin << Call->getSourceRange();
+ return true;
+ }
+
+ if (isa<CXXPseudoDestructorExpr>(CE->getCallee()->IgnoreParens())) {
+ S.Diag(BuiltinLoc, diag::err_argument_to_inline_intrinsic_pdotr_call)
+ << Builtin << Call->getSourceRange();
+ return true;
+ }
+
+ QualType ReturnTy = CE->getCallReturnType(S.Context);
+ QualType ArgTys[1] = {ReturnTy};
+ QualType BuiltinTy = S.Context.getFunctionType(
+ ReturnTy, ArgTys, FunctionProtoType::ExtProtoInfo());
+ QualType BuiltinPtrTy = S.Context.getPointerType(BuiltinTy);
+
+ Builtin =
+ S.ImpCastExprToType(Builtin, BuiltinPtrTy, CK_BuiltinFnToFnPtr).get();
+
+ BuiltinCall->setType(CE->getType());
+ BuiltinCall->setValueKind(CE->getValueKind());
+ BuiltinCall->setObjectKind(CE->getObjectKind());
+ BuiltinCall->setCallee(Builtin);
+
+ return false;
+}
+
static bool SemaBuiltinSEHScopeCheck(Sema &SemaRef, CallExpr *TheCall,
Scope::ScopeFlags NeededScopeFlags,
unsigned DiagID) {
@@ -1239,6 +1294,11 @@
if (SemaBuiltinCallWithStaticChain(*this, TheCall))
return ExprError();
break;
+ case Builtin::BI__builtin_always_inline:
+ case Builtin::BI__builtin_no_inline:
+ if (SemaBuiltinCallWithInline(*this, TheCall))
+ return ExprError();
+ break;
case Builtin::BI__exception_code:
case Builtin::BI_exception_code:
if (SemaBuiltinSEHScopeCheck(*this, TheCall, Scope::SEHExceptScope,
Index: lib/CodeGen/CodeGenTypes.h
===================================================================
--- lib/CodeGen/CodeGenTypes.h
+++ lib/CodeGen/CodeGenTypes.h
@@ -261,9 +261,11 @@
/// Free functions are functions that are compatible with an ordinary
/// C function pointer type.
const CGFunctionInfo &arrangeFunctionDeclaration(const FunctionDecl *FD);
- const CGFunctionInfo &arrangeFreeFunctionCall(const CallArgList &Args,
- const FunctionType *Ty,
- bool ChainCall);
+ const CGFunctionInfo &
+ arrangeFreeFunctionCall(const CallArgList &Args, const FunctionType *Ty,
+ bool ChainCall,
+ CGFunctionInfo::CallInlineKind InlineKind =
+ CGFunctionInfo::CallInlineKind::DefaultInline);
const CGFunctionInfo &arrangeFreeFunctionType(CanQual<FunctionProtoType> Ty,
const FunctionDecl *FD);
const CGFunctionInfo &arrangeFreeFunctionType(CanQual<FunctionNoProtoType> Ty);
@@ -309,10 +311,11 @@
unsigned ExtraSuffixArgs,
bool PassProtoArgs = true);
- const CGFunctionInfo &arrangeCXXMethodCall(const CallArgList &args,
- const FunctionProtoType *type,
- RequiredArgs required,
- unsigned numPrefixArgs);
+ const CGFunctionInfo &
+ arrangeCXXMethodCall(const CallArgList &args, const FunctionProtoType *type,
+ RequiredArgs required, unsigned numPrefixArgs,
+ CGFunctionInfo::CallInlineKind InlineKind =
+ CGFunctionInfo::CallInlineKind::DefaultInline);
const CGFunctionInfo &
arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD);
const CGFunctionInfo &arrangeMSCtorClosure(const CXXConstructorDecl *CD,
@@ -327,13 +330,13 @@
/// this.
///
/// \param argTypes - must all actually be canonical as params
- const CGFunctionInfo &arrangeLLVMFunctionInfo(CanQualType returnType,
- bool instanceMethod,
- bool chainCall,
- ArrayRef<CanQualType> argTypes,
- FunctionType::ExtInfo info,
- ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
- RequiredArgs args);
+ const CGFunctionInfo &arrangeLLVMFunctionInfo(
+ CanQualType returnType, bool instanceMethod, bool chainCall,
+ ArrayRef<CanQualType> argTypes, FunctionType::ExtInfo info,
+ ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
+ RequiredArgs args,
+ CGFunctionInfo::CallInlineKind InlineKind =
+ CGFunctionInfo::CallInlineKind::DefaultInline);
/// Compute a new LLVM record layout object for the given record.
CGRecordLayout *ComputeRecordLayout(const RecordDecl *D,
Index: lib/CodeGen/CodeGenFunction.h
===================================================================
--- lib/CodeGen/CodeGenFunction.h
+++ lib/CodeGen/CodeGenFunction.h
@@ -3565,15 +3565,21 @@
/// LLVM arguments and the types they were derived from.
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
ReturnValueSlot ReturnValue, const CallArgList &Args,
- llvm::Instruction **callOrInvoke, SourceLocation Loc);
+ llvm::Instruction **callOrInvoke, SourceLocation Loc,
+ CGFunctionInfo::CallInlineKind InlineKind =
+ CGFunctionInfo::CallInlineKind::DefaultInline);
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
ReturnValueSlot ReturnValue, const CallArgList &Args,
- llvm::Instruction **callOrInvoke = nullptr) {
+ llvm::Instruction **callOrInvoke = nullptr,
+ CGFunctionInfo::CallInlineKind InlineKind =
+ CGFunctionInfo::CallInlineKind::DefaultInline) {
return EmitCall(CallInfo, Callee, ReturnValue, Args, callOrInvoke,
- SourceLocation());
+ SourceLocation(), InlineKind);
}
RValue EmitCall(QualType FnType, const CGCallee &Callee, const CallExpr *E,
- ReturnValueSlot ReturnValue, llvm::Value *Chain = nullptr);
+ ReturnValueSlot ReturnValue, llvm::Value *Chain = nullptr,
+ CGFunctionInfo::CallInlineKind InlineKind =
+ CGFunctionInfo::CallInlineKind::DefaultInline);
RValue EmitCallExpr(const CallExpr *E,
ReturnValueSlot ReturnValue = ReturnValueSlot());
RValue EmitSimpleCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue);
@@ -3637,38 +3643,44 @@
void callCStructCopyAssignmentOperator(LValue Dst, LValue Src);
void callCStructMoveAssignmentOperator(LValue Dst, LValue Src);
- RValue
- EmitCXXMemberOrOperatorCall(const CXXMethodDecl *Method,
- const CGCallee &Callee,
- ReturnValueSlot ReturnValue, llvm::Value *This,
- llvm::Value *ImplicitParam,
- QualType ImplicitParamTy, const CallExpr *E,
- CallArgList *RtlArgs);
+ RValue EmitCXXMemberOrOperatorCall(
+ const CXXMethodDecl *Method, const CGCallee &Callee,
+ ReturnValueSlot ReturnValue, llvm::Value *This,
+ llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *E,
+ CallArgList *RtlArgs,
+ CGFunctionInfo::CallInlineKind InlineKind =
+ CGFunctionInfo::CallInlineKind::DefaultInline);
RValue EmitCXXDestructorCall(const CXXDestructorDecl *DD,
const CGCallee &Callee,
llvm::Value *This, llvm::Value *ImplicitParam,
QualType ImplicitParamTy, const CallExpr *E,
StructorType Type);
- RValue EmitCXXMemberCallExpr(const CXXMemberCallExpr *E,
- ReturnValueSlot ReturnValue);
- RValue EmitCXXMemberOrOperatorMemberCallExpr(const CallExpr *CE,
- const CXXMethodDecl *MD,
- ReturnValueSlot ReturnValue,
- bool HasQualifier,
- NestedNameSpecifier *Qualifier,
- bool IsArrow, const Expr *Base);
+ RValue
+ EmitCXXMemberCallExpr(const CXXMemberCallExpr *E, ReturnValueSlot ReturnValue,
+ CGFunctionInfo::CallInlineKind InlineKind =
+ CGFunctionInfo::CallInlineKind::DefaultInline);
+ RValue EmitCXXMemberOrOperatorMemberCallExpr(
+ const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue,
+ bool HasQualifier, NestedNameSpecifier *Qualifier, bool IsArrow,
+ const Expr *Base,
+ CGFunctionInfo::CallInlineKind InlineKind =
+ CGFunctionInfo::CallInlineKind::DefaultInline);
// Compute the object pointer.
Address EmitCXXMemberDataPointerAddress(const Expr *E, Address base,
llvm::Value *memberPtr,
const MemberPointerType *memberPtrType,
LValueBaseInfo *BaseInfo = nullptr,
TBAAAccessInfo *TBAAInfo = nullptr);
- RValue EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
- ReturnValueSlot ReturnValue);
-
- RValue EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
- const CXXMethodDecl *MD,
- ReturnValueSlot ReturnValue);
+ RValue EmitCXXMemberPointerCallExpr(
+ const CXXMemberCallExpr *E, ReturnValueSlot ReturnValue,
+ CGFunctionInfo::CallInlineKind InlineKind =
+ CGFunctionInfo::CallInlineKind::DefaultInline);
+
+ RValue EmitCXXOperatorMemberCallExpr(
+ const CXXOperatorCallExpr *E, const CXXMethodDecl *MD,
+ ReturnValueSlot ReturnValue,
+ CGFunctionInfo::CallInlineKind InlineKind =
+ CGFunctionInfo::CallInlineKind::DefaultInline);
RValue EmitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *E);
RValue EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
Index: lib/CodeGen/CGExprCXX.cpp
===================================================================
--- lib/CodeGen/CGExprCXX.cpp
+++ lib/CodeGen/CGExprCXX.cpp
@@ -80,15 +80,15 @@
RValue CodeGenFunction::EmitCXXMemberOrOperatorCall(
const CXXMethodDecl *MD, const CGCallee &Callee,
- ReturnValueSlot ReturnValue,
- llvm::Value *This, llvm::Value *ImplicitParam, QualType ImplicitParamTy,
- const CallExpr *CE, CallArgList *RtlArgs) {
+ ReturnValueSlot ReturnValue, llvm::Value *This, llvm::Value *ImplicitParam,
+ QualType ImplicitParamTy, const CallExpr *CE, CallArgList *RtlArgs,
+ CGFunctionInfo::CallInlineKind InlineKind) {
const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
CallArgList Args;
MemberCallInfo CallInfo = commonEmitCXXMemberOrOperatorCall(
*this, MD, This, ImplicitParam, ImplicitParamTy, CE, Args, RtlArgs);
auto &FnInfo = CGM.getTypes().arrangeCXXMethodCall(
- Args, FPT, CallInfo.ReqArgs, CallInfo.PrefixSize);
+ Args, FPT, CallInfo.ReqArgs, CallInfo.PrefixSize, InlineKind);
return EmitCall(FnInfo, Callee, ReturnValue, Args, nullptr,
CE ? CE->getExprLoc() : SourceLocation());
}
@@ -165,36 +165,39 @@
// Note: This function also emit constructor calls to support a MSVC
// extensions allowing explicit constructor function call.
-RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
- ReturnValueSlot ReturnValue) {
+RValue CodeGenFunction::EmitCXXMemberCallExpr(
+ const CXXMemberCallExpr *CE, ReturnValueSlot ReturnValue,
+ CGFunctionInfo::CallInlineKind InlineKind) {
const Expr *callee = CE->getCallee()->IgnoreParens();
- if (isa<BinaryOperator>(callee))
- return EmitCXXMemberPointerCallExpr(CE, ReturnValue);
+ if (isa<BinaryOperator>(callee)) {
+ llvm::errs() << "lolol\n";
+ return EmitCXXMemberPointerCallExpr(CE, ReturnValue, InlineKind);
+ }
const MemberExpr *ME = cast<MemberExpr>(callee);
const CXXMethodDecl *MD = cast<CXXMethodDecl>(ME->getMemberDecl());
if (MD->isStatic()) {
// The method is static, emit it as we would a regular call.
CGCallee callee = CGCallee::forDirect(CGM.GetAddrOfFunction(MD), MD);
return EmitCall(getContext().getPointerType(MD->getType()), callee, CE,
- ReturnValue);
+ ReturnValue, /*Chain = */ nullptr, InlineKind);
}
bool HasQualifier = ME->hasQualifier();
NestedNameSpecifier *Qualifier = HasQualifier ? ME->getQualifier() : nullptr;
bool IsArrow = ME->isArrow();
const Expr *Base = ME->getBase();
return EmitCXXMemberOrOperatorMemberCallExpr(
- CE, MD, ReturnValue, HasQualifier, Qualifier, IsArrow, Base);
+ CE, MD, ReturnValue, HasQualifier, Qualifier, IsArrow, Base, InlineKind);
}
RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr(
const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue,
bool HasQualifier, NestedNameSpecifier *Qualifier, bool IsArrow,
- const Expr *Base) {
+ const Expr *Base, CGFunctionInfo::CallInlineKind InlineKind) {
assert(isa<CXXMemberCallExpr>(CE) || isa<CXXOperatorCallExpr>(CE));
// Compute the object pointer.
@@ -406,12 +409,12 @@
return EmitCXXMemberOrOperatorCall(
CalleeDecl, Callee, ReturnValue, This.getPointer(),
- /*ImplicitParam=*/nullptr, QualType(), CE, RtlArgs);
+ /*ImplicitParam=*/nullptr, QualType(), CE, RtlArgs, InlineKind);
}
-RValue
-CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
- ReturnValueSlot ReturnValue) {
+RValue CodeGenFunction::EmitCXXMemberPointerCallExpr(
+ const CXXMemberCallExpr *E, ReturnValueSlot ReturnValue,
+ CGFunctionInfo::CallInlineKind InlineKind) {
const BinaryOperator *BO =
cast<BinaryOperator>(E->getCallee()->IgnoreParens());
const Expr *BaseExpr = BO->getLHS();
@@ -459,18 +462,18 @@
EmitCallArgs(Args, FPT, E->arguments());
return EmitCall(CGM.getTypes().arrangeCXXMethodCall(Args, FPT, required,
/*PrefixSize=*/0),
- Callee, ReturnValue, Args, nullptr, E->getExprLoc());
+ Callee, ReturnValue, Args, nullptr, E->getExprLoc(),
+ InlineKind);
}
-RValue
-CodeGenFunction::EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
- const CXXMethodDecl *MD,
- ReturnValueSlot ReturnValue) {
+RValue CodeGenFunction::EmitCXXOperatorMemberCallExpr(
+ const CXXOperatorCallExpr *E, const CXXMethodDecl *MD,
+ ReturnValueSlot ReturnValue, CGFunctionInfo::CallInlineKind InlineKind) {
assert(MD->isInstance() &&
"Trying to emit a member call expr on a static method!");
return EmitCXXMemberOrOperatorMemberCallExpr(
E, MD, ReturnValue, /*HasQualifier=*/false, /*Qualifier=*/nullptr,
- /*IsArrow=*/false, E->getArg(0));
+ /*IsArrow=*/false, E->getArg(0), InlineKind);
}
RValue CodeGenFunction::EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
Index: lib/CodeGen/CGExpr.cpp
===================================================================
--- lib/CodeGen/CGExpr.cpp
+++ lib/CodeGen/CGExpr.cpp
@@ -4585,9 +4585,11 @@
AlignmentSource::Decl);
}
-RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee,
- const CallExpr *E, ReturnValueSlot ReturnValue,
- llvm::Value *Chain) {
+RValue CodeGenFunction::EmitCall(QualType CalleeType,
+ const CGCallee &OrigCallee, const CallExpr *E,
+ ReturnValueSlot ReturnValue,
+ llvm::Value *Chain,
+ CGFunctionInfo::CallInlineKind InlineKind) {
// Get the actual function type. The callee type will always be a pointer to
// function type or a block pointer type.
assert(CalleeType->isFunctionPointerType() &&
@@ -4732,8 +4734,13 @@
EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), E->arguments(),
E->getDirectCallee(), /*ParamsToSkip*/ 0, Order);
- const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
- Args, FnType, /*isChainCall=*/Chain);
+ const CGFunctionInfo *FnInfo = &CGM.getTypes().arrangeFreeFunctionCall(
+ Args, FnType, /*isChainCall=*/Chain, InlineKind);
+
+ // CGFunctionInfo *FnInfo = new
+ // CGFunctionInfo(CGM.getTypes().arrangeFreeFunctionCall(
+ // Args, FnType, /*isChainCall=*/Chain));
+ // FnInfo->setInlineKind(InlineKind);
// C99 6.5.2.2p6:
// If the expression that denotes the called function has a type
@@ -4756,15 +4763,15 @@
// Chain calls use this same code path to add the invisible chain parameter
// to the function type.
if (isa<FunctionNoProtoType>(FnType) || Chain) {
- llvm::Type *CalleeTy = getTypes().GetFunctionType(FnInfo);
+ llvm::Type *CalleeTy = getTypes().GetFunctionType(*FnInfo);
CalleeTy = CalleeTy->getPointerTo();
llvm::Value *CalleePtr = Callee.getFunctionPointer();
CalleePtr = Builder.CreateBitCast(CalleePtr, CalleeTy, "callee.knr.cast");
Callee.setFunctionPointer(CalleePtr);
}
- return EmitCall(FnInfo, Callee, ReturnValue, Args, nullptr, E->getExprLoc());
+ return EmitCall(*FnInfo, Callee, ReturnValue, Args, nullptr, E->getExprLoc());
}
LValue CodeGenFunction::
Index: lib/CodeGen/CGCall.cpp
===================================================================
--- lib/CodeGen/CGCall.cpp
+++ lib/CodeGen/CGCall.cpp
@@ -559,12 +559,11 @@
/// Arrange a call as unto a free function, except possibly with an
/// additional number of formal parameters considered required.
static const CGFunctionInfo &
-arrangeFreeFunctionLikeCall(CodeGenTypes &CGT,
- CodeGenModule &CGM,
- const CallArgList &args,
- const FunctionType *fnType,
- unsigned numExtraRequiredArgs,
- bool chainCall) {
+arrangeFreeFunctionLikeCall(CodeGenTypes &CGT, CodeGenModule &CGM,
+ const CallArgList &args, const FunctionType *fnType,
+ unsigned numExtraRequiredArgs, bool chainCall,
+ CGFunctionInfo::CallInlineKind InlineKind =
+ CGFunctionInfo::CallInlineKind::DefaultInline) {
assert(args.size() >= numExtraRequiredArgs);
llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
@@ -599,19 +598,18 @@
return CGT.arrangeLLVMFunctionInfo(GetReturnType(fnType->getReturnType()),
/*instanceMethod=*/false, chainCall,
argTypes, fnType->getExtInfo(), paramInfos,
- required);
+ required, InlineKind);
}
/// Figure out the rules for calling a function with the given formal
/// type using the given arguments. The arguments are necessary
/// because the function might be unprototyped, in which case it's
/// target-dependent in crazy ways.
-const CGFunctionInfo &
-CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args,
- const FunctionType *fnType,
- bool chainCall) {
+const CGFunctionInfo &CodeGenTypes::arrangeFreeFunctionCall(
+ const CallArgList &args, const FunctionType *fnType, bool chainCall,
+ CGFunctionInfo::CallInlineKind InlineKind) {
return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType,
- chainCall ? 1 : 0, chainCall);
+ chainCall ? 1 : 0, chainCall, InlineKind);
}
/// A block function is essentially a free function with an
@@ -671,11 +669,10 @@
///
/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
/// does not count `this`.
-const CGFunctionInfo &
-CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args,
- const FunctionProtoType *proto,
- RequiredArgs required,
- unsigned numPrefixArgs) {
+const CGFunctionInfo &CodeGenTypes::arrangeCXXMethodCall(
+ const CallArgList &args, const FunctionProtoType *proto,
+ RequiredArgs required, unsigned numPrefixArgs,
+ CGFunctionInfo::CallInlineKind InlineKind) {
assert(numPrefixArgs + 1 <= args.size() &&
"Emitting a call with less args than the required prefix?");
// Add one to account for `this`. It's a bit awkward here, but we don't count
@@ -689,7 +686,7 @@
FunctionType::ExtInfo info = proto->getExtInfo();
return arrangeLLVMFunctionInfo(
GetReturnType(proto->getReturnType()), /*instanceMethod=*/true,
- /*chainCall=*/false, argTypes, info, paramInfos, required);
+ /*chainCall=*/false, argTypes, info, paramInfos, required, InlineKind);
}
const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
@@ -733,21 +730,18 @@
/// Arrange the argument and result information for an abstract value
/// of a given function type. This is the method which all of the
/// above functions ultimately defer to.
-const CGFunctionInfo &
-CodeGenTypes::arrangeLLVMFunctionInfo(CanQualType resultType,
- bool instanceMethod,
- bool chainCall,
- ArrayRef<CanQualType> argTypes,
- FunctionType::ExtInfo info,
- ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
- RequiredArgs required) {
+const CGFunctionInfo &CodeGenTypes::arrangeLLVMFunctionInfo(
+ CanQualType resultType, bool instanceMethod, bool chainCall,
+ ArrayRef<CanQualType> argTypes, FunctionType::ExtInfo info,
+ ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
+ RequiredArgs required, CGFunctionInfo::CallInlineKind InlineKind) {
assert(std::all_of(argTypes.begin(), argTypes.end(),
[](CanQualType T) { return T.isCanonicalAsParam(); }));
// Lookup or create unique function info.
llvm::FoldingSetNodeID ID;
CGFunctionInfo::Profile(ID, instanceMethod, chainCall, info, paramInfos,
- required, resultType, argTypes);
+ required, resultType, argTypes, InlineKind);
void *insertPos = nullptr;
CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
@@ -757,8 +751,8 @@
unsigned CC = ClangCallConvToLLVMCallConv(info.getCC());
// Construct the function info. We co-allocate the ArgInfos.
- FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info,
- paramInfos, resultType, argTypes, required);
+ FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info, paramInfos,
+ resultType, argTypes, required, InlineKind);
FunctionInfos.InsertNode(FI, insertPos);
bool inserted = FunctionsBeingProcessed.insert(FI).second;
@@ -793,14 +787,12 @@
return *FI;
}
-CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC,
- bool instanceMethod,
- bool chainCall,
- const FunctionType::ExtInfo &info,
- ArrayRef<ExtParameterInfo> paramInfos,
- CanQualType resultType,
- ArrayRef<CanQualType> argTypes,
- RequiredArgs required) {
+CGFunctionInfo *
+CGFunctionInfo::create(unsigned llvmCC, bool instanceMethod, bool chainCall,
+ const FunctionType::ExtInfo &info,
+ ArrayRef<ExtParameterInfo> paramInfos,
+ CanQualType resultType, ArrayRef<CanQualType> argTypes,
+ RequiredArgs required, CallInlineKind inlineKind) {
assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
void *buffer =
@@ -813,6 +805,7 @@
FI->ASTCallingConvention = info.getCC();
FI->InstanceMethod = instanceMethod;
FI->ChainCall = chainCall;
+ FI->InlineCall = static_cast<unsigned>(inlineKind);
FI->NoReturn = info.getNoReturn();
FI->ReturnsRetained = info.getProducesResult();
FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
@@ -2005,6 +1998,11 @@
llvm::AttributeSet::get(getLLVMContext(), Attrs);
}
+ if (FI.getInlineCallKind() == CGFunctionInfo::CallInlineKind::AlwaysInline)
+ FuncAttrs.addAttribute(llvm::Attribute::AlwaysInline);
+ else if (FI.getInlineCallKind() == CGFunctionInfo::CallInlineKind::NoInline)
+ FuncAttrs.addAttribute(llvm::Attribute::NoInline);
+
unsigned ArgNo = 0;
for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
E = FI.arg_end();
@@ -3784,7 +3782,8 @@
ReturnValueSlot ReturnValue,
const CallArgList &CallArgs,
llvm::Instruction **callOrInvoke,
- SourceLocation Loc) {
+ SourceLocation Loc,
+ CGFunctionInfo::CallInlineKind InlineKind) {
// FIXME: We no longer need the types from CallArgs; lift up and simplify.
assert(Callee.isOrdinary() || Callee.isVirtual());
@@ -4255,6 +4254,13 @@
llvm::Attribute::AlwaysInline);
}
+ if (InlineKind != CGFunctionInfo::CallInlineKind::DefaultInline)
+ Attrs = Attrs.addAttribute(
+ getLLVMContext(), llvm::AttributeList::FunctionIndex,
+ InlineKind == CGFunctionInfo::CallInlineKind::NoInline
+ ? llvm::Attribute::NoInline
+ : llvm::Attribute::AlwaysInline);
+
// Disable inlining inside SEH __try blocks.
if (isSEHTryScope()) {
Attrs =
Index: lib/CodeGen/CGBuiltin.cpp
===================================================================
--- lib/CodeGen/CGBuiltin.cpp
+++ lib/CodeGen/CGBuiltin.cpp
@@ -2993,6 +2993,30 @@
EmitCallee(Call->getCallee()), Call, ReturnValue,
EmitScalarExpr(Chain));
}
+
+ case Builtin::BI__builtin_no_inline:
+ case Builtin::BI__builtin_always_inline: {
+ CGFunctionInfo::CallInlineKind CIK =
+ BuiltinID == Builtin::BI__builtin_no_inline
+ ? CGFunctionInfo::CallInlineKind::NoInline
+ : CGFunctionInfo::CallInlineKind::AlwaysInline;
+
+ auto *Arg = E->getArg(0);
+ if (const CXXMemberCallExpr *MemberCall = dyn_cast<CXXMemberCallExpr>(Arg))
+ return EmitCXXMemberCallExpr(MemberCall, ReturnValue, CIK);
+
+ if (const CXXOperatorCallExpr *OperatorCall =
+ dyn_cast<CXXOperatorCallExpr>(Arg))
+ if (const CXXMethodDecl *MD =
+ dyn_cast_or_null<CXXMethodDecl>(OperatorCall->getCalleeDecl()))
+ return EmitCXXOperatorMemberCallExpr(OperatorCall, MD, ReturnValue,
+ CIK);
+
+ const CallExpr *Call = cast<CallExpr>(Arg);
+ return EmitCall(Call->getCallee()->getType(), EmitCallee(Call->getCallee()),
+ Call, ReturnValue, nullptr, CIK);
+ }
+
case Builtin::BI_InterlockedExchange8:
case Builtin::BI_InterlockedExchange16:
case Builtin::BI_InterlockedExchange:
Index: include/clang/CodeGen/CGFunctionInfo.h
===================================================================
--- include/clang/CodeGen/CGFunctionInfo.h
+++ include/clang/CodeGen/CGFunctionInfo.h
@@ -514,6 +514,8 @@
/// Whether this is a chain call.
unsigned ChainCall : 1;
+ unsigned InlineCall : 2;
+
/// Whether this function is noreturn.
unsigned NoReturn : 1;
@@ -557,14 +559,17 @@
CGFunctionInfo() : Required(RequiredArgs::All) {}
public:
- static CGFunctionInfo *create(unsigned llvmCC,
- bool instanceMethod,
- bool chainCall,
- const FunctionType::ExtInfo &extInfo,
- ArrayRef<ExtParameterInfo> paramInfos,
- CanQualType resultType,
- ArrayRef<CanQualType> argTypes,
- RequiredArgs required);
+ enum class CallInlineKind : unsigned char {
+ DefaultInline = 0,
+ NoInline = 1,
+ AlwaysInline = 2
+ };
+ static CGFunctionInfo *
+ create(unsigned llvmCC, bool instanceMethod, bool chainCall,
+ const FunctionType::ExtInfo &extInfo,
+ ArrayRef<ExtParameterInfo> paramInfos, CanQualType resultType,
+ ArrayRef<CanQualType> argTypes, RequiredArgs required,
+ CallInlineKind inlineKind = CallInlineKind::DefaultInline);
void operator delete(void *p) { ::operator delete(p); }
// Friending class TrailingObjects is apparently not good enough for MSVC,
@@ -604,6 +609,9 @@
bool isInstanceMethod() const { return InstanceMethod; }
bool isChainCall() const { return ChainCall; }
+ CallInlineKind getInlineCallKind() const {
+ return static_cast<CallInlineKind>(InlineCall);
+ }
bool isNoReturn() const { return NoReturn; }
@@ -672,11 +680,15 @@
ArgStruct = Ty;
ArgStructAlign = Align.getQuantity();
}
+ void setInlineKind(CallInlineKind InlineKind) {
+ InlineCall = static_cast<unsigned>(InlineKind);
+ }
void Profile(llvm::FoldingSetNodeID &ID) {
ID.AddInteger(getASTCallingConvention());
ID.AddBoolean(InstanceMethod);
ID.AddBoolean(ChainCall);
+ ID.AddInteger(InlineCall);
ID.AddBoolean(NoReturn);
ID.AddBoolean(ReturnsRetained);
ID.AddBoolean(NoCallerSavedRegs);
@@ -693,17 +705,16 @@
for (const auto &I : arguments())
I.type.Profile(ID);
}
- static void Profile(llvm::FoldingSetNodeID &ID,
- bool InstanceMethod,
- bool ChainCall,
- const FunctionType::ExtInfo &info,
- ArrayRef<ExtParameterInfo> paramInfos,
- RequiredArgs required,
- CanQualType resultType,
- ArrayRef<CanQualType> argTypes) {
+ static void
+ Profile(llvm::FoldingSetNodeID &ID, bool InstanceMethod, bool ChainCall,
+ const FunctionType::ExtInfo &info,
+ ArrayRef<ExtParameterInfo> paramInfos, RequiredArgs required,
+ CanQualType resultType, ArrayRef<CanQualType> argTypes,
+ CallInlineKind inlineKind = CallInlineKind::DefaultInline) {
ID.AddInteger(info.getCC());
ID.AddBoolean(InstanceMethod);
ID.AddBoolean(ChainCall);
+ ID.AddInteger(static_cast<unsigned>(inlineKind));
ID.AddBoolean(info.getNoReturn());
ID.AddBoolean(info.getProducesResult());
ID.AddBoolean(info.getNoCallerSavedRegs());
Index: include/clang/Basic/DiagnosticSemaKinds.td
===================================================================
--- include/clang/Basic/DiagnosticSemaKinds.td
+++ include/clang/Basic/DiagnosticSemaKinds.td
@@ -8230,6 +8230,15 @@
def err_second_argument_to_cwsc_not_pointer : Error<
"second argument to __builtin_call_with_static_chain must be of pointer type">;
+def err_argument_to_inline_intrinsic_not_call : Error<
+ "argument to %0 must be a function, member function, or operator call">;
+def err_argument_to_inline_intrinsic_builtin_call : Error<
+ "argument to %0 must not be a builtin call">;
+def err_argument_to_inline_intrinsic_pdotr_call : Error<
+ "argument to %0 must not be a pseudo-destructor call">;
+def err_argument_to_inline_intrinsic_block_call : Error<
+ "argument to %0 must not be a block call">;
+
def err_vector_incorrect_num_initializers : Error<
"%select{too many|too few}0 elements in vector initialization (expected %1 elements, have %2)">;
def err_altivec_empty_initializer : Error<"expected initializer">;
Index: include/clang/Basic/Builtins.def
===================================================================
--- include/clang/Basic/Builtins.def
+++ include/clang/Basic/Builtins.def
@@ -544,6 +544,10 @@
BUILTIN(__builtin_alloca_with_align, "v*zIz", "Fn")
BUILTIN(__builtin_call_with_static_chain, "v.", "nt")
+// Clang builtins (not available in GCC).
+BUILTIN(__builtin_always_inline, "v.", "nt")
+BUILTIN(__builtin_no_inline, "v.", "nt")
+
// "Overloaded" Atomic operator builtins. These are overloaded to support data
// types of i8, i16, i32, i64, and i128. The front-end sees calls to the
// non-suffixed version of these (which has a bogus type) and transforms them to
_______________________________________________
cfe-commits mailing list
[email protected]
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
