https://gcc.gnu.org/g:28fbf4db2cb28be3df399d4eb2ac61ab5ea0d465
commit 28fbf4db2cb28be3df399d4eb2ac61ab5ea0d465 Author: Josef Melcr <melcr...@fit.cvut.cz> Date: Tue Feb 4 19:33:02 2025 +0100 omp-cp: Refactor code, improve callback handler, add multiple callbacks support gcc/ChangeLog: * builtin-attrs.def (0): Add new list with 0. (DEF_CALLBACK_ATTRIBUTE_NOTHROW): Remove macro. (GOMP): Add callback for GOMP functions. (ATTR_NOTHROW_CALLBACK_GOMP_LIST): Remove list. (DEF_CALLBACK_ATTRIBUTE): New macro for callback attribute creation. (OACC): Callback attribute for OACC functions. (ATTR_CALLBACK_GOMP_LIST): Add list for for GOMP functions with callbacks as their first argument and data as their second argument. (ATTR_CALLBACK_GOMP_TASK_HELPER_LIST): Helper list for GOMP_task, needed because it has 2 callbacks. (ATTR_CALLBACK_GOMP_TASK_LIST): Attribute list for GOMP_task (ATTR_CALLBACK_OACC_LIST): Attribute list for OACC functions with callbacks. * cgraph.cc (cgraph_edge::set_call_stmt): Refactor callback section. (symbol_table::create_edge): Add check for callback to creation assert. (cgraph_edge::get_callback_parent_edge): Refactor function. (cgraph_edge::redirect_call_stmt_to_callee): Refactor callback redirection. (cgraph_edge::maybe_hot_p): Make callback edge hot when it's parent is hot, not working for now. (cgraph_node::verify_node): Add verifiers for callbacks. * cgraph.h: Add docs for callback utils. * doc/extend.texi: Add callback attr docs. * ipa-fnsummary.cc (analyze_function_body): Refactor. (compute_fn_summary): Refactor. * ipa-inline.cc (can_inline_edge_p): Refactor. * ipa-prop.cc (calc_callback_args_idx): Remove function. (init_callback_edge_summary): New function. (ipa_compute_jump_functions_for_edge): Refactor, add multiple callback support. * ipa-prop.h (struct cb_arg_info): Remove struct. (class ipa_edge_args): Remove cb_arg_info uses. * omp-builtins.def (BUILT_IN_GOACC_PARALLEL): Use new attr list. (BUILT_IN_GOMP_PARALLEL_LOOP_STATIC): Likewise. (BUILT_IN_GOMP_PARALLEL_LOOP_GUIDED): Likewise. (BUILT_IN_GOMP_PARALLEL_LOOP_NONMONOTONIC_DYNAMIC): Likewise. (BUILT_IN_GOMP_PARALLEL_LOOP_NONMONOTONIC_RUNTIME): Likewise. (BUILT_IN_GOMP_PARALLEL): Likewise. (BUILT_IN_GOMP_TASK): Likewise. (BUILT_IN_GOMP_PARALLEL_SECTIONS): Likewise. (BUILT_IN_GOMP_TEAMS_REG): Likewise. * attr-callback.h: New file. gcc/c-family/ChangeLog: * c-attribs.cc (handle_callback_attribute): Move handler to attr_callback.h, add more checks. Signed-off-by: Josef Melcr <melcr...@fit.cvut.cz> Diff: --- gcc/attr-callback.h | 273 ++++++++++++++++++++++++++++++++++++++++++++++ gcc/builtin-attrs.def | 25 +++-- gcc/c-family/c-attribs.cc | 35 +----- gcc/cgraph.cc | 121 ++++++++++++++------ gcc/cgraph.h | 56 +++++----- gcc/doc/extend.texi | 37 +++++++ gcc/ipa-fnsummary.cc | 26 +++-- gcc/ipa-inline.cc | 6 +- gcc/ipa-prop.cc | 113 +++++++++---------- gcc/ipa-prop.h | 11 +- gcc/omp-builtins.def | 28 ++--- 11 files changed, 535 insertions(+), 196 deletions(-) diff --git a/gcc/attr-callback.h b/gcc/attr-callback.h new file mode 100644 index 000000000000..ea9c9cbbc780 --- /dev/null +++ b/gcc/attr-callback.h @@ -0,0 +1,273 @@ +/* Callback attribute handling + Copyright (C) 2025 Free Software Foundation, Inc. + Contributed by Josef Melcr <melcr...@fit.cvut.cz> + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GCC; see the file COPYING3. If not see + <http://www.gnu.org/licenses/>. */ + +#ifndef ATTR_CALLBACK_H +#define ATTR_CALLBACK_H +#include "attribs.h" +#include "system.h" +#include "tree.h" +#include "function.h" +#include "basic-block.h" +#include "coretypes.h" +#include "is-a.h" +#include "predict.h" +#include "internal-fn.h" +#include "tree-ssa-alias.h" +#include "gimple-expr.h" +#include "gimple.h" +#include "vec.h" + +enum callback_position +{ + /* Value used when an argument of a callback function + is unknown or when multiple values may be used. */ + CB_UNKNOWN_POS = 0 +}; + +/* Given an instance of callback attribute, return the 0-based + index of the called function in question. */ +inline int +callback_get_fn_index (tree cb_attr) +{ + tree args = TREE_VALUE (cb_attr); + int idx = TREE_INT_CST_LOW (TREE_VALUE (args)) - 1; + return idx; +} + +/* Given an instance of callback attribute, return the 0-base indices + of arguments passed to the callback. For a callback function taking + n parameters, returns a vector of n indices of their values in the parameter + list of it's caller. Indices with unknown positions will be filled with + an identity. */ +inline auto_vec<int> +callback_get_arg_mapping (tree decl) +{ + tree attr = lookup_attribute ("callback", DECL_ATTRIBUTES (decl)); + gcc_checking_assert (attr); + tree args = TREE_VALUE (attr); + auto_vec<int> res; + tree it; + + /* Skip over the first argument, which denotes + which argument is the called function. */ + for (it = TREE_CHAIN (args); it != NULL_TREE; it = TREE_CHAIN (it)) + { + int idx = TREE_INT_CST_LOW (TREE_VALUE (it)); + + /* CB_UNKNOWN_POS signifies an unknown argument, + replace it with identity for convenience */ + if (idx == CB_UNKNOWN_POS) + idx = res.length (); + /* arguments use 1-based indexing, so we have + to subtract 1 */ + else + idx -= 1; + + res.safe_push (idx); + } + + return res; +} + +/* Given a call statement of the parent, it's attribute list and + a decl of the callback, returns a 0-based index of the callback + function in the parameters of it's caller function. Arguments + are extracted from the call statement. If kernel_decl is a decl + of a clone, it's parent decl will be considered as well. */ +inline int +callback_fetch_fn_position (gcall *call, tree attr_list, tree kernel_decl) +{ + tree original_decl = DECL_ORIGIN (kernel_decl); + tree cb_attr = lookup_attribute ("callback", attr_list); + gcc_checking_assert (cb_attr); + int res = -1; + for (; cb_attr; cb_attr = lookup_attribute ("callback", TREE_CHAIN (cb_attr))) + { + int idx = callback_get_fn_index (cb_attr); + tree arg = gimple_call_arg (call, idx); + if (TREE_CODE (arg) == ADDR_EXPR) + { + tree pointee = TREE_OPERAND (arg, 0); + if (pointee != NULL_TREE + && (pointee == kernel_decl || pointee == original_decl)) + { + res = idx; + break; + } + } + } + gcc_checking_assert (res != -1); + return res; +} + +/* Returns the element at index idx in the list or NULL_TREE if + the list isn't long enough. NULL_TREE is used as the endpoint. */ +static tree +get_nth_list_elem (tree list, unsigned idx) +{ + tree res = NULL_TREE; + unsigned i = 0; + tree it; + for (it = list; it != NULL_TREE; it = TREE_CHAIN (it), i++) + { + if (i == idx) + { + res = TREE_VALUE (it); + break; + } + } + return res; +} + +/* Handle a "callback" attribute; arguments as in + struct attribute_spec.handler. */ +inline tree +handle_callback_attribute (tree *node, tree name, tree args, + int ARG_UNUSED (flags), bool *no_add_attrs) +{ + tree decl = *node; + if (TREE_CODE (decl) != FUNCTION_DECL) + { + error_at (DECL_SOURCE_LOCATION (decl), + "%qE attribute can only be used on functions", name); + *no_add_attrs = true; + } + + tree cb_fn_idx_node = TREE_VALUE (args); + if (TREE_CODE (cb_fn_idx_node) != INTEGER_CST) + { + error_at (DECL_SOURCE_LOCATION (decl), + "argument specifying callback function position is not an " + "integer constant"); + *no_add_attrs = true; + return NULL_TREE; + } + /* We have to use the function type for validation, as + DECL_ARGUMENTS returns NULL at this point. */ + unsigned callback_fn_idx = TREE_INT_CST_LOW (cb_fn_idx_node) - 1; + tree decl_type_args = TYPE_ARG_TYPES (TREE_TYPE (decl)); + unsigned decl_nargs = list_length (decl_type_args); + if (callback_fn_idx >= decl_nargs) + { + error_at (DECL_SOURCE_LOCATION (decl), + "callback function position out of range"); + *no_add_attrs = true; + return NULL_TREE; + } + + /* Search for the type of the callback function + in parameters of the original function. */ + tree cfn = get_nth_list_elem(decl_type_args, callback_fn_idx); + if (cfn == NULL_TREE) + { + error_at (DECL_SOURCE_LOCATION (decl), + "could not retrieve callback function from arguments"); + *no_add_attrs = true; + return NULL_TREE; + } + tree cfn_pointee_type = TREE_TYPE (cfn); + if (TREE_CODE (cfn) != POINTER_TYPE + || TREE_CODE (cfn_pointee_type) != FUNCTION_TYPE) + { + error_at (DECL_SOURCE_LOCATION (decl), + "argument no. %d is not an address of a function", + callback_fn_idx); + *no_add_attrs = true; + return NULL_TREE; + } + + tree it; + tree type_args = TYPE_ARG_TYPES (cfn_pointee_type); + /* Compare the length of the list of argument indices + and the real number of parameters the callback takes. */ + unsigned cfn_nargs = list_length (TREE_CHAIN (args)); + unsigned type_nargs = list_length (type_args); + for (it = type_args; it != NULL_TREE; it = TREE_CHAIN (it)) + if (it == void_list_node) + { + --type_nargs; + break; + } + if (cfn_nargs != type_nargs) + { + error_at (DECL_SOURCE_LOCATION (decl), + "argument number mismatch, %d expected, got %d", type_nargs, + cfn_nargs); + *no_add_attrs = true; + return NULL_TREE; + } + + unsigned curr = 0; + tree cfn_it; + /* Validate type compatibility of the arguments passed + from caller function to callback. "it" is used to step + through the parameters of the caller, "cfn_it" is + stepping through the parameters of the callback. */ + for (it = type_args, cfn_it = TREE_CHAIN (args); curr < type_nargs; + it = TREE_CHAIN (it), cfn_it = TREE_CHAIN (cfn_it), curr++) + { + if (TREE_CODE (TREE_VALUE (cfn_it)) != INTEGER_CST) + { + error_at (DECL_SOURCE_LOCATION (decl), + "argument no. %d is not an integer constant", curr + 1); + *no_add_attrs = true; + continue; + } + + unsigned arg_idx = TREE_INT_CST_LOW (TREE_VALUE (cfn_it)); + + /* No need to check for type compatibility, + if we don't know what we are passing. */ + if (arg_idx == CB_UNKNOWN_POS) + { + continue; + } + + arg_idx -= 1; + /* Report an error if the position is out of bounds, + but we can still check the rest of the arguments. */ + if (arg_idx >= decl_nargs) + { + error_at (DECL_SOURCE_LOCATION (decl), + "callback argument index %d is out of range", arg_idx + 1); + *no_add_attrs = true; + continue; + } + + tree arg_type = get_nth_list_elem (decl_type_args, arg_idx); + tree expected_type = TREE_VALUE (it); + /* Check the type of the value we are about to pass ("arg_type") + for compatibility with the actual type the callback function + expects ("expected_type"). */ + if (!types_compatible_p (expected_type, arg_type)) + { + error_at (DECL_SOURCE_LOCATION (decl), + "argument type at index %d is not compatible with callback " + "argument type at index %d", + curr, arg_idx); + *no_add_attrs = true; + continue; + } + } + + return NULL_TREE; +} + +#endif /* ATTR_CALLBACK_H */ diff --git a/gcc/builtin-attrs.def b/gcc/builtin-attrs.def index 8a0b3463adcb..0676a12177b6 100644 --- a/gcc/builtin-attrs.def +++ b/gcc/builtin-attrs.def @@ -75,6 +75,7 @@ DEF_ATTR_FOR_STRING (STRERRNOP, ".P") #define DEF_LIST_INT_INT(VALUE1, VALUE2) \ DEF_ATTR_TREE_LIST (ATTR_LIST_##VALUE1##_##VALUE2, ATTR_NULL, \ ATTR_##VALUE1, ATTR_LIST_##VALUE2) +DEF_LIST_INT_INT (0,2) DEF_LIST_INT_INT (1,0) DEF_LIST_INT_INT (1,2) DEF_LIST_INT_INT (1,3) @@ -389,13 +390,23 @@ DEF_FORMAT_ATTRIBUTE_NOTHROW(STRFMON,3,3_4) #undef DEF_FORMAT_ATTRIBUTE_BOTH /* Callback attr */ -#define DEF_CALLBACK_ATTRIBUTE_NOTHROW(TYPE, CA, VALUES) \ - DEF_ATTR_TREE_LIST (ATTR_CALLBACK_##TYPE##_NOTHROW_##VALUES, ATTR_CALLBACK,\ - ATTR_NOTHROW_NONNULL_##CA, ATTR_LIST_##VALUES) - -DEF_CALLBACK_ATTRIBUTE_NOTHROW(GOMP, 1, 2) -DEF_ATTR_TREE_LIST(ATTR_NOTHROW_CALLBACK_GOMP_LIST, ATTR_CALLBACK, ATTR_CALLBACK_GOMP_NOTHROW_2, ATTR_NOTHROW_LIST) -#undef DEF_CALLBACK_ATTRIBUTE_NOTHROW +#define DEF_CALLBACK_ATTRIBUTE(TYPE, CA, VALUES) \ + DEF_ATTR_TREE_LIST (ATTR_CALLBACK_##TYPE##_##CA##_##VALUES, ATTR_CALLBACK,\ + ATTR_##CA, ATTR_LIST_##VALUES) + +DEF_CALLBACK_ATTRIBUTE(GOMP, 1, 0) +DEF_CALLBACK_ATTRIBUTE(GOMP, 1, 2) +DEF_CALLBACK_ATTRIBUTE(OACC, 2, 4) +DEF_CALLBACK_ATTRIBUTE(GOMP, 3, 0_2) +DEF_ATTR_TREE_LIST(ATTR_CALLBACK_GOMP_LIST, ATTR_CALLBACK, + ATTR_CALLBACK_GOMP_1_2, ATTR_NOTHROW_LIST) +DEF_ATTR_TREE_LIST(ATTR_CALLBACK_GOMP_TASK_HELPER_LIST, ATTR_CALLBACK, + ATTR_CALLBACK_GOMP_1_0, ATTR_NOTHROW_LIST) +DEF_ATTR_TREE_LIST(ATTR_CALLBACK_GOMP_TASK_LIST, ATTR_CALLBACK, + ATTR_CALLBACK_GOMP_3_0_2, ATTR_CALLBACK_GOMP_TASK_HELPER_LIST) +DEF_ATTR_TREE_LIST(ATTR_CALLBACK_OACC_LIST, ATTR_CALLBACK, + ATTR_CALLBACK_OACC_2_4, ATTR_NOTHROW_LIST) +#undef DEF_CALLBACK_ATTRIBUTE /* Transactional memory variants of the above. */ diff --git a/gcc/c-family/c-attribs.cc b/gcc/c-family/c-attribs.cc index 6fb1f8d034d1..89c3b0ea9c5a 100644 --- a/gcc/c-family/c-attribs.cc +++ b/gcc/c-family/c-attribs.cc @@ -48,6 +48,7 @@ along with GCC; see the file COPYING3. If not see #include "gimplify.h" #include "tree-pretty-print.h" #include "gcc-rich-location.h" +#include "attr-callback.h" static tree handle_packed_attribute (tree *, tree, tree, int, bool *); static tree handle_nocommon_attribute (tree *, tree, tree, int, bool *); @@ -188,7 +189,6 @@ static tree handle_retain_attribute (tree *, tree, tree, int, bool *); static tree handle_fd_arg_attribute (tree *, tree, tree, int, bool *); static tree handle_flag_enum_attribute (tree *, tree, tree, int, bool *); static tree handle_null_terminated_string_arg_attribute (tree *, tree, tree, int, bool *); -static tree handle_callback_attribute (tree *, tree, tree, int, bool *); /* Helper to define attribute exclusions. */ #define ATTR_EXCL(name, function, type, variable) \ @@ -5193,39 +5193,6 @@ get_argument (tree fndecl, unsigned argno) return NULL_TREE; } -static tree -handle_callback_attribute (tree *node, tree name, tree args, - int ARG_UNUSED (flags), - bool ARG_UNUSED (*no_add_attrs)) -{ - tree decl = *node; - if (TREE_CODE (decl) != FUNCTION_DECL) - { - error_at (DECL_SOURCE_LOCATION (decl), - "%qE attribute can only be used on functions", name); - } - tree callback_fn_idx_node = args; - for (int i = 0; i < 3; - i++, callback_fn_idx_node = TREE_VALUE (callback_fn_idx_node)) - ; - int callback_fn_idx = TREE_INT_CST_LOW (callback_fn_idx_node) - 1; - if (callback_fn_idx < 0) - error_at (DECL_SOURCE_LOCATION (decl), - "callback function position out of range"); - // - // tree cfn = get_argument (decl, callback_fn_idx); - // if (cfn == NULL_TREE) - // error_at(DECL_SOURCE_LOCATION(decl), "couldnt retrieve callback function - // from arguments"); - // - // if (TREE_CODE(cfn) != ADDR_EXPR || TREE_CODE(TREE_OPERAND(cfn, 0)) != - // FUNCTION_DECL) - // error_at(DECL_SOURCE_LOCATION(decl), "argument no. %d is not a fndecl", - // callback_fn_idx); - // - return NULL_TREE; -} - /* Attempt to append attribute access specification ATTRSPEC, optionally described by the human-readable string ATTRSTR, for type T, to one in ATTRS. VBLIST is an optional list of bounds of variable length array diff --git a/gcc/cgraph.cc b/gcc/cgraph.cc index 484f2e3b45ff..eabba370aa1f 100644 --- a/gcc/cgraph.cc +++ b/gcc/cgraph.cc @@ -69,6 +69,7 @@ along with GCC; see the file COPYING3. If not see #include "tree-nested.h" #include "symtab-thunks.h" #include "symtab-clones.h" +#include "attr-callback.h" /* FIXME: Only for PROP_loops, but cgraph shouldn't have to know about this. */ #include "tree-pass.h" @@ -838,24 +839,22 @@ cgraph_edge::set_call_stmt (cgraph_edge *e, gcall *new_stmt, return e_indirect ? indirect : direct; } - if (update_speculative && (e->callback || e->has_callback) - /* If we are about to resolve the speculation by calling make_direct - below, do not bother going over all the speculative edges now. */ - ) + /* Callback edges also need their call stmts changed. + We can use the same flag as for speculative edges. */ + if (update_speculative && (e->callback || e->has_callback)) { - fprintf (stderr, "set_call_stmt callback\n"); - cgraph_edge *direct, *next; + cgraph_edge *current, *next; - direct = e; - - gcall *old_stmt = direct->call_stmt; - for (cgraph_edge *d = direct; d; d = next) + current = e; + gcall *old_stmt = current->call_stmt; + for (cgraph_edge *d = current; d; d = next) { next = d->next_callee; for (; next; next = next->next_callee) { - if ((next->callback || next->has_callback) - && old_stmt == next->call_stmt) + /* has_callback doesn't need to checked, as their + call statements wouldn't match */ + if (next->callback && old_stmt == next->call_stmt) break; } cgraph_edge *d2 = set_call_stmt (d, new_stmt, false); @@ -908,7 +907,7 @@ symbol_table::create_edge (cgraph_node *caller, cgraph_node *callee, construction of call stmt hashtable. */ cgraph_edge *e; gcc_checking_assert (!(e = caller->get_edge (call_stmt)) - || e->speculative || e->has_callback); + || e->speculative || e->has_callback || e->callback); gcc_assert (is_gimple_call (call_stmt)); } @@ -1162,6 +1161,17 @@ cgraph_edge::make_speculative (cgraph_node *n2, profile_count direct_count, return e2; } +/* Turn edge into a callback edge calling N2. Callback edges + never get turned into actual calls, they are just used + as clues and allow for optimizing functions which do not + have any callsites during compile time, e.g. functions + passed to standard library functions. + + The edge will be attached to the same call statement as + it's parent, which is the instance this method is called on. + + Return the resulting callback edge. */ + cgraph_edge * cgraph_edge::make_callback (cgraph_node *n2) { @@ -1184,20 +1194,23 @@ cgraph_edge::make_callback (cgraph_node *n2) return e2; } +/* Returns the parent edge of a callback edge on which + it is called on or NULL when no such edge can be found. + + An edge is taken to be a parent if it has it's has_callback + flag set and the edges share their call statements. */ + cgraph_edge * cgraph_edge::get_callback_parent_edge () { gcc_checking_assert (callback); cgraph_edge *e; - fprintf (stderr, "get parent: %s -> %s\n", caller->name (), callee->name ()); for (e = caller->callees; e; e = e->next_callee) { - fprintf (stderr, "get parent for loop: %s -> %s\n", caller->name (), - callee->name ()); if (e->has_callback && e->call_stmt == call_stmt) - return e; + break; } - gcc_unreachable (); + return e; } /* Speculative call consists of an indirect edge and one or more @@ -1554,22 +1567,18 @@ cgraph_edge::redirect_call_stmt_to_callee (cgraph_edge *e, } } - if (e->has_callback) - { - debug_gimple_stmt (e->call_stmt); - fprintf (stderr, "gimple pointer: %p\n", (void *) e->call_stmt); - } - if (e->indirect_unknown_callee || decl == e->callee->decl) return e->call_stmt; + /* When redirecting a callback edge, all we need to do is replace + the original address with the address of the function we are + redirecting to. */ if (e->callback) { - fprintf (stderr, "redirecting to %s\n", e->callee->name ()); - fprintf (stderr, "gimple pointer before: %p\n", (void *) e->call_stmt); - gimple_call_set_arg (e->call_stmt, 0, build_addr(e->callee->decl)); - debug_gimple_stmt (e->call_stmt); - fprintf (stderr, "gimple pointer after: %p\n", (void *) e->call_stmt); + int fn_idx + = callback_fetch_fn_position (e->call_stmt, DECL_ATTRIBUTES (decl), + e->callee->decl); + gimple_call_set_arg (e->call_stmt, fn_idx, build_addr (e->callee->decl)); return e->call_stmt; } @@ -3069,7 +3078,15 @@ cgraph_edge::cannot_lead_to_return_p (void) bool cgraph_edge::maybe_hot_p (void) { - if (callback) return true; + /* A callback edge is considered hot when it's parent edge + is hot*/ + if (callback) + { + cgraph_edge *parent = get_callback_parent_edge (); + bool ret = parent->maybe_hot_p (); + ret = true; + return ret; + } if (!maybe_hot_count_p (NULL, count.ipa ())) return false; if (caller->frequency == NODE_FREQUENCY_UNLIKELY_EXECUTED @@ -3943,7 +3960,10 @@ cgraph_node::verify_node (void) } if (!e->indirect_unknown_callee) { - if (!e->callback && e->verify_corresponds_to_fndecl (decl)) + /* Callback edges violate this assertion + in the same way speculative edges do. */ + if (!e->callback + && e->verify_corresponds_to_fndecl (decl)) { error ("edge points to wrong declaration:"); debug_tree (e->callee->decl); @@ -3985,6 +4005,45 @@ cgraph_node::verify_node (void) for (e = callees; e; e = e->next_callee) { + if (e->callback && e->has_callback) + { + error ("edge has both callback and has_callback set"); + error_found = true; + } + + if (e->callback) + { + if (!e->get_callback_parent_edge ()) + { + error ("callback edge %s->%s has no parent", + identifier_to_locale (e->caller->name ()), + identifier_to_locale (e->callee->name ())); + error_found = true; + } + } + + if (e->has_callback) + { + int ncallbacks = 0; + int nfound_edges = 0; + for (tree cb = lookup_attribute ("callback", DECL_ATTRIBUTES ( + e->callee->decl)); + cb; cb = lookup_attribute ("callback", TREE_CHAIN (cb)), + ncallbacks++) + ; + for (cgraph_edge *cbe = callees; cbe; cbe = cbe->next_callee) + if (cbe->callback && cbe->call_stmt == e->call_stmt) + nfound_edges++; + if (ncallbacks != nfound_edges) + { + error ("callback edge %s->%s child edge count mismach, " + "expected %d, found %d", + identifier_to_locale (e->caller->name ()), + identifier_to_locale (e->callee->name ()), ncallbacks, + nfound_edges); + } + } + if (!e->aux && !e->speculative && !e->callback && !e->has_callback) { error ("edge %s->%s has no corresponding call_stmt", diff --git a/gcc/cgraph.h b/gcc/cgraph.h index 089e12f6ec30..00d2285671ed 100644 --- a/gcc/cgraph.h +++ b/gcc/cgraph.h @@ -1735,11 +1735,17 @@ public: cgraph_edge *make_speculative (cgraph_node *n2, profile_count direct_count, unsigned int speculative_id = 0); - /* TODO DOCS */ + /* Turns edge into a callback edge, representing an indirect call to n2 + passed to a function by argument. Sets has_callback flag of the original + edge. Both edges are attached to the same call statement. Returns created + callback edge. */ cgraph_edge *make_callback (cgraph_node *n2); - /* TODO DOCS */ - cgraph_edge *get_callback_parent_edge(); + /* Returns the parent edge of a callback edge or NULL, if such edge + cannot be found. An edge is considered a parent, if it has it's + has_callback flag set and shares it's call statement with the edge + this method is caled on. */ + cgraph_edge *get_callback_parent_edge (); /* Speculative call consists of an indirect edge and one or more direct edge+ref pairs. Speculative will expand to the following sequence: @@ -1772,10 +1778,6 @@ public: target2. */ cgraph_edge *next_speculative_call_target () { - if (callback) - { - return NULL; - } cgraph_edge *e = this; gcc_checking_assert (speculative && callee); @@ -1790,29 +1792,16 @@ public: indirect call edge in the speculative call sequence. */ cgraph_edge *speculative_call_indirect_edge () { - gcc_checking_assert (speculative || callback); + gcc_checking_assert (speculative); if (!callee) return this; - - cgraph_edge * e2 = NULL; - for (e2 = caller->indirect_calls; - e2; e2 = e2->next_callee) - if (e2->speculative - && call_stmt == e2->call_stmt + + cgraph_edge *e2; + for (e2 = caller->indirect_calls; e2; e2 = e2->next_callee) + if (e2->speculative && call_stmt == e2->call_stmt && lto_stmt_uid == e2->lto_stmt_uid) - return e2; - - if (!e2 && callback) - { - for (e2 = caller->callees; e2; e2 = e2->next_callee) - { - if (e2->has_callback && call_stmt == e2->call_stmt) - { - return e2; - } - } - } - gcc_unreachable(); + break; + return e2; } /* When called on any edge in speculative call and when given any target @@ -1975,9 +1964,18 @@ public: Optimizers may later redirect direct call to clone, so 1) and 3) do not need to necessarily agree with destination. */ unsigned int speculative : 1; - /* TODO DOCS */ + /* Edges with CALLBACK flag represent indirect calls to functions passed + to their callers by argument. This is useful in cases, where the body + of these caller functions is not known, e. g. qsort in glibc or + GOMP_parallel in libgomp. These edges are never made into real calls, + but are used instead to optimize these callback functions and later replace + their addresses with their optimized versions. Edges with this flag set + share their call statement with their parent edge. */ unsigned int callback : 1; - /* TODO DOCS */ + /* Edges with this flag set have one or more child callabck edges. They share + their call statements with this edge. This flag represents the fact that + the callee of this edge takes a function and it's parameters by argument + and calls it at a later time. */ unsigned int has_callback : 1; /* Set to true when caller is a constructor or destructor of polymorphic type. */ diff --git a/gcc/doc/extend.texi b/gcc/doc/extend.texi index c55991df8f10..5769ef3991f8 100644 --- a/gcc/doc/extend.texi +++ b/gcc/doc/extend.texi @@ -2905,6 +2905,43 @@ declares that @code{my_alloc1} returns 16-byte aligned pointers and that @code{my_alloc2} returns a pointer whose value modulo 32 is equal to 8. +@cindex @code{callback} function attribute +@item callback (@var{function-pos}, @var{...}) +The @code{callback} attribute specifies that a function takes a pointer to +a callback function as one of it's parameters and passes it some of it's own +parameters. For example: + +@smallexample +void foo(void (*fn)(int*), int *data) __attribute__((callback(1, 2))); +@end smallexample + +where body of @code{foo} looks something like: + +@smallexample +void foo(void (*fn)(int*), int *data) +@{ + ... + fn(data); + ... +@} +@end smallexample + +This is particuarly useful for cases, where body of functions with callbacks +is unknown at compile-time. Using this attribute allows GCC to perform +optimizations on the callback function, namely constant propagation. +The parameter @var{function-pos} specifies the position of the pointer +to the callback function. All indices start from 1. This parameter should be +followed by @var{n} positions of arguments passed to the callback function +(where @var{n} is the number of arguments the callback function takes) in order +in which they are passed. Value 0 should be used in places where the position +for a given argument is unknown or the value is not passed through the caller. +When used with non-static C++ methods, all indices should start at 2, since the +first argument is implicit @code{this}. + +In the example above, function @code{foo} takes it's callback function as it's +first argument and passes it it's second argument, so the correct values of +parameters are 1 and 2. + @cindex @code{cold} function attribute @item cold The @code{cold} attribute on functions is used to inform the compiler that diff --git a/gcc/ipa-fnsummary.cc b/gcc/ipa-fnsummary.cc index 348bf2c17724..7a44ad1c5f2f 100644 --- a/gcc/ipa-fnsummary.cc +++ b/gcc/ipa-fnsummary.cc @@ -3080,7 +3080,7 @@ analyze_function_body (struct cgraph_node *node, bool early) es->call_stmt_time = this_time; es->loop_depth = bb_loop_depth (bb); edge_set_predicate (edge, &bb_predicate); - if (edge->speculative || edge->callback) + if (edge->speculative) { cgraph_edge *indirect = edge->speculative_call_indirect_edge (); @@ -3103,6 +3103,16 @@ analyze_function_body (struct cgraph_node *node, bool early) es, es3); } } + + /* Treat callback edges the same way + we treat speculative edges. */ + if (edge->callback) + { + cgraph_edge *parent = edge->get_callback_parent_edge (); + ipa_call_summary *es2 + = ipa_call_summaries->get_create (parent); + ipa_call_summaries->duplicate (edge, parent, es, es2); + } } /* TODO: When conditional jump or switch is known to be constant, but @@ -3480,22 +3490,20 @@ compute_fn_summary (struct cgraph_node *node, bool early) /* Code above should compute exactly the same result as ipa_update_overall_fn_summary except for case when speculative - edges are present since these are accounted to size but not + or callback edges are present since these are accounted to size but not self_size. Do not compare time since different order the roundoff errors result in slight changes. */ ipa_update_overall_fn_summary (node); if (flag_checking) { for (e = node->indirect_calls; e; e = e->next_callee) - if (e->speculative) - break; + if (e->speculative) + break; if (!e) - { - for (e = node->callees; e; e = e->next_callee) - if (e->callback) - break; - } + for (e = node->callees; e; e = e->next_callee) + if (e->callback) + break; gcc_assert (e || size_info->size == size_info->self_size); } } diff --git a/gcc/ipa-inline.cc b/gcc/ipa-inline.cc index 1cb5dc2d2d05..94c0f53f49ae 100644 --- a/gcc/ipa-inline.cc +++ b/gcc/ipa-inline.cc @@ -371,10 +371,10 @@ can_inline_edge_p (struct cgraph_edge *e, bool report, { gcc_checking_assert (e->inline_failed); - if(e->callback) { - fprintf(stderr, "skipping inline - callback\n"); + /* Always skip inlining callback edges, they need to be treated as + speculative edges in this regard. */ + if(e->callback) return false; - } if (cgraph_inline_failed_type (e->inline_failed) == CIF_FINAL_ERROR) { diff --git a/gcc/ipa-prop.cc b/gcc/ipa-prop.cc index ae0e32b1fe65..4a086ad19745 100644 --- a/gcc/ipa-prop.cc +++ b/gcc/ipa-prop.cc @@ -62,6 +62,7 @@ along with GCC; see the file COPYING3. If not see #include "attribs.h" #include "ipa-ref.h" #include "is-a.h" +#include "attr-callback.h" /* Function summary where the parameter infos are actually stored. */ ipa_node_params_t *ipa_node_params_sum = NULL; @@ -2318,40 +2319,16 @@ ipa_set_jfunc_vr (ipa_jump_func *jf, const ipa_vr &vr) ipa_set_jfunc_vr (jf, tmp); } -static void -calc_callback_args_idx (cgraph_edge *og, cgraph_edge *cbe) -{ - gcc_checking_assert (cbe->callback); - ipa_edge_args *args = ipa_edge_args_sum->get_create (cbe); - int argn = 0; - tree arg; - tree cb_args - = lookup_attribute ("callback", DECL_ATTRIBUTES (og->callee->decl)); - for (arg = TREE_CHAIN (TREE_VALUE (cb_args)); arg; - arg = TREE_CHAIN (arg), argn++) - ; - vec_safe_grow_cleared (args->jump_functions, argn, true); - vec_safe_grow_cleared (args->callback_args, argn, true); - gcc_checking_assert (cb_args); - int idx = 0; - for (arg = TREE_CHAIN (cb_args); arg; arg = TREE_CHAIN (arg), idx++) - { - tree val = TREE_VALUE (arg); - if (TREE_CODE (val) == INTEGER_CST) - { - int arg_idx = TREE_INT_CST_LOW (val); - bool is_data = arg_idx != -1; - if (arg_idx == -1) - { - arg_idx = idx; - } - (*args->callback_args)[idx] = {arg_idx, is_data}; - } - else - { - gcc_checking_assert (0); - } - } +/* Initializes ipa_edge_args summary of CBE given it's parent edge. + This primarily means allocating the correct amount of jump functions. */ + +static inline void +init_callback_edge_summary (struct cgraph_edge *parent, struct cgraph_edge *cbe) +{ + ipa_edge_args *parent_args = ipa_edge_args_sum->get (parent); + ipa_edge_args *cb_args = ipa_edge_args_sum->get_create (cbe); + vec_safe_grow_cleared (cb_args->jump_functions, + parent_args->jump_functions->length (), true); } /* Compute jump function for all arguments of callsite CS and insert the @@ -2379,7 +2356,7 @@ ipa_compute_jump_functions_for_edge (struct ipa_func_body_info *fbi, if (ipa_func_spec_opts_forbid_analysis_p (cs->caller)) return; - cgraph_edge * callback_edge = NULL; + auto_vec<cgraph_edge*> callback_edges; for (n = 0; n < arg_num; n++) { struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, n); @@ -2466,14 +2443,30 @@ ipa_compute_jump_functions_for_edge (struct ipa_func_body_info *fbi, if (TREE_CODE (arg) == ADDR_EXPR) { tree pointee = TREE_OPERAND (arg, 0); - if (TREE_CODE (pointee) == FUNCTION_DECL - && lookup_attribute ("callback", DECL_ATTRIBUTES (cs->callee->decl)) - && !cs->callback) + if (TREE_CODE (pointee) == FUNCTION_DECL && !cs->callback) { - cgraph_node *kernel_node = cgraph_node::get_create (pointee); - gcc_checking_assert (!callback_edge); - callback_edge = cs->make_callback (kernel_node); - calc_callback_args_idx (cs, callback_edge); + /* Argument is a pointer to a function. Look for a callback attribute + describing this argument. */ + tree callback_attr + = lookup_attribute ("callback", + DECL_ATTRIBUTES (cs->callee->decl)); + for (; callback_attr; + callback_attr + = lookup_attribute ("callback", + TREE_CHAIN (callback_attr))) + if (callback_get_fn_index (callback_attr) == n) + break; + /* If a callback attribute describing this pointer is found, + create a callback edge to pointee function to allow for + further optimizations. */ + if (callback_attr) + { + cgraph_node *kernel_node + = cgraph_node::get_create (pointee); + cgraph_edge *cbe = cs->make_callback (kernel_node); + init_callback_edge_summary (cs, cbe); + callback_edges.safe_push (cbe); + } } } } @@ -2535,29 +2528,31 @@ ipa_compute_jump_functions_for_edge (struct ipa_func_body_info *fbi, determine_known_aggregate_parts (fbi, call, arg, param_type, jfunc); } - if (callback_edge) + if (!callback_edges.is_empty ()) { - class ipa_edge_args *cb_summary = ipa_edge_args_sum->get (callback_edge); - unsigned i; - for (i = 0; i < cb_summary->callback_args->length (); i++) + /* For every callback edge, fetch jump functions of arguments + passed to them and copy them over to their respective summaries. + This avoids recalculating them for every callback edge, since their + arguments are just passed through. */ + unsigned j; + for (j = 0; j < callback_edges.length (); j++) { - cb_arg_info cb = (*cb_summary->callback_args)[i]; - class ipa_jump_func * src = ipa_get_ith_jump_func(args, cb.idx); - class ipa_jump_func * dst = ipa_get_ith_jump_func(cb_summary, i); - ipa_duplicate_jump_function(cs, callback_edge, src, dst); - if (cb.is_data_arg) + cgraph_edge *callback_edge = callback_edges[j]; + ipa_edge_args *cb_summary + = ipa_edge_args_sum->get_create (callback_edge); + auto_vec<int> arg_mapping + = callback_get_arg_mapping (cs->callee->decl); + unsigned i; + for (i = 0; i < arg_mapping.length (); i++) { - unsigned j; - for (j = 0; src->agg.items && (j < src->agg.items->length ()); - j++) - { - if ((*src->agg.items)[j].jftype == IPA_JF_PASS_THROUGH) - (*src->agg.items)[j].value.pass_through.agg_preserved - = true; - } + class ipa_jump_func *src + = ipa_get_ith_jump_func (args, arg_mapping[i]); + class ipa_jump_func *dst = ipa_get_ith_jump_func (cb_summary, i); + ipa_duplicate_jump_function (cs, callback_edge, src, dst); } } } + if (!useful_context) vec_free (args->polymorphic_call_contexts); } diff --git a/gcc/ipa-prop.h b/gcc/ipa-prop.h index 96eb07ebdb31..1ed5ed4e0fe4 100644 --- a/gcc/ipa-prop.h +++ b/gcc/ipa-prop.h @@ -1004,13 +1004,6 @@ void ipa_set_node_agg_value_chain (struct cgraph_node *node, void ipcp_transformation_initialize (void); void ipcp_free_transformation_sum (void); - -struct GTY(()) cb_arg_info { - int idx; - bool is_data_arg; -}; - - /* ipa_edge_args stores information related to a callsite and particularly its arguments. It can be accessed by the IPA_EDGE_REF macro. */ @@ -1019,7 +1012,7 @@ class GTY((for_user)) ipa_edge_args public: /* Default constructor. */ - ipa_edge_args () : jump_functions (NULL), polymorphic_call_contexts (NULL), callback_args(NULL) + ipa_edge_args () : jump_functions (NULL), polymorphic_call_contexts (NULL) {} /* Destructor. */ @@ -1031,14 +1024,12 @@ class GTY((for_user)) ipa_edge_args vec_free (jf->agg.items); vec_free (jump_functions); vec_free (polymorphic_call_contexts); - vec_free (callback_args); } /* Vectors of the callsite's jump function and polymorphic context information of each parameter. */ vec<ipa_jump_func, va_gc> *jump_functions; vec<ipa_polymorphic_call_context, va_gc> *polymorphic_call_contexts; - vec<cb_arg_info, va_gc> *callback_args; }; /* ipa_edge_args access functions. Please use these to access fields that diff --git a/gcc/omp-builtins.def b/gcc/omp-builtins.def index 46a68668d894..6e4f5be7338b 100644 --- a/gcc/omp-builtins.def +++ b/gcc/omp-builtins.def @@ -42,7 +42,7 @@ DEF_GOACC_BUILTIN (BUILT_IN_GOACC_EXIT_DATA, "GOACC_exit_data", ATTR_NOTHROW_LIST) DEF_GOACC_BUILTIN (BUILT_IN_GOACC_PARALLEL, "GOACC_parallel_keyed", BT_FN_VOID_INT_OMPFN_SIZE_PTR_PTR_PTR_VAR, - ATTR_NOTHROW_LIST) + ATTR_CALLBACK_OACC_LIST) DEF_GOACC_BUILTIN (BUILT_IN_GOACC_UPDATE, "GOACC_update", BT_FN_VOID_INT_SIZE_PTR_PTR_PTR_INT_INT_VAR, ATTR_NOTHROW_LIST) @@ -351,35 +351,35 @@ DEF_GOMP_BUILTIN (BUILT_IN_GOMP_LOOP_ULL_ORDERED_RUNTIME_NEXT, DEF_GOMP_BUILTIN (BUILT_IN_GOMP_PARALLEL_LOOP_STATIC, "GOMP_parallel_loop_static", BT_FN_VOID_OMPFN_PTR_UINT_LONG_LONG_LONG_LONG_UINT, - ATTR_NOTHROW_LIST) + ATTR_CALLBACK_GOMP_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_PARALLEL_LOOP_DYNAMIC, "GOMP_parallel_loop_dynamic", BT_FN_VOID_OMPFN_PTR_UINT_LONG_LONG_LONG_LONG_UINT, - ATTR_NOTHROW_LIST) + ATTR_CALLBACK_GOMP_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_PARALLEL_LOOP_GUIDED, "GOMP_parallel_loop_guided", BT_FN_VOID_OMPFN_PTR_UINT_LONG_LONG_LONG_LONG_UINT, - ATTR_NOTHROW_LIST) + ATTR_CALLBACK_GOMP_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_PARALLEL_LOOP_RUNTIME, "GOMP_parallel_loop_runtime", BT_FN_VOID_OMPFN_PTR_UINT_LONG_LONG_LONG_UINT, - ATTR_NOTHROW_LIST) + ATTR_CALLBACK_GOMP_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_PARALLEL_LOOP_NONMONOTONIC_DYNAMIC, "GOMP_parallel_loop_nonmonotonic_dynamic", BT_FN_VOID_OMPFN_PTR_UINT_LONG_LONG_LONG_LONG_UINT, - ATTR_NOTHROW_LIST) + ATTR_CALLBACK_GOMP_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_PARALLEL_LOOP_NONMONOTONIC_GUIDED, "GOMP_parallel_loop_nonmonotonic_guided", BT_FN_VOID_OMPFN_PTR_UINT_LONG_LONG_LONG_LONG_UINT, - ATTR_NOTHROW_LIST) + ATTR_CALLBACK_GOMP_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_PARALLEL_LOOP_NONMONOTONIC_RUNTIME, "GOMP_parallel_loop_nonmonotonic_runtime", BT_FN_VOID_OMPFN_PTR_UINT_LONG_LONG_LONG_UINT, - ATTR_NOTHROW_LIST) + ATTR_CALLBACK_GOMP_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_PARALLEL_LOOP_MAYBE_NONMONOTONIC_RUNTIME, "GOMP_parallel_loop_maybe_nonmonotonic_runtime", BT_FN_VOID_OMPFN_PTR_UINT_LONG_LONG_LONG_UINT, - ATTR_NOTHROW_LIST) + ATTR_CALLBACK_GOMP_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_LOOP_END, "GOMP_loop_end", BT_FN_VOID, ATTR_NOTHROW_LEAF_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_LOOP_END_CANCEL, "GOMP_loop_end_cancel", @@ -399,13 +399,13 @@ DEF_GOMP_BUILTIN (BUILT_IN_GOMP_DOACROSS_ULL_POST, "GOMP_doacross_ull_post", DEF_GOMP_BUILTIN (BUILT_IN_GOMP_DOACROSS_ULL_WAIT, "GOMP_doacross_ull_wait", BT_FN_VOID_ULL_VAR, ATTR_NOTHROW_LEAF_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_PARALLEL, "GOMP_parallel", - BT_FN_VOID_OMPFN_PTR_UINT_UINT, ATTR_NOTHROW_CALLBACK_GOMP_LIST) + BT_FN_VOID_OMPFN_PTR_UINT_UINT, ATTR_CALLBACK_GOMP_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_PARALLEL_REDUCTIONS, "GOMP_parallel_reductions", - BT_FN_UINT_OMPFN_PTR_UINT_UINT, ATTR_NOTHROW_LIST) + BT_FN_UINT_OMPFN_PTR_UINT_UINT, ATTR_CALLBACK_GOMP_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_TASK, "GOMP_task", BT_FN_VOID_OMPFN_PTR_OMPCPYFN_LONG_LONG_BOOL_UINT_PTR_INT_PTR, - ATTR_NOTHROW_LIST) + ATTR_CALLBACK_GOMP_TASK_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_TASKLOOP, "GOMP_taskloop", BT_FN_VOID_OMPFN_PTR_OMPCPYFN_LONG_LONG_UINT_LONG_INT_LONG_LONG_LONG, ATTR_NOTHROW_LIST) @@ -420,7 +420,7 @@ DEF_GOMP_BUILTIN (BUILT_IN_GOMP_SECTIONS_NEXT, "GOMP_sections_next", BT_FN_UINT, ATTR_NOTHROW_LEAF_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_PARALLEL_SECTIONS, "GOMP_parallel_sections", - BT_FN_VOID_OMPFN_PTR_UINT_UINT_UINT, ATTR_NOTHROW_LIST) + BT_FN_VOID_OMPFN_PTR_UINT_UINT_UINT, ATTR_CALLBACK_GOMP_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_SECTIONS_END, "GOMP_sections_end", BT_FN_VOID, ATTR_NOTHROW_LEAF_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_SECTIONS_END_CANCEL, @@ -461,7 +461,7 @@ DEF_GOMP_BUILTIN (BUILT_IN_GOMP_TARGET_MAP_INDIRECT_PTR, DEF_GOMP_BUILTIN (BUILT_IN_GOMP_TEAMS4, "GOMP_teams4", BT_FN_BOOL_UINT_UINT_UINT_BOOL, ATTR_NOTHROW_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_TEAMS_REG, "GOMP_teams_reg", - BT_FN_VOID_OMPFN_PTR_UINT_UINT_UINT, ATTR_NOTHROW_LIST) + BT_FN_VOID_OMPFN_PTR_UINT_UINT_UINT, ATTR_CALLBACK_GOMP_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_TASKGROUP_REDUCTION_REGISTER, "GOMP_taskgroup_reduction_register", BT_FN_VOID_PTR, ATTR_NOTHROW_LEAF_LIST)