Hi,
This patch adds support of atomic update of profiles counters. The goal is to
improve
the poor counter values for highly thread programs.
The atomic update is under a new option -fprofile-gen-atomic=<N>
N=0: default, no atomic update
N=1: atomic update edge counters.
N=2: atomic update some of value profile counters (currently indirect-call and
one value profile).
N=3: both edge counter and the above value profile counters.
Other value: fall back to the default.
This patch is a simple porting of the version in google-4_7 branch. It uses
__atomic_fetch_add
based on Andrew Pinski's suggestion. Note I did not apply to all the value
profiles as
the indirect-call profile is the most relevant one here.
Test with bootstrap.
Comments and suggestions are welcomed.
Thanks,
-Rong
2012-12-20 Rong Xu <x...@google.com>
* libgcc/libgcov.c (__gcov_one_value_profiler_body_atomic): New
function. Atomic update profile counters.
(__gcov_one_value_profiler_atomic): Ditto.
(__gcov_indirect_call_profiler_atomic): Ditto.
* gcc/gcov-io.h: Macros for atomic update.
* gcc/common.opt: New option.
* gcc/tree-profile.c (gimple_init_edge_profiler): Atomic
update profile counters.
(gimple_gen_edge_profiler): Ditto.
Index: libgcc/libgcov.c
===================================================================
--- libgcc/libgcov.c (revision 194652)
+++ libgcc/libgcov.c (working copy)
@@ -1113,12 +1113,35 @@ __gcov_one_value_profiler_body (gcov_type *counter
counters[2]++;
}
+/* Atomic update version of __gcov_one_value_profile_body(). */
+static inline void
+__gcov_one_value_profiler_body_atomic (gcov_type *counters, gcov_type value)
+{
+ if (value == counters[0])
+ GCOV_TYPE_ATOMIC_FETCH_ADD_FN (&counters[1], 1, MEMMODEL_RELAXED);
+ else if (counters[1] == 0)
+ {
+ counters[1] = 1;
+ counters[0] = value;
+ }
+ else
+ GCOV_TYPE_ATOMIC_FETCH_ADD_FN (&counters[1], -1, MEMMODEL_RELAXED);
+ GCOV_TYPE_ATOMIC_FETCH_ADD_FN (&counters[2], 1, MEMMODEL_RELAXED);
+}
+
#ifdef L_gcov_one_value_profiler
void
__gcov_one_value_profiler (gcov_type *counters, gcov_type value)
{
__gcov_one_value_profiler_body (counters, value);
}
+
+void
+__gcov_one_value_profiler_atomic (gcov_type *counters, gcov_type value)
+{
+ __gcov_one_value_profiler_body_atomic (counters, value);
+}
+
#endif
#ifdef L_gcov_indirect_call_profiler
@@ -1153,6 +1176,17 @@ __gcov_indirect_call_profiler (gcov_type* counter,
&& *(void **) cur_func == *(void **) callee_func))
__gcov_one_value_profiler_body (counter, value);
}
+
+/* Atomic update version of __gcov_indirect_call_profiler(). */
+void
+__gcov_indirect_call_profiler_atomic (gcov_type* counter, gcov_type value,
+ void* cur_func, void* callee_func)
+{
+ if (cur_func == callee_func
+ || (VTABLE_USES_DESCRIPTORS && callee_func
+ && *(void **) cur_func == *(void **) callee_func))
+ __gcov_one_value_profiler_body_atomic (counter, value);
+}
#endif
Index: gcc/gcov-io.h
===================================================================
--- gcc/gcov-io.h (revision 194652)
+++ gcc/gcov-io.h (working copy)
@@ -202,7 +202,15 @@ typedef unsigned gcov_type_unsigned __attribute__
#endif
#endif
+#if LONG_LONG_TYPE_SIZE > 32
+#define GCOV_TYPE_ATOMIC_FETCH_ADD_FN __atomic_fetch_add_8
+#define GCOV_TYPE_ATOMIC_FETCH_ADD BUILT_IN_ATOMIC_FETCH_ADD_8
+#else
+#define GCOV_TYPE_ATOMIC_FETCH_ADD_FN __atomic_fetch_add_4
+#define GCOV_TYPE_ATOMIC_FETCH_ADD BUILT_IN_ATOMIC_FETCH_ADD_4
+#endif
+
#if defined (TARGET_POSIX_IO)
#define GCOV_LOCKED 1
#else
@@ -212,6 +220,18 @@ typedef unsigned gcov_type_unsigned __attribute__
#else /* !IN_LIBGCOV */
/* About the host */
+#if LONG_LONG_TYPE_SIZE > 32
+#define GCOV_TYPE_ATOMIC_FETCH_ADD_FN __atomic_fetch_add_8
+#define GCOV_TYPE_ATOMIC_FETCH_ADD BUILT_IN_ATOMIC_FETCH_ADD_8
+#else
+#define GCOV_TYPE_ATOMIC_FETCH_ADD_FN __atomic_fetch_add_4
+#define GCOV_TYPE_ATOMIC_FETCH_ADD BUILT_IN_ATOMIC_FETCH_ADD_4
+#endif
+#define PROFILE_GEN_EDGE_ATOMIC (flag_profile_gen_atomic == 1 || \
+ flag_profile_gen_atomic == 3)
+#define PROFILE_GEN_VALUE_ATOMIC (flag_profile_gen_atomic == 2 || \
+ flag_profile_gen_atomic == 3)
+
typedef unsigned gcov_unsigned_t;
typedef unsigned gcov_position_t;
/* gcov_type is typedef'd elsewhere for the compiler */
Index: gcc/common.opt
===================================================================
--- gcc/common.opt (revision 194652)
+++ gcc/common.opt (working copy)
@@ -1635,6 +1635,15 @@ fprofile-correction
Common Report Var(flag_profile_correction)
Enable correction of flow inconsistent profile data input
+; fprofile-gen-atomic=0: disable atomically update.
+; fprofile-gen-atomic=1: atomically update edge profile counters.
+; fprofile-gen-atomic=2: atomically update value profile counters.
+; fprofile-gen-atomic=3: atomically update edge and value profile counters.
+; other values will be ignored (fall back to the default of 0).
+fprofile-gen-atomic=
+Common Joined UInteger Report Var(flag_profile_gen_atomic) Init(0) Optimization
+fprofile-gen-atomic=[0..3] Atomically increments for profile counters.
+
fprofile-generate
Common
Enable common options for generating profile info for profile feedback
directed optimizations
Index: gcc/tree-profile.c
===================================================================
--- gcc/tree-profile.c (revision 194652)
+++ gcc/tree-profile.c (working copy)
@@ -147,7 +147,12 @@ gimple_init_edge_profiler (void)
= build_function_type_list (void_type_node,
gcov_type_ptr, gcov_type_node,
NULL_TREE);
- tree_one_value_profiler_fn
+ if (PROFILE_GEN_VALUE_ATOMIC)
+ tree_one_value_profiler_fn
+ = build_fn_decl ("__gcov_one_value_profiler_atomic",
+ one_value_profiler_fn_type);
+ else
+ tree_one_value_profiler_fn
= build_fn_decl ("__gcov_one_value_profiler",
one_value_profiler_fn_type);
TREE_NOTHROW (tree_one_value_profiler_fn) = 1;
@@ -163,9 +168,14 @@ gimple_init_edge_profiler (void)
gcov_type_ptr, gcov_type_node,
ptr_void,
ptr_void, NULL_TREE);
- tree_indirect_call_profiler_fn
- = build_fn_decl ("__gcov_indirect_call_profiler",
- ic_profiler_fn_type);
+ if (PROFILE_GEN_VALUE_ATOMIC)
+ tree_indirect_call_profiler_fn
+ = build_fn_decl ("__gcov_indirect_call_profiler_atomic",
+ ic_profiler_fn_type);
+ else
+ tree_indirect_call_profiler_fn
+ = build_fn_decl ("__gcov_indirect_call_profiler",
+ ic_profiler_fn_type);
TREE_NOTHROW (tree_indirect_call_profiler_fn) = 1;
DECL_ATTRIBUTES (tree_indirect_call_profiler_fn)
= tree_cons (get_identifier ("leaf"), NULL,
@@ -211,8 +221,21 @@ gimple_gen_edge_profiler (int edgeno, edge e)
tree ref, one, gcov_type_tmp_var;
gimple stmt1, stmt2, stmt3;
+ one = build_int_cst (gcov_type_node, 1);
+ if (PROFILE_GEN_EDGE_ATOMIC)
+ {
+ ref = tree_coverage_counter_addr (GCOV_COUNTER_ARCS, edgeno);
+ /* __atomic_fetch_add (&counter, 1, MEMMODEL_RELAXED); */
+ stmt1 = gimple_build_call (builtin_decl_explicit (
+ GCOV_TYPE_ATOMIC_FETCH_ADD),
+ 3, ref, one,
+ build_int_cst (integer_type_node,
+ MEMMODEL_RELAXED));
+ gsi_insert_on_edge (e, stmt1);
+ return;
+ }
+
ref = tree_coverage_counter_ref (GCOV_COUNTER_ARCS, edgeno);
- one = build_int_cst (gcov_type_node, 1);
gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
NULL, "PROF_edge_counter");
stmt1 = gimple_build_assign (gcov_type_tmp_var, ref);
--
This patch is available for review at http://codereview.appspot.com/7000044
