Hi,
This patch adds GUESSED_GLOBAL0_AFDO profile quality. It can
be used to preserve local counts of functions which have 0 AFDO
profile.
I originally did not include it as it was not clear it will be useful and
it turns quality from 3bits to 4bits which means that we need to steal another
bit from the actual counters.
It is likely not a problem for profile_count since counting up to 2^60 still
takes a while. However with profile_probability I run into a problem that
gimple FE testcases encodes probability with current representation and thus
changing profile_probability::n_bits would require updating all testcases.
Since probabilities never use GLOBAL0 qualities (they are always local)
addoing bits is not necessary, but it requires encoding quality in
the data type that required adding accessors.
While working on this I also noticed that GUESSED_GLOBAL0 and
GUESSED_GLOBAL0_ADJUSED are misordered. Qualities should be in
increasing order. This is also fixed.
auto-profile will be updated later.
Bootstrapped/regtested x86_64-linux, comitted.
Honza
gcc/ChangeLog:
* cgraph.cc (cgraph_node::make_profile_global0): Support
GUESSED_GLOBAL0_AFDO
* ipa-cp.cc (update_profiling_info): Use
GUESSED_GLOBAL0_AFDO.
* profile-count.cc (profile_probability::dump): Use
quality ().
(profile_probability::stream_in): Use m_adjusted_quality.
(profile_probability::stream_out): Use m_adjusted_quality.
(profile_count::combine_with_ipa_count): Use quality ().
(profile_probability::sqrt): Likewise.
* profile-count.h (enum profile_quality): Add
GUESSED_GLOBAL0_AFDO; reoder GUESSED_GLOBAL0_ADJUSTED and
GUESSED_GLOBAL0.
(profile_probability): Add min_quality; replase m_quality
by m_adjused_quality; add set_quality; update all users
of quality.
(profile_count): Set n_bits to 60; make m_quality 4 bits;
update uses of quality.
(profile_count::afdo_zero, profile_count::globa0afdo): New.
diff --git a/gcc/cgraph.cc b/gcc/cgraph.cc
index 2441c9e866d..94a2e6e6105 100644
--- a/gcc/cgraph.cc
+++ b/gcc/cgraph.cc
@@ -200,7 +200,8 @@ cgraph_node::make_profile_local ()
}
/* Turn profile to global0. Walk into inlined functions.
- QUALITY must be GUESSED_GLOBAL0 or GUESSED_GLOBAL0_ADJUSTED */
+ QUALITY must be GUESSED_GLOBAL0, GUESSED_GLOBAL0_ADJUSTED
+ or GUESSED_GLOBAL0_AFDO */
void
cgraph_node::make_profile_global0 (profile_quality quality)
{
@@ -219,6 +220,14 @@ cgraph_node::make_profile_global0 (profile_quality quality)
return;
count = count.global0adjusted ();
}
+ else if (quality == GUESSED_GLOBAL0_AFDO)
+ {
+ if (count.quality () == GUESSED_GLOBAL0
+ || count.quality () == GUESSED_GLOBAL0_ADJUSTED
+ || count.quality () == GUESSED_GLOBAL0_AFDO)
+ return;
+ count = count.global0afdo ();
+ }
else
gcc_unreachable ();
for (cgraph_edge *e = callees; e; e = e->next_callee)
@@ -231,6 +240,8 @@ cgraph_node::make_profile_global0 (profile_quality quality)
e->count = e->count.global0 ();
else if (quality == GUESSED_GLOBAL0_ADJUSTED)
e->count = e->count.global0adjusted ();
+ else if (quality == GUESSED_GLOBAL0_AFDO)
+ e->count = e->count.global0afdo ();
else
gcc_unreachable ();
}
@@ -241,6 +252,8 @@ cgraph_node::make_profile_global0 (profile_quality quality)
e->count = e->count.global0 ();
else if (quality == GUESSED_GLOBAL0_ADJUSTED)
e->count = e->count.global0adjusted ();
+ else if (quality == GUESSED_GLOBAL0_AFDO)
+ e->count = e->count.global0afdo ();
else
gcc_unreachable ();
}
diff --git a/gcc/ipa-cp.cc b/gcc/ipa-cp.cc
index 3bf0117612e..3e073af662a 100644
--- a/gcc/ipa-cp.cc
+++ b/gcc/ipa-cp.cc
@@ -4773,6 +4773,8 @@ update_profiling_info (struct cgraph_node *orig_node,
to global0adjusted or to local if we have partial training. */
if (opt_for_fn (orig_node->decl, flag_profile_partial_training))
orig_node->make_profile_local ();
+ if (new_sum.quality () == AFDO)
+ orig_node->make_profile_global0 (GUESSED_GLOBAL0_AFDO);
else
orig_node->make_profile_global0 (GUESSED_GLOBAL0_ADJUSTED);
orig_edges_processed = true;
@@ -4802,6 +4804,8 @@ update_profiling_info (struct cgraph_node *orig_node,
the latter and bail out. */
if (opt_for_fn (orig_node->decl, flag_profile_partial_training))
orig_node->make_profile_local ();
+ else if (orig_nonrec_call_count.quality () == AFDO)
+ orig_node->make_profile_global0 (GUESSED_GLOBAL0_AFDO);
else
orig_node->make_profile_global0 (GUESSED_GLOBAL0_ADJUSTED);
return;
diff --git a/gcc/profile-count.cc b/gcc/profile-count.cc
index e857cddea7e..190bbebb5a7 100644
--- a/gcc/profile-count.cc
+++ b/gcc/profile-count.cc
@@ -39,8 +39,9 @@ const char *profile_quality_names[] =
{
"uninitialized",
"guessed_local",
- "guessed_global0",
+ "guessed_global0afdo",
"guessed_global0adjusted",
+ "guessed_global0",
"guessed",
"afdo",
"adjusted",
@@ -76,8 +77,9 @@ const char *profile_quality_display_names[] =
{
NULL,
"estimated locally",
- "estimated locally, globally 0",
+ "estimated locally, globally 0 auto FDO",
"estimated locally, globally 0 adjusted",
+ "estimated locally, globally 0",
"guessed",
"auto FDO",
"adjusted",
@@ -184,11 +186,11 @@ profile_probability::dump (char *buffer) const
else
buffer += sprintf (buffer, "%3.1f%%", (double)m_val * 100 /
max_probability);
- if (m_quality == ADJUSTED)
+ if (quality () == ADJUSTED)
sprintf (buffer, " (adjusted)");
- else if (m_quality == AFDO)
+ else if (quality () == AFDO)
sprintf (buffer, " (auto FDO)");
- else if (m_quality == GUESSED)
+ else if (quality () == GUESSED)
sprintf (buffer, " (guessed)");
}
}
@@ -246,7 +248,7 @@ profile_probability::stream_in (class lto_input_block *ib)
{
profile_probability ret;
ret.m_val = streamer_read_uhwi (ib);
- ret.m_quality = (profile_quality) streamer_read_uhwi (ib);
+ ret.m_adjusted_quality = streamer_read_uhwi (ib);
return ret;
}
@@ -256,7 +258,7 @@ void
profile_probability::stream_out (struct output_block *ob)
{
streamer_write_uhwi (ob, m_val);
- streamer_write_uhwi (ob, m_quality);
+ streamer_write_uhwi (ob, m_adjusted_quality);
}
/* Stream THIS to OB. */
@@ -265,7 +267,7 @@ void
profile_probability::stream_out (struct lto_output_stream *ob)
{
streamer_write_uhwi_stream (ob, m_val);
- streamer_write_uhwi_stream (ob, m_quality);
+ streamer_write_uhwi_stream (ob, m_adjusted_quality);
}
/* Compute RES=(a*b + c/2)/c capping and return false if overflow happened. */
@@ -406,6 +408,8 @@ profile_count::combine_with_ipa_count (profile_count ipa)
return *this;
if (ipa == zero ())
return this->global0 ();
+ if (ipa == afdo_zero ())
+ return this->global0afdo ();
return this->global0adjusted ();
}
@@ -495,7 +499,7 @@ profile_probability::sqrt () const
if (!initialized_p () || *this == never () || *this == always ())
return *this;
profile_probability ret = *this;
- ret.m_quality = MIN (ret.m_quality, ADJUSTED);
+ ret.set_quality (MIN (ret.quality (), ADJUSTED));
uint32_t min_range = m_val;
uint32_t max_range = max_probability;
if (!m_val)
diff --git a/gcc/profile-count.h b/gcc/profile-count.h
index d6231b3b17d..216054033c5 100644
--- a/gcc/profile-count.h
+++ b/gcc/profile-count.h
@@ -37,19 +37,21 @@ enum profile_quality {
*/
GUESSED_LOCAL,
+ /* Same as GUESSED_GLOBAL0 but global count is afdo 0. */
+ GUESSED_GLOBAL0_AFDO,
+
+ /* Same as GUESSED_GLOBAL0 but global count is adjusted 0. */
+ GUESSED_GLOBAL0_ADJUSTED,
+
/* Profile was read by feedback and was 0, we used local heuristics to guess
better. This is the case of functions not run in profile feedback.
Never used by probabilities. */
GUESSED_GLOBAL0,
- /* Same as GUESSED_GLOBAL0 but global count is adjusted 0. */
- GUESSED_GLOBAL0_ADJUSTED,
-
/* Profile is based on static branch prediction heuristics. It may or may
not reflect the reality but it can be compared interprocedurally
(for example, we inlined function w/o profile feedback into function
- with feedback and propagated from that).
- Never used by probabilities. */
+ with feedback and propagated from that). */
GUESSED,
/* Profile was determined by autofdo. */
@@ -151,26 +153,39 @@ class GTY((user)) profile_probability
static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
static const uint32_t uninitialized_probability
= ((uint32_t) 1 << (n_bits - 1)) - 1;
+ /* For probibilityes quality is either UNINITIALIZED (0)
+ or greated then GUESSED. To save bits we store it in
+ adjusted form which skips the invalid values. */
+ static const int min_quality = GUESSED;
uint32_t m_val : 29;
- enum profile_quality m_quality : 3;
+ unsigned m_adjusted_quality : 3;
friend struct profile_count;
+
+ /* Set the quality of the probability. */
+ void
+ set_quality (profile_quality quality)
+ {
+ gcc_checking_assert (quality == UNINITIALIZED_PROFILE
+ || (quality >= min_quality && quality <= PRECISE));
+ m_adjusted_quality = quality ? quality - min_quality + 1 : 0;
+ }
+
public:
- profile_probability (): m_val (uninitialized_probability),
- m_quality (GUESSED)
- {}
+ profile_probability (): m_val (uninitialized_probability)
+ { set_quality (GUESSED); }
profile_probability (uint32_t val, profile_quality quality):
- m_val (val), m_quality (quality)
- {}
+ m_val (val)
+ { set_quality (quality); }
/* Named probabilities. */
static profile_probability never ()
{
profile_probability ret;
ret.m_val = 0;
- ret.m_quality = PRECISE;
+ ret.set_quality (PRECISE);
return ret;
}
@@ -178,7 +193,7 @@ public:
{
profile_probability ret;
ret.m_val = 0;
- ret.m_quality = GUESSED;
+ ret.set_quality (GUESSED);
return ret;
}
@@ -222,7 +237,7 @@ public:
{
profile_probability ret;
ret.m_val = max_probability;
- ret.m_quality = GUESSED;
+ ret.set_quality (GUESSED);
return ret;
}
@@ -230,7 +245,7 @@ public:
{
profile_probability ret;
ret.m_val = max_probability;
- ret.m_quality = PRECISE;
+ ret.set_quality (PRECISE);
return ret;
}
@@ -240,7 +255,7 @@ public:
{
profile_probability c;
c.m_val = uninitialized_probability;
- c.m_quality = GUESSED;
+ c.set_quality (GUESSED);
return c;
}
@@ -253,7 +268,7 @@ public:
/* Return true if value can be trusted. */
bool reliable_p () const
{
- return m_quality >= ADJUSTED;
+ return quality () >= ADJUSTED;
}
/* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
@@ -263,7 +278,7 @@ public:
profile_probability ret;
gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
- ret.m_quality = GUESSED;
+ ret.set_quality (GUESSED);
return ret;
}
@@ -273,7 +288,7 @@ public:
profile_probability ret = *this;
if (!initialized_p ())
return *this;
- ret.m_quality = ADJUSTED;
+ ret.set_quality (ADJUSTED);
return ret;
}
@@ -288,14 +303,14 @@ public:
{
profile_probability ret;
ret.m_val = ((unsigned int)v) / 8;
- ret.m_quality = (enum profile_quality)(v & 7);
+ ret.m_adjusted_quality = ((unsigned int)v) & 7;
return ret;
}
int to_reg_br_prob_note () const
{
gcc_checking_assert (initialized_p ());
- int ret = m_val * 8 + m_quality;
+ int ret = m_val * 8 + m_adjusted_quality;
gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
return ret;
}
@@ -315,14 +330,14 @@ public:
gcc_checking_assert (tmp <= max_probability);
ret.m_val = tmp;
}
- ret.m_quality = PRECISE;
+ ret.set_quality (PRECISE);
return ret;
}
/* Basic operations. */
bool operator== (const profile_probability &other) const
{
- return m_val == other.m_val && m_quality == other.m_quality;
+ return m_val == other.m_val && quality () == other.quality ();
}
profile_probability operator+ (const profile_probability &other) const
@@ -336,7 +351,7 @@ public:
profile_probability ret;
ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
- ret.m_quality = MIN (m_quality, other.m_quality);
+ ret.set_quality (MIN (quality (), other.quality ()));
return ret;
}
@@ -354,7 +369,7 @@ public:
else
{
m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
- m_quality = MIN (m_quality, other.m_quality);
+ set_quality (MIN (quality (), other.quality ()));
}
return *this;
}
@@ -368,7 +383,7 @@ public:
return uninitialized ();
profile_probability ret;
ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
- ret.m_quality = MIN (m_quality, other.m_quality);
+ ret.set_quality (MIN (quality (), other.quality ()));
return ret;
}
@@ -382,7 +397,7 @@ public:
else
{
m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
- m_quality = MIN (m_quality, other.m_quality);
+ set_quality (MIN (quality (), other.quality ()));
}
return *this;
}
@@ -396,7 +411,7 @@ public:
return uninitialized ();
profile_probability ret;
ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
- ret.m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
+ ret.set_quality (MIN (quality (), other.quality ()));
return ret;
}
@@ -410,7 +425,7 @@ public:
else
{
m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
- m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
+ set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
}
return *this;
}
@@ -426,8 +441,8 @@ public:
if (m_val >= other.m_val)
{
ret.m_val = max_probability;
- ret.m_quality = MIN (MIN (m_quality, other.m_quality),
- GUESSED);
+ ret.set_quality (MIN (MIN (quality (), other.quality ()),
+ GUESSED));
return ret;
}
else if (!m_val)
@@ -439,7 +454,7 @@ public:
other.m_val),
max_probability);
}
- ret.m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
+ ret.set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
return ret;
}
@@ -451,13 +466,13 @@ public:
return *this = uninitialized ();
else
{
- /* If we get probability above 1, mark it as unreliable
+ /* If we get probability above 1, mark it as unreliable
and return 1. */
if (m_val > other.m_val)
{
m_val = max_probability;
- m_quality = MIN (MIN (m_quality, other.m_quality),
- GUESSED);
+ set_quality (MIN (MIN (quality (), other.quality ()),
+ GUESSED));
return *this;
}
else if (!m_val)
@@ -469,7 +484,7 @@ public:
other.m_val),
max_probability);
}
- m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
+ set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
}
return *this;
}
@@ -519,7 +534,7 @@ public:
profile_probability guessed () const
{
profile_probability ret = *this;
- ret.m_quality = GUESSED;
+ ret.set_quality (GUESSED);
return ret;
}
@@ -527,7 +542,7 @@ public:
profile_probability afdo () const
{
profile_probability ret = *this;
- ret.m_quality = AFDO;
+ ret.set_quality (AFDO);
return ret;
}
@@ -542,7 +557,7 @@ public:
uint64_t tmp;
safe_scale_64bit (m_val, num, den, &tmp);
ret.m_val = MIN (tmp, max_probability);
- ret.m_quality = MIN (m_quality, ADJUSTED);
+ ret.set_quality (MIN (quality (), ADJUSTED));
return ret;
}
@@ -564,8 +579,8 @@ public:
uint64_t val;
safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
ret.m_val = MIN (val, max_probability);
- ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
- num.m_quality), den.m_quality);
+ ret.set_quality (MIN (MIN (MIN (quality (), ADJUSTED),
+ num.quality ()), den.quality ()));
return ret;
}
@@ -587,7 +602,7 @@ public:
CFG if we find this information useful on multiple places. */
bool probably_reliable_p () const
{
- if (m_quality >= ADJUSTED)
+ if (quality () >= ADJUSTED)
return true;
if (!initialized_p ())
return false;
@@ -598,10 +613,10 @@ public:
/* Return false if profile_probability is bogus. */
bool verify () const
{
- gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
+ gcc_checking_assert (quality () != UNINITIALIZED_PROFILE);
if (m_val == uninitialized_probability)
- return m_quality == GUESSED;
- else if (m_quality < GUESSED)
+ return quality () == GUESSED;
+ else if (quality () < GUESSED)
return false;
return m_val <= max_probability;
}
@@ -656,11 +671,16 @@ public:
/* Compute sware root. */
profile_probability sqrt () const;
- /* Get the value of the count. */
+ /* Get the value of the probability. */
uint32_t value () const { return m_val; }
- /* Get the quality of the count. */
- enum profile_quality quality () const { return m_quality; }
+ /* Get the quality of the probability. */
+ enum profile_quality quality () const
+ {
+ return (profile_quality) (m_adjusted_quality
+ ? m_adjusted_quality + min_quality - 1
+ : UNINITIALIZED_PROFILE);
+ }
/* Output THIS to F. */
void dump (FILE *f) const;
@@ -749,11 +769,11 @@ public:
struct GTY(()) profile_count
{
public:
- /* Use 62bit to hold basic block counters. Should be at least
+ /* Use 60bit to hold basic block counters. Should be at least
64bit. Although a counter cannot be negative, we use a signed
type to hold various extra stages. */
- static const int n_bits = 61;
+ static const int n_bits = 60;
static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
private:
static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
@@ -769,7 +789,7 @@ private:
#endif
uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
#undef UINT64_BIT_FIELD_ALIGN
- enum profile_quality m_quality : 3;
+ enum profile_quality m_quality : 4;
public:
/* Return true if both values can meaningfully appear in single function
@@ -808,6 +828,14 @@ public:
return c;
}
+ static profile_count afdo_zero ()
+ {
+ profile_count c;
+ c.m_val = 0;
+ c.m_quality = AFDO;
+ return c;
+ }
+
static profile_count guessed_zero ()
{
profile_count c;
@@ -853,7 +881,7 @@ public:
/* Return true if value can be operated inter-procedurally. */
bool ipa_p () const
{
- return !initialized_p () || m_quality >= GUESSED_GLOBAL0;
+ return !initialized_p () || m_quality >= GUESSED_GLOBAL0_AFDO;
}
/* Return true if quality of profile is precise. */
@@ -929,7 +957,7 @@ public:
return *this = uninitialized ();
else
{
- gcc_checking_assert (compatible_p (other));
+ gcc_checking_assert (compatible_p (other));
uint64_t ret_val = m_val + other.m_val;
m_val = MIN (ret_val, max_count);
m_quality = MIN (m_quality, other.m_quality);
@@ -958,7 +986,7 @@ public:
return *this = uninitialized ();
else
{
- gcc_checking_assert (compatible_p (other));
+ gcc_checking_assert (compatible_p (other));
m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
m_quality = MIN (m_quality, other.m_quality);
}
@@ -1092,7 +1120,7 @@ public:
if (ret.m_val == 0)
{
ret.m_val = 1;
- ret.m_quality = MIN (m_quality, ADJUSTED);
+ ret.m_quality = MIN (m_quality, ADJUSTED);
}
return ret;
}
@@ -1153,7 +1181,7 @@ public:
safe_scale_64bit (m_val, prob.m_val,
profile_probability::max_probability,
&tmp);
ret.m_val = tmp;
- ret.m_quality = MIN (m_quality, prob.m_quality);
+ ret.m_quality = MIN (m_quality, prob.quality ());
return ret;
}
@@ -1191,7 +1219,7 @@ public:
safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
ret.m_val = MIN (val, max_count);
ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
- num.m_quality), den.m_quality);
+ num.m_quality), den.m_quality);
/* Be sure that ret is not local if num is global.
Also ensure that ret is not global0 when num is global. */
if (num.ipa_p ())
@@ -1220,6 +1248,17 @@ public:
return ret;
}
+ /* We know that profile is globally afdo 0 but keep local profile
+ if present. */
+ profile_count global0afdo () const
+ {
+ profile_count ret = *this;
+ if (!initialized_p ())
+ return *this;
+ ret.m_quality = GUESSED_GLOBAL0_AFDO;
+ return ret;
+ }
+
/* We know that profile is globally adjusted 0 but keep local profile
if present. */
profile_count global0adjusted () const
@@ -1252,12 +1291,14 @@ public:
across functions. */
profile_count ipa () const
{
- if (m_quality > GUESSED_GLOBAL0_ADJUSTED)
+ if (m_quality > GUESSED_GLOBAL0)
return *this;
if (m_quality == GUESSED_GLOBAL0)
return zero ();
if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
return adjusted_zero ();
+ if (m_quality == GUESSED_GLOBAL0_AFDO)
+ return afdo_zero ();
return uninitialized ();
}
@@ -1287,14 +1328,14 @@ public:
if (overall.m_val < m_val)
{
ret.m_val = profile_probability::max_probability;
- ret.m_quality = GUESSED;
+ ret.set_quality (GUESSED);
return ret;
}
else
ret.m_val = RDIV (m_val * profile_probability::max_probability,
overall.m_val);
- ret.m_quality = MIN (MAX (MIN (m_quality, overall.m_quality),
- GUESSED), ADJUSTED);
+ ret.set_quality (MIN (MAX (MIN (m_quality, overall.m_quality),
+ GUESSED), ADJUSTED));
return ret;
}
