> -----Original Message----- > From: Kewen Lin <[email protected]> > Sent: Monday, June 1, 2026 1:52 PM > To: [email protected] > Cc: Uros Bizjak <[email protected]>; [email protected]; Liu, Hongtao > <[email protected]>; Feng Xue <[email protected]>; Qingkuan Lai > <[email protected]>; Tianle Shi <[email protected]> > Subject: [PATCH] i386: Add avx512_avoid_vec_perm tune for c86-4g-m7 > > Hi, > > This implements the AVX512 avoid-vec-perm cost model for c86-4g-m7 > following the existing AVX256 support r15-4234-g9eaecce3d8c1d9. > > Extend the vectorizer cost model to treat 512-bit cross-lane vec_perm as > unprofitable when this tune is enabled, since c86-4g-m7 implements > AVX512 as two 256-bit halves and cross-lane reorder is slower than staying in > AVX256. > > Factor out the cross-lane permutation classification into a helper so the > AVX256 and AVX512 cost handling can share the same logic without > duplicating it. > > Bootstrapped and regress-tested on one c86-4g-m7 machine, is it ok for > trunk?
LGTM. > > BR, > Kewen > ----- > > gcc/ChangeLog: > > * config/i386/i386.cc (ix86_vector_costs): Add > m_num_avx512_vec_perm. > (ix86_vector_costs::ix86_vector_costs): Initialize it. > (ix86_count_cross_lane_perm_p): New helper to classify > cross-lane vec_perm for AVX256 and AVX512 handling. > (ix86_vector_costs::add_stmt_cost): Use the helper for > vec_perm accounting, record 256-bit and 512-bit counts, > and print detected width as avx%u. > (ix86_vector_costs::finish_cost): Reject AVX512 costs > when TARGET_AVX512_AVOID_VEC_PERM is set and 512-bit > cross-lane vec_perm is present. > * config/i386/i386.h (TARGET_AVX512_AVOID_VEC_PERM): > New macro. > * config/i386/x86-tune.def > (X86_TUNE_AVX512_AVOID_VEC_PERM): > New tune for c86-4g-m7. > > gcc/testsuite/ChangeLog: > > * gcc.target/i386/avx512_avoid_vec_perm-1.c: New test. > * gcc.target/i386/avx512_avoid_vec_perm-2.c: New test. > * gcc.target/i386/avx512_avoid_vec_perm-3.c: New test. > > Authored-by: shitianle <[email protected]> > Signed-off-by: shitianle <[email protected]> > --- > gcc/config/i386/i386.cc | 105 ++++++++++-------- > gcc/config/i386/i386.h | 2 + > gcc/config/i386/x86-tune.def | 4 + > .../gcc.target/i386/avx512_avoid_vec_perm-1.c | 27 > +++++ .../gcc.target/i386/avx512_avoid_vec_perm-2.c | 25 > +++++ .../gcc.target/i386/avx512_avoid_vec_perm-3.c | 32 ++++++ > 6 files changed, 150 insertions(+), 45 deletions(-) create mode 100644 > gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-1.c > create mode 100644 gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm- > 2.c > create mode 100644 gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm- > 3.c > > diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc index > 9148ac1e057..71781210fe2 100644 > --- a/gcc/config/i386/i386.cc > +++ b/gcc/config/i386/i386.cc > @@ -26183,6 +26183,8 @@ private: > unsigned m_num_sse_needed[3]; > /* Number of 256-bit vector permutation. */ > unsigned m_num_avx256_vec_perm[3]; > + /* Number of 512-bit vector permutation. */ unsigned > + m_num_avx512_vec_perm[3]; > /* Number of reductions for FMA/DOT_PROD_EXPR/SAD_EXPR */ > unsigned m_num_reduc[X86_REDUC_LAST]; > /* Don't do unroll if m_prefer_unroll is false, default is true. */ @@ > -26194,6 > +26196,7 @@ ix86_vector_costs::ix86_vector_costs (vec_info* vinfo, bool > costing_for_scalar) > m_num_gpr_needed (), > m_num_sse_needed (), > m_num_avx256_vec_perm (), > + m_num_avx512_vec_perm (), > m_num_reduc (), > m_prefer_unroll (true) > {} > @@ -26206,6 +26209,47 @@ ix86_vectorize_create_costs (vec_info *vinfo, > bool costing_for_scalar) > return new ix86_vector_costs (vinfo, costing_for_scalar); } > > +/* Return true if a vec_perm should be counted as a cross-lane vector > + permutation for a vector with NUNITS elements. */ static bool > +ix86_count_cross_lane_perm_p (vec_info *vinfo, slp_tree node, unsigned > +nunits) { > + /* TODO: For loop vectorization with no SLP load-permutation > + information, conservatively treat these perms as cross-lane. > + Repeated-index cases such as {0, 0, 0, 0} are emitted as > + separate vec_perm_exprs for each index, so we cannot reliably > + separate false positives from real cross-lane shuffles yet. */ > + if (!node > + || !SLP_TREE_LOAD_PERMUTATION (node).exists () > + || !is_a<bb_vec_info> (vinfo)) > + return true; > + > + unsigned half = nunits / 2; > + bool allsame = true; > + unsigned first = SLP_TREE_LOAD_PERMUTATION (node)[0]; bool > + cross_lane_p = false; > + > + for (unsigned i = 0; i != SLP_TREE_LANES (node); i++) > + { > + unsigned tmp = SLP_TREE_LOAD_PERMUTATION (node)[i]; > + /* allsame is just a broadcast. */ > + if (tmp != first) > + allsame = false; > + > + /* The load permutation can cover multiple vectors, so compare > + source and destination lanes modulo NUNITS. */ > + tmp = tmp & (nunits - 1); > + unsigned index = i & (nunits - 1); > + if ((index < half && tmp >= half) || (index >= half && tmp < half)) > + cross_lane_p = true; > + > + if (!allsame && cross_lane_p) > + return true; > + } > + > + return false; > +} > + > unsigned > ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind, > stmt_vec_info stmt_info, slp_tree node, @@ > -26771,58 +26815,25 @@ ix86_vector_costs::add_stmt_cost (int count, > vect_cost_for_stmt kind, > if (stmt_cost == -1) > stmt_cost = ix86_default_vector_cost (kind, mode); > > - if (kind == vec_perm && vectype > - && GET_MODE_SIZE (TYPE_MODE (vectype)) == 32 > - /* BIT_FIELD_REF <vect_**, 64, 0> 0 times vec_perm costs 0 in body. */ > - && count != 0) > + /* BIT_FIELD_REF <vect_**, 64, 0> with count 0 costs 0 in body. */ > + if (kind == vec_perm && vectype && count != 0) > { > - bool real_perm = true; > + unsigned vec_size = GET_MODE_SIZE (TYPE_MODE (vectype)); > unsigned nunits = TYPE_VECTOR_SUBPARTS (vectype); > + unsigned *num_vec_perm = NULL; > > - if (node > - && SLP_TREE_LOAD_PERMUTATION (node).exists () > - /* Loop vectorization will have 4 times vec_perm > - with index as {0, 0, 0, 0}. > - But it actually generates > - vec_perm_expr <vect, vect, 0, 0, 0, 0> > - vec_perm_expr <vect, vect, 1, 1, 1, 1> > - vec_perm_expr <vect, vect, 2, 2, 2, 2> > - Need to be handled separately. */ > - && is_a <bb_vec_info> (m_vinfo)) > - { > - unsigned half = nunits / 2; > - unsigned i = 0; > - bool allsame = true; > - unsigned first = SLP_TREE_LOAD_PERMUTATION (node)[0]; > - bool cross_lane_p = false; > - for (i = 0 ; i != SLP_TREE_LANES (node); i++) > - { > - unsigned tmp = SLP_TREE_LOAD_PERMUTATION (node)[i]; > - /* allsame is just a broadcast. */ > - if (tmp != first) > - allsame = false; > - > - /* 4 times vec_perm with number of lanes multiple of nunits. */ > - tmp = tmp & (nunits - 1); > - unsigned index = i & (nunits - 1); > - if ((index < half && tmp >= half) > - || (index >= half && tmp < half)) > - cross_lane_p = true; > - > - if (!allsame && cross_lane_p) > - break; > - } > - > - if (i == SLP_TREE_LANES (node)) > - real_perm = false; > - } > + if (vec_size == 32) > + num_vec_perm = m_num_avx256_vec_perm; > + else if (vec_size == 64) > + num_vec_perm = m_num_avx512_vec_perm; > > - if (real_perm) > + if (num_vec_perm && ix86_count_cross_lane_perm_p (m_vinfo, node, > + nunits)) > { > - m_num_avx256_vec_perm[where] += count; > + num_vec_perm[where] += count; > if (dump_file && (dump_flags & TDF_DETAILS)) > { > - fprintf (dump_file, "Detected avx256 cross-lane permutation: "); > + fprintf (dump_file, > + "Detected avx%u cross-lane permutation: ", vec_size * 8); > if (stmt_info) > print_gimple_expr (dump_file, stmt_info->stmt, 0, TDF_SLIM); > fprintf (dump_file, " \n"); > @@ -26945,6 +26956,10 @@ ix86_vector_costs::finish_cost (const > vector_costs *scalar_costs) > && TARGET_AVX256_AVOID_VEC_PERM) > m_costs[i] = INT_MAX; > > + for (int i = 0; i != 3; i++) > + if (m_num_avx512_vec_perm[i] && TARGET_AVX512_AVOID_VEC_PERM) > + m_costs[i] = INT_MAX; > + > /* When X86_TUNE_AVX512_TWO_EPILOGUES is enabled arrange for both > a AVX2 and a SSE epilogue for AVX512 vectorized loops. */ > if (loop_vinfo > diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h index > ac3f2213e78..a5d20c263ff 100644 > --- a/gcc/config/i386/i386.h > +++ b/gcc/config/i386/i386.h > @@ -480,6 +480,8 @@ extern unsigned char > ix86_tune_features[X86_TUNE_LAST]; > ix86_tune_features[X86_TUNE_AVX256_AVOID_VEC_PERM] > #define TARGET_AVX512_SPLIT_REGS \ > ix86_tune_features[X86_TUNE_AVX512_SPLIT_REGS] > +#define TARGET_AVX512_AVOID_VEC_PERM \ > + ix86_tune_features[X86_TUNE_AVX512_AVOID_VEC_PERM] > #define TARGET_GENERAL_REGS_SSE_SPILL \ > ix86_tune_features[X86_TUNE_GENERAL_REGS_SSE_SPILL] > #define TARGET_AVOID_MEM_OPND_FOR_CMOVE \ diff --git > a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def index > 202b38ad50b..81388cc690f 100644 > --- a/gcc/config/i386/x86-tune.def > +++ b/gcc/config/i386/x86-tune.def > @@ -638,6 +638,10 @@ DEF_TUNE > (X86_TUNE_AVX256_AVOID_VEC_PERM, DEF_TUNE > (X86_TUNE_AVX512_SPLIT_REGS, "avx512_split_regs", m_ZNVER4 > | m_C86_4G_M7) > > +/* X86_TUNE_AVX512_AVOID_VEC_PERM: Avoid using 512-bit cross-lane > + vector permutation instructions in the auto-vectorizer. */ DEF_TUNE > +(X86_TUNE_AVX512_AVOID_VEC_PERM, "avx512_avoid_vec_perm", > m_C86_4G_M7) > + > /* It's better to align MOVE_MAX with prefer_vector_width to reduce > risk of STLF stalls(small store followed by big load.) */ > /* X86_TUNE_AVX256_MOVE_BY_PIECES: Optimize move_by_pieces with > 256-bit diff --git a/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-1.c > b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-1.c > new file mode 100644 > index 00000000000..bc8226e0a22 > --- /dev/null > +++ b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-1.c > @@ -0,0 +1,27 @@ > +/* { dg-do compile } */ > +/* { dg-options "-O2 -march=c86-4g-m7 -fdump-tree-vect-details" } */ > +/* { dg-final { scan-tree-dump "loop vectorized using 32 byte vectors" > +"vect" } } */ > + > +int a[512], b[512]; > + > +void foo(void) { > + int i; > + for (i = 0; i < 32; ++i) { > + b[i * 16 + 0] = a[i * 16 + 0]; > + b[i * 16 + 1] = a[i * 16 + 0]; > + b[i * 16 + 2] = a[i * 16 + 3]; > + b[i * 16 + 3] = a[i * 16 + 5]; > + b[i * 16 + 4] = a[i * 16 + 4]; > + b[i * 16 + 5] = a[i * 16 + 6]; > + b[i * 16 + 6] = a[i * 16 + 4]; > + b[i * 16 + 7] = a[i * 16 + 9]; > + b[i * 16 + 8] = a[i * 16 + 8]; > + b[i * 16 + 9] = a[i * 16 + 8]; > + b[i * 16 + 10] = a[i * 16 + 11]; > + b[i * 16 + 11] = a[i * 16 + 13]; > + b[i * 16 + 12] = a[i * 16 + 12]; > + b[i * 16 + 13] = a[i * 16 + 14]; > + b[i * 16 + 14] = a[i * 16 + 12]; > + b[i * 16 + 15] = a[i * 16 + 14]; > + } > +} > diff --git a/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-2.c > b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-2.c > new file mode 100644 > index 00000000000..fd2dd4c1e7f > --- /dev/null > +++ b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-2.c > @@ -0,0 +1,25 @@ > +/* { dg-do compile } */ > +/* { dg-options "-march=c86-4g-m7 -O2 -fdump-tree-slp-details" } */ > +/* { dg-final { scan-tree-dump-times {(?n)Detected avx512 cross-lane > +permutation} 1 "slp2" } } */ > + > +void foo(double *a, double *__restrict b, int c, int n) { > + a[0] = b[100] * b[4]; > + a[1] = b[100] * b[5]; > + a[2] = b[100] * b[6]; > + a[3] = b[100] * b[7]; > + a[4] = b[100] * b[0]; > + a[5] = b[100] * b[1]; > + a[6] = b[100] * b[2]; > + a[7] = b[100] * b[3]; > +} > + > +void foo1(double *a, double *__restrict b, int c, int n) { > + a[0] = b[100] * b[0]; > + a[1] = b[100] * b[1]; > + a[2] = b[100] * b[3]; > + a[3] = b[100] * b[2]; > + a[4] = b[100] * b[4]; > + a[5] = b[100] * b[5]; > + a[6] = b[100] * b[7]; > + a[7] = b[100] * b[6]; > +} > diff --git a/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-3.c > b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-3.c > new file mode 100644 > index 00000000000..5988865b6b8 > --- /dev/null > +++ b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-3.c > @@ -0,0 +1,32 @@ > +/* { dg-do compile } */ > +/* { dg-options "-O2 -march=c86-4g-m7 -fdump-tree-vect-details" } */ > +/* { dg-final { scan-tree-dump "loop vectorized using 64 byte vectors" > +"vect" { xfail *-*-* } } } */ > + > +/* This case only has false cross-lane permutations, which should ideally be > + excluded from the loop vectorizer cost downgrade. The current loop-vect > + handling does not distinguish false cross-lane permutations from real > ones, > + so the expected full-width vectorization is marked as xfail. */ > + > +int a[512], b[512]; > + > +void foo(void) { > + int i; > + for (i = 0; i < 32; ++i) { > + b[i * 16 + 0] = a[i * 16 + 0]; > + b[i * 16 + 1] = a[i * 16 + 0]; > + b[i * 16 + 2] = a[i * 16 + 3]; > + b[i * 16 + 3] = a[i * 16 + 3]; > + b[i * 16 + 4] = a[i * 16 + 4]; > + b[i * 16 + 5] = a[i * 16 + 6]; > + b[i * 16 + 6] = a[i * 16 + 4]; > + b[i * 16 + 7] = a[i * 16 + 6]; > + b[i * 16 + 8] = a[i * 16 + 8]; > + b[i * 16 + 9] = a[i * 16 + 8]; > + b[i * 16 + 10] = a[i * 16 + 11]; > + b[i * 16 + 11] = a[i * 16 + 11]; > + b[i * 16 + 12] = a[i * 16 + 12]; > + b[i * 16 + 13] = a[i * 16 + 14]; > + b[i * 16 + 14] = a[i * 16 + 12]; > + b[i * 16 + 15] = a[i * 16 + 14]; > + } > +} > -- > 2.53.0
