From 46968fce3960772ddbefb9e62bde7512fd8fb8b3 Mon Sep 17 00:00:00 2001 From: Kugan Vivekanandarajah Date: Tue, 11 Nov 2025 18:00:08 -0800 Subject: [PATCH 1/4] [Autofdo] Add hierarchical discriminator support This patch introduces the basic infrastructure for hierarchical discriminators with format [Base:8][Multiplicity:7][CopyID:11][Unused:6]. It adds helper functions to create and extract discriminator components. gcc/ChangeLog: * Makefile.in: Add hierarchical_discriminator.o to OBJS. * hierarchical_discriminator.cc: New file. * hierarchical_discriminator.h: New file. * input.cc (location_with_discriminator_components): New function. (get_discriminator_components_from_loc): Likewise. * input.h (DISCR_BASE_BITS): New constant. (DISCR_MULTIPLICITY_BITS): Likewise. (DISCR_COPYID_BITS): Likewise. (DISCR_UNUSED_BITS): Likewise. (DISCR_BASE_MASK): Likewise. (DISCR_MULTIPLICITY_MASK): Likewise. (DISCR_COPYID_MASK): Likewise. (DISCR_BASE_SHIFT): Likewise. (DISCR_MULTIPLICITY_SHIFT): Likewise. (DISCR_COPYID_SHIFT): Likewise. (DISCR_BASE_MAX): Likewise. (DISCR_MULTIPLICITY_MAX): Likewise. (DISCR_COPYID_MAX): Likewise. (location_with_discriminator_components): New function declaration. (get_discriminator_components_from_loc): Likewise. Signed-off-by: Kugan Vivekanandarajah --- gcc/Makefile.in | 1 + gcc/hierarchical_discriminator.cc | 108 ++++++++++++++++++++++++++++++ gcc/hierarchical_discriminator.h | 80 ++++++++++++++++++++++ gcc/input.cc | 35 ++++++++++ gcc/input.h | 40 +++++++++++ 5 files changed, 264 insertions(+) create mode 100644 gcc/hierarchical_discriminator.cc create mode 100644 gcc/hierarchical_discriminator.h diff --git a/gcc/Makefile.in b/gcc/Makefile.in index 5c24a9aab00..26278c5d143 100644 --- a/gcc/Makefile.in +++ b/gcc/Makefile.in @@ -1456,6 +1456,7 @@ OBJS = \ dce.o \ ddg.o \ debug.o \ + hierarchical_discriminator.o \ dep-fusion.o \ df-core.o \ df-problems.o \ diff --git a/gcc/hierarchical_discriminator.cc b/gcc/hierarchical_discriminator.cc new file mode 100644 index 00000000000..cb9f55b9624 --- /dev/null +++ b/gcc/hierarchical_discriminator.cc @@ -0,0 +1,108 @@ +/* Copyright The GNU Toolchain Authors + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 3, or (at your option) any later +version. + +GCC is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with GCC; see the file COPYING3. If not see +. */ + +#include "config.h" +#include "system.h" +#include "coretypes.h" +#include "backend.h" +#include "tree.h" +#include "gimple.h" +#include "tree-pass.h" +#include "ssa.h" +#include "gimple-iterator.h" +#include "tree-cfg.h" +#include "cfgloop.h" +#include "hierarchical_discriminator.h" +#include "cfghooks.h" + +/* Assign discriminators to all statements in a basic block. This + function updates the multiplicity and/or copyid discriminator components for + all statements in the given basic block, while preserving the base + discriminator. */ + +void +assign_discriminators_to_bb (basic_block bb, + unsigned int multiplicity_value, + unsigned int copyid_value, + bool update_multiplicity, + bool update_copyid) +{ + gimple_stmt_iterator gsi; + + for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) + { + gimple *stmt = gsi_stmt (gsi); + location_t loc = gimple_location (stmt); + + if (loc == UNKNOWN_LOCATION || is_gimple_debug (stmt)) + continue; + + /* Get existing discriminator components. */ + unsigned int base, multiplicity, copyid; + get_discriminator_components_from_loc (loc, &base, + &multiplicity, ©id); + + /* Update requested components. */ + if (update_multiplicity) + multiplicity = multiplicity_value; + if (update_copyid) + copyid = copyid_value; + + /* Set new location. */ + location_t new_loc = location_with_discriminator_components (loc, base, + multiplicity, + copyid); + gimple_set_location (stmt, new_loc); + } +} + +/* Assign discriminators to all basic blocks in a loop. This function is + used by loop versioning passes to assign version IDs and vectorization + factors to all statements in a loop version. + + multiplicity_value: vectorization factor (0 to keep unchanged) + copyid_value: version ID (1=vectorized, 2=scalar, etc.) */ + +void +assign_discriminators_to_loop (class loop *loop, + unsigned int multiplicity_value, + unsigned int copyid_value) +{ + basic_block *bbs; + unsigned int i; + bool update_multiplicity = (multiplicity_value != 0); + bool update_copyid = (copyid_value != 0); + + /* Validate parameters are in valid ranges. */ + if (update_multiplicity) + gcc_assert (multiplicity_value <= DISCR_MULTIPLICITY_MAX); + if (update_copyid) + gcc_assert (copyid_value <= DISCR_COPYID_MAX); + + /* Get all basic blocks in the loop. */ + bbs = get_loop_body (loop); + + /* Assign discriminators to all blocks in the loop. */ + for (i = 0; i < loop->num_nodes; i++) + assign_discriminators_to_bb (bbs[i], multiplicity_value, copyid_value, + update_multiplicity, update_copyid); + + free (bbs); +} + + diff --git a/gcc/hierarchical_discriminator.h b/gcc/hierarchical_discriminator.h new file mode 100644 index 00000000000..6605f9fcfad --- /dev/null +++ b/gcc/hierarchical_discriminator.h @@ -0,0 +1,80 @@ +/* Copyright The GNU Toolchain Authors + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 3, or (at your option) any later +version. + +GCC is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with GCC; see the file COPYING3. If not see +. */ + +#include "config.h" +#include "system.h" +#include "coretypes.h" +#ifndef GCC_HIERARCHICAL_DISCRIMINATOR_H +#define GCC_HIERARCHICAL_DISCRIMINATOR_H + +#include "gimple.h" +#include "tree.h" +#include "basic-block.h" +#include "input.h" + +/* Hierarchical discriminator layout (32 bits total): + Discriminator format: [Base:8][Multiplicity:7][CopyID:11][Unused:6] + - Base: bits 0-7 (8 bits, 0-255) + - Multiplicity: bits 8-14 (7 bits, 0-127) + - CopyID: bits 15-25 (11 bits, 0-2047) + - Unused: bits 26-31 (6 bits, reserved) + + Base discriminator: Used by front-end and early passes to distinguish + different statements on the same source line. + + Multiplicity: Duplication factor for unrolling/vectorization. + Represents how many times code is duplicated: + - Loop unrolling factor + - Vectorization width + + CopyID: Unique identifier for code copies to distinguish: + - Inlining contexts (different callsites) + - Function cloning + + Note: Loop unrolling and versioning use multiplicity, not copyid. + */ + +/* Loop versioning discriminators (CopyID values). */ +#define DISCRIMINATOR_LOOP_VERSION_VECTORIZED 1 /* Vectorized version. */ +#define DISCRIMINATOR_LOOP_VERSION_SCALAR 2 /* Scalar version. */ +#define DISCRIMINATOR_LOOP_VERSION_ALIGNED 3 /* Aligned version. */ +#define DISCRIMINATOR_LOOP_VERSION_UNALIGNED 4 /* Unaligned version. */ +#define DISCRIMINATOR_LOOP_PROLOG 6 /* Prolog loop. */ +#define DISCRIMINATOR_LOOP_EPILOG 7 /* Epilog loop. */ +#define DISCRIMINATOR_LOOP_EPILOG_VECTORIZED 8 /* Vector epilog loop. */ + +/* Helper function to assign discriminators to all statements in a basic + block. This preserves the base discriminator and only updates the + requested components. */ +extern void assign_discriminators_to_bb (basic_block bb, + unsigned int multiplicity_value, + unsigned int copyid_value, + bool update_multiplicity, + bool update_copyid); + +/* Helper function to assign discriminators to all basic blocks in a loop. + This is used by loop versioning passes to distinguish different versions + of the same loop and to indicate vectorization factors. + + multiplicity_value: vectorization factor (0 to keep unchanged) + copyid_value: version ID (1=vectorized, 2=scalar, etc.) */ +extern void assign_discriminators_to_loop (class loop *loop, + unsigned int multiplicity_value, + unsigned int copyid_value); + +#endif /* GCC_HIERARCHICAL_DISCRIMINATOR_H. */ diff --git a/gcc/input.cc b/gcc/input.cc index aad98394711..520ae33add8 100644 --- a/gcc/input.cc +++ b/gcc/input.cc @@ -1074,6 +1074,41 @@ get_discriminator_from_loc (location_t locus) return get_discriminator_from_loc (line_table, locus); } +/* Create a location with hierarchical discriminator components. */ + +location_t +location_with_discriminator_components (location_t locus, + unsigned int base, + unsigned int multiplicity, + unsigned int copyid) +{ + gcc_assert (base <= DISCR_BASE_MAX); + gcc_assert (multiplicity <= DISCR_MULTIPLICITY_MAX); + gcc_assert (copyid <= DISCR_COPYID_MAX); + unsigned int discriminator = (base << DISCR_BASE_SHIFT) + | (multiplicity << DISCR_MULTIPLICITY_SHIFT) + | (copyid << DISCR_COPYID_SHIFT); + return location_with_discriminator (locus, discriminator); +} + +/* Get hierarchical discriminator components from a location. */ + +void +get_discriminator_components_from_loc (location_t locus, + unsigned int *base, + unsigned int *multiplicity, + unsigned int *copyid) +{ + unsigned int discriminator = get_discriminator_from_loc (locus); + if (base) + *base = discriminator & DISCR_BASE_MASK; + if (multiplicity) + *multiplicity = (discriminator >> DISCR_MULTIPLICITY_SHIFT) + & DISCR_MULTIPLICITY_MASK; + if (copyid) + *copyid = (discriminator >> DISCR_COPYID_SHIFT) & DISCR_COPYID_MASK; +} + #if CHECKING_P namespace selftest { diff --git a/gcc/input.h b/gcc/input.h index 4d2d7741592..8683058e77a 100644 --- a/gcc/input.h +++ b/gcc/input.h @@ -89,6 +89,46 @@ extern location_t location_with_discriminator (location_t, int); extern bool has_discriminator (location_t); extern int get_discriminator_from_loc (location_t); +/* Hierarchical discriminator support for AutoFDO. +Discriminator format: [Base:8][Multiplicity:7][CopyID:11][Unused:6] +- Base discriminator (bits 0-7): Distinguishes instructions at same line +- Multiplicity (bits 8-14): Duplication factor for unrolling/vectorization +- CopyID (bits 15-25): Unique identifier for code copies +- Unused (bits 26-31): Reserved. */ + +/* Discriminator bit layout constants. */ +#define DISCR_BASE_BITS 8 +#define DISCR_MULTIPLICITY_BITS 7 +#define DISCR_COPYID_BITS 11 +#define DISCR_UNUSED_BITS 6 + +#define DISCR_BASE_MASK ((1u << DISCR_BASE_BITS) - 1) +#define DISCR_MULTIPLICITY_MASK ((1u << DISCR_MULTIPLICITY_BITS) - 1) +#define DISCR_COPYID_MASK ((1u << DISCR_COPYID_BITS) - 1) + +#define DISCR_BASE_SHIFT 0 +#define DISCR_MULTIPLICITY_SHIFT DISCR_BASE_BITS +#define DISCR_COPYID_SHIFT (DISCR_BASE_BITS + DISCR_MULTIPLICITY_BITS) + +/* Maximum values for each discriminator field. */ +#define DISCR_BASE_MAX DISCR_BASE_MASK +#define DISCR_MULTIPLICITY_MAX DISCR_MULTIPLICITY_MASK +#define DISCR_COPYID_MAX DISCR_COPYID_MASK + +/* Create location with hierarchical discriminator. */ +extern location_t +location_with_discriminator_components (location_t, + unsigned int base, + unsigned int multiplicity, + unsigned int copyid); + +/* Get discriminator components from location. */ +extern void +get_discriminator_components_from_loc (location_t, + unsigned int *base, + unsigned int *multiplicity, + unsigned int *copyid); + #define LOCATION_FILE(LOC) ((expand_location (LOC)).file) #define LOCATION_LINE(LOC) ((expand_location (LOC)).line) #define LOCATION_COLUMN(LOC)((expand_location (LOC)).column) -- 2.34.1