This patch serves to add runtime support for detecting, and optionally enabling, Unified Shared Memory. It is controlled by an environment variable, 'GOMP_RUNTIME_USM', which may be set to 'disabled', 'auto', or 'enabled'. Setting it to 'disabled' will maintain current behavior, 'auto' will allow the runtime to make a decision based on support and performance considerations, and 'enabled' will trust the hardware's self-report. It currently defaults to 'disabled'.
For the 'auto' case, we currently only consider AMD integrated GPUs, referred to as APUs. We similarly only concern ourselves with those supporting XNACK. Functionally, this currently means the MI300A, although the scaffolding is there such that we should be able to adapt to later APUs. Similarly, if we ever receive a nvptx equivalent, the scaffolding is there to allow the plugin to report compatibility. Built/tested x86_64-linux-gnu.
From bd22d6c3cc355f6a55d5e010ae56706cb4d8ff82 Mon Sep 17 00:00:00 2001 From: supers1ngular <[email protected]> Date: Mon, 13 Jul 2026 03:14:43 -0700 Subject: [PATCH] libgomp: Runtime USM This patch introduces support for runtime USM, meaning that libgomp can now automatically detect and set Unified Shared Memory capabilities. It can be controlled via a new environment variable, 'GOMP_RUNTIME_USM', which can be set to 'enabled', 'auto', or 'disabled'. Documentation for the feature has also been added. libgomp/ChangeLog: * env.c (enum gomp_runtime_usm_t): New environment variable. (parse_rt_usm): Parse new environment variable. (initialize_env): Behavior handling for new variable. * libgomp-plugin.h (GOMP_OFFLOAD_CAP_AUTO_USM): New capability. (GOMP_OFFLOAD_get_dev_caps): New function for enumeration. * libgomp.h (enum gomp_device_num): Fix whitespace. (enum gomp_runtime_usm_t): Define. (struct gomp_device_descr): Add 'get_dev_caps_func'. * libgomp.texi: Update documentation. * plugin/plugin-gcn.c (get_memory_region): Fix whitespace. (process_reverse_offload): Ditto. (gcn_exec): Remove preprocessor elision. (is_apu_with_xnack): New function. (GCN_DEVICE): Query XNACK support. (GOMP_OFFLOAD_get_dev_caps): New function. * plugin/plugin-nvptx.c (GOMP_OFFLOAD_get_dev_caps): Ditto. * target.c (gomp_init_device): Logic for deciding USM behavior. (gomp_load_plugin_for_device): Get symbol. (gomp_target_init): New comment, and fix style. --- libgomp/env.c | 71 ++++++++++++++++++++++++++++++++++- libgomp/libgomp-plugin.h | 2 + libgomp/libgomp.h | 13 ++++++- libgomp/libgomp.texi | 20 ++++++++++ libgomp/plugin/plugin-gcn.c | 56 +++++++++++++++++++++++++-- libgomp/plugin/plugin-nvptx.c | 19 ++++++++++ libgomp/target.c | 29 ++++++++++++-- 7 files changed, 199 insertions(+), 11 deletions(-) diff --git a/libgomp/env.c b/libgomp/env.c index ff0044b42e2..e8727c95770 100644 --- a/libgomp/env.c +++ b/libgomp/env.c @@ -121,6 +121,8 @@ int gomp_teams_thread_limit_var; bool gomp_display_affinity_var; char *gomp_affinity_format_var = "level %L thread %i affinity %A"; size_t gomp_affinity_format_len; +enum gomp_runtime_usm_t gomp_runtime_usm_var + = GOMP_RUNTIME_USM_DISABLED; char *goacc_device_type; int goacc_device_num; int goacc_default_dims[GOMP_DIM_MAX]; @@ -1056,6 +1058,53 @@ parse_spincount (const char *name, unsigned long long *pvalue) return false; } +static bool +parse_rt_usm (const char *name, enum gomp_runtime_usm_t *val) +{ + char *env, *end; + env = getenv (name); + if (env == NULL) + return false; + end = env; + while (isspace ((unsigned char) *env)) + ++env; + if (*env == '\0') + { + gomp_error ("Invalid value for environment variable %s", name); + return false; + } + enum gomp_runtime_usm_t store_state = GOMP_RUNTIME_USM_DISABLED; + if (strncasecmp (env, "disabled", 8) == 0) + { + store_state = GOMP_RUNTIME_USM_DISABLED; + end += 8; + } + else if (strncasecmp (env, "auto", 4) == 0) + { + store_state = GOMP_RUNTIME_USM_AUTO; + end += 4; + } + else if (strncasecmp (env, "enabled", 7) == 0) + { + store_state = GOMP_RUNTIME_USM_ENABLED; + end += 7; + } + else + { + gomp_error ("Invalid value for environment variable %s", name); + return false; + } + while (isspace ((unsigned char) *end)) + ++end; + if (*end != '\0') + { + gomp_error ("Invalid value for environment variable %s", name); + return false; + } + *val = store_state; + return true; +} + /* Parse a boolean value for environment variable NAME and store the result in VALUE. Return true if one was present and it was successfully parsed. */ @@ -1974,8 +2023,21 @@ omp_display_env (int verbose) fprintf (stderr, " [host] GOMP_SPINCOUNT = '%lu'\n", (unsigned long) gomp_spin_count_var); #endif + fputs (" [device] GOMP_RUNTIME_USM = '", stderr); + switch (gomp_runtime_usm_var) + { + case GOMP_RUNTIME_USM_DISABLED: + fputs ("DISABLED", stderr); + break; + case GOMP_RUNTIME_USM_AUTO: + fputs ("AUTO", stderr); + break; + case GOMP_RUNTIME_USM_ENABLED: + fputs ("ENABLED", stderr); + break; + } + fputs ("'\n", stderr); } - fputs ("OPENMP DISPLAY ENVIRONMENT END\n", stderr); } ialias (omp_display_env) @@ -2455,6 +2517,13 @@ initialize_env (void) if (gomp_throttled_spin_count_var > gomp_spin_count_var) gomp_throttled_spin_count_var = gomp_spin_count_var; + /* If we fail, we currently default to 'disabled', both here and on + initialization of 'gomp_runtime_usm_var'. Eventually, it may + bear consideration if we wish to rather default to 'auto' - but + until then, we do not wish to surprise. */ + if (!parse_rt_usm ("GOMP_RUNTIME_USM", &gomp_runtime_usm_var)) + gomp_runtime_usm_var = GOMP_RUNTIME_USM_DISABLED; + /* Not strictly environment related, but ordering constructors is tricky. */ pthread_attr_init (&gomp_thread_attr); diff --git a/libgomp/libgomp-plugin.h b/libgomp/libgomp-plugin.h index db7c03fb350..71e40369ffa 100644 --- a/libgomp/libgomp-plugin.h +++ b/libgomp/libgomp-plugin.h @@ -50,6 +50,7 @@ extern "C" { #define GOMP_OFFLOAD_CAP_NATIVE_EXEC (1 << 1) #define GOMP_OFFLOAD_CAP_OPENMP_400 (1 << 2) #define GOMP_OFFLOAD_CAP_OPENACC_200 (1 << 3) +#define GOMP_OFFLOAD_CAP_AUTO_USM (1 << 4) /* Type of offload target device. Keep in sync with include/gomp-constants.h. */ enum offload_target_type @@ -164,6 +165,7 @@ extern int GOMP_OFFLOAD_supported_teams_dim (int, int); extern int GOMP_OFFLOAD_supported_threads_dim (int, int); extern unsigned int GOMP_OFFLOAD_get_caps (void); +extern unsigned int GOMP_OFFLOAD_get_dev_caps (int); extern int GOMP_OFFLOAD_get_type (void); extern int GOMP_OFFLOAD_get_num_devices (unsigned int); extern bool GOMP_OFFLOAD_init_device (int); diff --git a/libgomp/libgomp.h b/libgomp/libgomp.h index 3c8a2a1115e..7740508c0ac 100644 --- a/libgomp/libgomp.h +++ b/libgomp/libgomp.h @@ -494,7 +494,7 @@ enum gomp_device_num section 2.3.1. Those described as having one copy per task are stored within the structure; those described as having one copy for the whole program are (naturally) global variables. */ - + struct gomp_task_icv { unsigned long nthreads_var; @@ -590,6 +590,13 @@ enum gomp_target_offload_t GOMP_TARGET_OFFLOAD_DISABLED }; +enum gomp_runtime_usm_t +{ + GOMP_RUNTIME_USM_DISABLED, + GOMP_RUNTIME_USM_AUTO, + GOMP_RUNTIME_USM_ENABLED +}; + #define gomp_supported_active_levels UCHAR_MAX extern struct gomp_task_icv gomp_global_icv; @@ -620,6 +627,7 @@ extern const size_t gomp_omp_allocator_data_size; extern const struct gomp_default_icv gomp_default_icv_values; extern struct gomp_icv_list *gomp_initial_icv_list; extern struct gomp_offload_icv_list *gomp_offload_icv_list; +extern enum gomp_runtime_usm_t gomp_runtime_usm_var; extern int goacc_device_num; extern char *goacc_device_type; extern int goacc_default_dims[GOMP_DIM_MAX]; @@ -1380,7 +1388,7 @@ typedef struct acc_dispatch_t __typeof (GOMP_OFFLOAD_openacc_create_thread_data) *create_thread_data_func; __typeof (GOMP_OFFLOAD_openacc_destroy_thread_data) *destroy_thread_data_func; - + struct { /* Once created and put into the "active" list, asyncqueues are then never destructed and removed from the "active" list, other than if the TODO @@ -1453,6 +1461,7 @@ struct gomp_device_descr __typeof (GOMP_OFFLOAD_supported_teams_dim) *supported_teams_dim_func; __typeof (GOMP_OFFLOAD_supported_threads_dim) *supported_threads_dim_func; __typeof (GOMP_OFFLOAD_get_caps) *get_caps_func; + __typeof (GOMP_OFFLOAD_get_dev_caps) *get_dev_caps_func; __typeof (GOMP_OFFLOAD_get_type) *get_type_func; __typeof (GOMP_OFFLOAD_get_num_devices) *get_num_devices_func; __typeof (GOMP_OFFLOAD_init_device) *init_device_func; diff --git a/libgomp/libgomp.texi b/libgomp/libgomp.texi index 5d29da0984d..cc6dffa5374 100644 --- a/libgomp/libgomp.texi +++ b/libgomp/libgomp.texi @@ -4569,6 +4569,7 @@ variable is not set. * OMP_WAIT_POLICY:: How waiting threads are handled * GOMP_CPU_AFFINITY:: Bind threads to specific CPUs * GOMP_DEBUG:: Enable debugging output +* GOMP_RUNTIME_USM:: Control runtime enabling of Unified Shared Memory * GOMP_STACKSIZE:: Set default thread stack size * GOMP_SPINCOUNT:: Set the busy-wait spin count * GOMP_RTEMS_THREAD_POOLS:: Set the RTEMS specific thread pools @@ -5198,6 +5199,25 @@ This is currently not specified in more detail, and subject to change. +@node GOMP_RUNTIME_USM +@section @env{GOMP_RUNTIME_USM} -- Control runtime enabling of Unified Shared Memory +@cindex Environment Variable +@table @asis +@item @emph{Description}: +Control the behavior of automatic Unified Shared Memory (USM) in the runtime. +The default is @code{disabled}, meaning that Unified Shared Memory will +only be used if specified by the @code{requires} clause. The variable +may also be set to @code{auto}, in which the runtime will decide whether +or not to use USM based on safety and performance considerations. More +specifically, it will only utilize USM if it detects XNACK support and +an APU. Lastly, one can set the variable to @code{enabled}, which will +unconditionally trust the device's self-report. This can potentially +cause instability, as some devices report capabilities that they may not +actually have. +@end table + + + @node GOMP_STACKSIZE @section @env{GOMP_STACKSIZE} -- Set default thread stack size @cindex Environment Variable diff --git a/libgomp/plugin/plugin-gcn.c b/libgomp/plugin/plugin-gcn.c index cc47d205e1e..76f26eaf8f3 100644 --- a/libgomp/plugin/plugin-gcn.c +++ b/libgomp/plugin/plugin-gcn.c @@ -1747,7 +1747,7 @@ get_memory_region (hsa_region_t region, hsa_region_t *retval, } /* Callback of hsa_agent_iterate_regions. - + Selects a kernargs memory region. */ static hsa_status_t @@ -2238,7 +2238,7 @@ process_reverse_offload (uint64_t fn, uint64_t mapnum, uint64_t hostaddrs, We print all entries from the last item printed to the next entry without a "written" flag. If the "final" flag is set then it'll continue right to the end. - + The print buffer is circular, but the from and to locations don't wrap when the buffer does, so the output limit is UINT_MAX. The target blocks on output when the buffer is full. */ @@ -3646,7 +3646,6 @@ gcn_exec (struct kernel_info *kernel, struct gomp_offload_session *session, /* }}} */ /* {{{ Generic Plugin API */ -#if 0 /* TODO: Use to enable self-mapping/USM automatically. */ /* FIXME: The auto-self-map feature depends on still mapping 'declare target' variables, even if ignoring all other mappings. Cf. PR 115279. */ @@ -3699,7 +3698,37 @@ is_integrated_apu (struct agent_info *agent, bool check_xnack) } return is_apu; } -#endif + +static bool +is_apu_with_xnack (struct agent_info *agent) +{ + /* We do not care for non-APU targets, at the moment. + If this ever changes, we can just elide the below check. + + Another point is that we are assuming the gfx902 does not + incur performance penalties, as it reports USM and XNACK+. + If it is later shown that enabling USM by default on this + platform incurs performance issues, then we need to add + an additional carveout here. */ + if (!is_integrated_apu (agent, false)) + return false; + enum { + HSACO_ATTR_UNSUPPORTED, + HSACO_ATTR_OFF, + HSACO_ATTR_ON, + HSACO_ATTR_ANY, + HSACO_ATTR_DEFAULT + }; + + switch (agent->device_isa) + { +#define GCN_DEVICE(name, NAME, ELF, ISA, XNACK, ...) \ + case ELF: return (XNACK == HSACO_ATTR_ANY); +#include "../../gcc/config/gcn/gcn-devices.def" + default: return false; + } + return false; +} /* Return the name of the accelerator, which is "gcn". */ @@ -3809,6 +3838,25 @@ GOMP_OFFLOAD_get_caps (void) | GOMP_OFFLOAD_CAP_OPENACC_200; } + + +unsigned int +GOMP_OFFLOAD_get_dev_caps (int ord) +{ + struct agent_info *agent = get_agent_info (ord); + bool claims_usm_p = 0; + unsigned int flags = GOMP_OFFLOAD_CAP_OPENMP_400 + | GOMP_OFFLOAD_CAP_OPENACC_200; + hsa_system_info_t type = HSA_AMD_SYSTEM_INFO_SVM_ACCESSIBLE_BY_DEFAULT; + hsa_status_t status = hsa_fns.hsa_system_get_info_fn (type, &claims_usm_p); + if (status != HSA_STATUS_SUCCESS) + GOMP_PLUGIN_error ("Could not fetch SVM_ACCESSIBLE_BY_DEFAULT"); + if (claims_usm_p) + flags |= GOMP_OFFLOAD_CAP_SHARED_MEM; + if (claims_usm_p && is_apu_with_xnack (agent)) + flags |= GOMP_OFFLOAD_CAP_AUTO_USM; + return flags; +} /* Identify as GCN accelerator. */ int diff --git a/libgomp/plugin/plugin-nvptx.c b/libgomp/plugin/plugin-nvptx.c index c85ff89002e..27b86469cde 100644 --- a/libgomp/plugin/plugin-nvptx.c +++ b/libgomp/plugin/plugin-nvptx.c @@ -1434,6 +1434,25 @@ GOMP_OFFLOAD_get_caps (void) return GOMP_OFFLOAD_CAP_OPENACC_200 | GOMP_OFFLOAD_CAP_OPENMP_400; } +/* We duplicate functionality here for consistency, as this function + in plugin-gcn.c is for runtime USM. The gcn function determines + the case for auto-usm as well, however, here we only concern + ourselves for the enabled and disabled case. */ + +unsigned int +GOMP_OFFLOAD_get_dev_caps (int ord) +{ + unsigned int flags = GOMP_OFFLOAD_CAP_OPENACC_200 + | GOMP_OFFLOAD_CAP_OPENMP_400; + int pi = 0; + CUresult r; + r = CUDA_CALL_NOCHECK (cuDeviceGetAttribute, &pi, + CU_DEVICE_ATTRIBUTE_UNIFIED_ADDRESSING, ord); + if (r == CUDA_SUCCESS && pi) + flags |= GOMP_OFFLOAD_CAP_SHARED_MEM; + return flags; +} + int GOMP_OFFLOAD_get_type (void) { diff --git a/libgomp/target.c b/libgomp/target.c index e49eacc0071..4db6a7d2c39 100644 --- a/libgomp/target.c +++ b/libgomp/target.c @@ -3127,6 +3127,24 @@ gomp_init_device (struct gomp_device_descr *devicep) gomp_fatal ("device initialization failed"); } + /* Now that we have initialized the device, we can evaluate auto USM. + If the user has explicitly requested USM, we can just skip the check. */ + if (!(devicep->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)) + { + devicep->capabilities = devicep->get_dev_caps_func (devicep->target_id); + if ((gomp_runtime_usm_var == GOMP_RUNTIME_USM_AUTO) + && !(devicep->capabilities & GOMP_OFFLOAD_CAP_AUTO_USM)) + devicep->capabilities &= ~GOMP_OFFLOAD_CAP_SHARED_MEM; + if (gomp_runtime_usm_var == GOMP_RUNTIME_USM_DISABLED) + devicep->capabilities &= ~GOMP_OFFLOAD_CAP_SHARED_MEM; + } + /* Peel off the GOMP_OFFLOAD_CAP_AUTO_USM, if it was set by the plugin, as + we no longer need it. */ + devicep->capabilities &= ~GOMP_OFFLOAD_CAP_AUTO_USM; + /* We can now set the requires mask based on the capabilities. + This makes it so the runtime treats it as if the user requested USM. */ + if (devicep->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) + omp_requires_mask |= GOMP_REQUIRES_UNIFIED_SHARED_MEMORY; /* Load to device all images registered by the moment. */ for (i = 0; i < num_offload_images; i++) { @@ -3140,6 +3158,7 @@ gomp_init_device (struct gomp_device_descr *devicep) /* Initialize OpenACC asynchronous queues. */ goacc_init_asyncqueues (devicep); + gomp_debug (0, "capabilities: %d\n", devicep->capabilities); devicep->state = GOMP_DEVICE_INITIALIZED; } @@ -6295,6 +6314,7 @@ gomp_load_plugin_for_device (struct gomp_device_descr *device, DLSYM_OPT (supported_threads_dim, supported_threads_dim); DLSYM_OPT (supported_teams_dim, supported_teams_dim); DLSYM (get_caps); + DLSYM (get_dev_caps); DLSYM (get_type); DLSYM (get_num_devices); DLSYM (init_device); @@ -6320,7 +6340,8 @@ gomp_load_plugin_for_device (struct gomp_device_descr *device, DLSYM (get_interop_str); DLSYM (get_interop_type_desc); } - + /* Returns offloading capabilities, but does not say anything about + auto USM yet. */ device->capabilities = device->get_caps_func (); device->session.size = 0; if (device->capabilities & GOMP_OFFLOAD_CAP_OPENMP_400) @@ -6488,10 +6509,10 @@ gomp_target_init (void) { /* Augment DEVICES and NUM_DEVICES. */ - /* If USM has been requested and is supported by all devices - of this type, set the capability accordingly. */ + /* If USM has been requested, set the capability. */ if (omp_requires_mask - & (GOMP_REQUIRES_UNIFIED_SHARED_MEMORY | GOMP_REQUIRES_SELF_MAPS)) + & (GOMP_REQUIRES_UNIFIED_SHARED_MEMORY + | GOMP_REQUIRES_SELF_MAPS)) current_device.capabilities |= GOMP_OFFLOAD_CAP_SHARED_MEM; devs = realloc (devs, (num_devs + new_num_devs) -- 2.54.0
