https://github.com/jhuber6 updated
https://github.com/llvm/llvm-project/pull/126143
>From ea890db7533d007ed63a42775949d4b1d7d4443b Mon Sep 17 00:00:00 2001
From: Joseph Huber <hube...@outlook.com>
Date: Thu, 6 Feb 2025 15:54:19 -0600
Subject: [PATCH] [OpenMP] Remove 'libomptarget.devicertl.a' fatbinary and use
static library
Summary:
Currently, we build a single `libomptarget.devicertl.a` which is a
fatbinary. It is a host object file that contains the embedded archive
files for both the NVIDIA and AMDGPU targets. This was done primarily as
a convenience due to naming conflicts. Now that the clang driver for the
GPU targets can appropriate link via the per-target runtime-dir, we can
just make two separate static libraries and remove the indirection.
This patch creates two new static libraries that get installed into
```
/lib/amdgcn-amd-amdhsa/libomp.a
/lib/nvptx64-nvidia-cuda/libomp.a
```
for AMDGPU and NVPTX respectively. The link job created by the linker
wrapper now simply needs to do `-lomp` and it will search those
directories and link those static libraries. This requires far less
special handling.
This patch is a precursor to changing the build system entirely to be a
runtimes based one. Soon this target will be a standard `add_library`
and done through the GPU runtime targets.
NOTE that this actually does remove an additional optimization step.
Previously we merged all of the files into a single bitcode object and
forcibly internalized some definitions. This, instead, just treats them
like a normal static library. This may possibly affect performance for
some files, but I think it's better overall to use static library
semantics because it allows us to have an 'include-what-you-use'
relationship with the library.
Performance testing will be required. If we really need the merged blob
then we can simply pack that into a new static library.
---
clang/lib/Driver/ToolChains/Clang.cpp | 4 +
clang/lib/Driver/ToolChains/CommonArgs.cpp | 3 -
clang/test/Driver/openmp-offload-gpu.c | 6 +-
offload/DeviceRTL/CMakeLists.txt | 150 +++++----------------
offload/DeviceRTL/src/exports | 19 ---
offload/test/lit.cfg | 10 +-
6 files changed, 42 insertions(+), 150 deletions(-)
delete mode 100644 offload/DeviceRTL/src/exports
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp
b/clang/lib/Driver/ToolChains/Clang.cpp
index c0891d46b0a62cd..fd690ab11c1c2c3 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -9209,6 +9209,10 @@ void LinkerWrapper::ConstructJob(Compilation &C, const
JobAction &JA,
A->render(Args, LinkerArgs);
}
+ // If this is OpenMP the device linker will need `-lomp`.
+ if (Kind == Action::OFK_OpenMP && !Args.hasArg(OPT_nogpulib))
+ LinkerArgs.emplace_back("-lomp");
+
// Forward all of these to the appropriate toolchain.
for (StringRef Arg : CompilerArgs)
CmdArgs.push_back(Args.MakeArgString(
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp
b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 699aadec86dcba9..93031d2f5302386 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1289,9 +1289,6 @@ bool tools::addOpenMPRuntime(const Compilation &C,
ArgStringList &CmdArgs,
if (IsOffloadingHost)
CmdArgs.push_back("-lomptarget");
- if (IsOffloadingHost && !Args.hasArg(options::OPT_nogpulib))
- CmdArgs.push_back("-lomptarget.devicertl");
-
addArchSpecificRPath(TC, Args, CmdArgs);
addOpenMPRuntimeLibraryPath(TC, Args, CmdArgs);
diff --git a/clang/test/Driver/openmp-offload-gpu.c
b/clang/test/Driver/openmp-offload-gpu.c
index 1f7e2996068c4af..807ea2c6e5f8619 100644
--- a/clang/test/Driver/openmp-offload-gpu.c
+++ b/clang/test/Driver/openmp-offload-gpu.c
@@ -324,18 +324,18 @@
// RUN: %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp
--offload-arch=sm_52 --cuda-path=%S/Inputs/CUDA_102/usr/local/cuda \
// RUN: -foffload-lto %s 2>&1 | FileCheck --check-prefix=CHECK-LTO-LIBRARY
%s
-// CHECK-LTO-LIBRARY: {{.*}}-lomptarget{{.*}}-lomptarget.devicertl
+// CHECK-LTO-LIBRARY: --device-linker={{.*}}=-lomp{{.*}}-lomptarget{{.*}}
// RUN: %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp
--offload-arch=sm_52 --cuda-path=%S/Inputs/CUDA_102/usr/local/cuda \
// RUN:
--libomptarget-nvptx-bc-path=%S/Inputs/libomptarget/libomptarget-nvptx-test.bc
%s 2>&1 \
// RUN: | FileCheck --check-prefix=CHECK-NO-LTO-LIBRARY %s
-// CHECK-NO-LTO-LIBRARY: {{.*}}-lomptarget{{.*}}-lomptarget.devicertl
+// CHECK-NO-LTO-LIBRARY: --device-linker={{.*}}=-lomp{{.*}}-lomptarget{{.*}}
// RUN: %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp
--offload-arch=sm_52 -nogpulib \
// RUN: -foffload-lto %s 2>&1 | FileCheck --check-prefix=CHECK-NO-LIBRARY
%s
-// CHECK-NO-LIBRARY-NOT: {{.*}}-lomptarget{{.*}}-lomptarget.devicertl
+// CHECK-NO-LIBRARY-NOT: --device-linker={{.*}}=-lomp{{.*}}-lomptarget{{.*}}
// RUN: %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp
--offload-arch=sm_52 -nogpulib \
// RUN: -Xoffload-linker a -Xoffload-linker-nvptx64-nvidia-cuda b
-Xoffload-linker-nvptx64 c \
diff --git a/offload/DeviceRTL/CMakeLists.txt b/offload/DeviceRTL/CMakeLists.txt
index 8f2a1fd01fabcc8..cf26e78d6e3d448 100644
--- a/offload/DeviceRTL/CMakeLists.txt
+++ b/offload/DeviceRTL/CMakeLists.txt
@@ -16,23 +16,10 @@ if (LLVM_DIR)
# Builds that use pre-installed LLVM have LLVM_DIR set.
# A standalone or LLVM_ENABLE_RUNTIMES=openmp build takes this route
find_program(CLANG_TOOL clang PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH)
- find_program(PACKAGER_TOOL clang-offload-packager PATHS
${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH)
- find_program(LINK_TOOL llvm-link PATHS ${LLVM_TOOLS_BINARY_DIR}
NO_DEFAULT_PATH)
- find_program(OPT_TOOL opt PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH)
- if ((NOT CLANG_TOOL) OR (NOT LINK_TOOL) OR (NOT OPT_TOOL) OR (NOT
PACKAGER_TOOL))
- message(STATUS "Not building DeviceRTL. Missing clang: ${CLANG_TOOL},
llvm-link: ${LINK_TOOL}, opt: ${OPT_TOOL}, or clang-offload-packager:
${PACKAGER_TOOL}")
- return()
- else()
- message(STATUS "Building DeviceRTL. Using clang: ${CLANG_TOOL}, llvm-link:
${LINK_TOOL} and opt: ${OPT_TOOL}")
- endif()
elseif (LLVM_TOOL_CLANG_BUILD AND NOT CMAKE_CROSSCOMPILING AND NOT
OPENMP_STANDALONE_BUILD)
# LLVM in-tree builds may use CMake target names to discover the tools.
# A LLVM_ENABLE_PROJECTS=openmp build takes this route
set(CLANG_TOOL $<TARGET_FILE:clang>)
- set(PACKAGER_TOOL $<TARGET_FILE:clang-offload-packager>)
- set(LINK_TOOL $<TARGET_FILE:llvm-link>)
- set(OPT_TOOL $<TARGET_FILE:opt>)
- message(STATUS "Building DeviceRTL. Using clang from in-tree build")
else()
message(STATUS "Not building DeviceRTL. No appropriate clang found")
return()
@@ -82,8 +69,6 @@ set(src_files
# propagation. That said, we will run the vectorizer again after the runtime
# has been linked into the user program.
set(clang_opt_flags -O3 -mllvm -openmp-opt-disable
-DSHARED_SCRATCHPAD_SIZE=512 -mllvm -vectorize-slp=false )
-set(link_opt_flags -O3 -openmp-opt-disable -attributor-enable=module
-vectorize-slp=false )
-set(link_export_flag -passes=internalize
-internalize-public-api-file=${source_directory}/exports)
# If the user built with the GPU C library enabled we will use that instead.
if(${LIBOMPTARGET_GPU_LIBC_SUPPORT})
@@ -107,15 +92,15 @@ set(bc_flags -c -flto -std=c++17 -fvisibility=hidden
)
# first create an object target
-add_library(omptarget.devicertl.all_objs OBJECT IMPORTED)
function(compileDeviceRTLLibrary target_name target_triple)
set(target_bc_flags ${ARGN})
set(bc_files "")
+ add_library(omp.${target_name}.all_objs OBJECT IMPORTED)
foreach(src ${src_files})
get_filename_component(infile ${src} ABSOLUTE)
get_filename_component(outfile ${src} NAME)
- set(outfile "${outfile}-${target_name}.bc")
+ set(outfile "${outfile}-${target_name}.o")
set(depfile "${outfile}.d")
# Passing an empty CPU to -march= suppressed target specific metadata.
@@ -142,99 +127,36 @@ function(compileDeviceRTLLibrary target_name
target_triple)
endif()
set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES
${outfile})
- list(APPEND bc_files ${outfile})
+ list(APPEND obj_files ${CMAKE_CURRENT_BINARY_DIR}/${outfile})
endforeach()
-
- set(bclib_name "libomptarget-${target_name}.bc")
-
- # Link to a bitcode library.
- add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name}
- COMMAND ${LINK_TOOL}
- -o ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name} ${bc_files}
- DEPENDS ${bc_files}
- COMMENT "Linking LLVM bitcode ${bclib_name}"
- )
-
- if(TARGET llvm-link)
- add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name}
- DEPENDS llvm-link
- APPEND)
- endif()
-
- add_custom_command(OUTPUT
${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name}
- COMMAND ${OPT_TOOL} ${link_export_flag}
${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name}
- -o ${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name}
- DEPENDS ${source_directory}/exports
${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name}
- COMMENT "Internalizing LLVM bitcode ${bclib_name}"
+ set_property(TARGET omp.${target_name}.all_objs
+ APPEND PROPERTY IMPORTED_OBJECTS ${obj_files})
+
+ # Archive all the object files generated above into a static library
+ add_library(omp.${target_name} STATIC)
+ set_target_properties(omp.${target_name} PROPERTIES
+ ARCHIVE_OUTPUT_DIRECTORY
"${LIBOMPTARGET_LLVM_LIBRARY_INTDIR}/${target_triple}"
+ ARCHIVE_OUTPUT_NAME omp
+ LINKER_LANGUAGE CXX
)
- if(TARGET opt)
- add_custom_command(OUTPUT
${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name}
- DEPENDS opt
- APPEND)
- endif()
-
- add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name}
- COMMAND ${OPT_TOOL} ${link_opt_flags}
${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name}
- -o ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name}
- DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name}
- COMMENT "Optimizing LLVM bitcode ${bclib_name}"
- )
- if(TARGET opt)
- add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name}
- DEPENDS opt
- APPEND)
- endif()
-
- set(bclib_target_name "omptarget-${target_name}-bc")
- add_custom_target(${bclib_target_name} DEPENDS
${CMAKE_CURRENT_BINARY_DIR}/${bclib_name})
-
- # Copy library to destination.
- add_custom_command(TARGET ${bclib_target_name} POST_BUILD
- COMMAND ${CMAKE_COMMAND} -E copy
${CMAKE_CURRENT_BINARY_DIR}/${bclib_name}
- ${LIBOMPTARGET_LIBRARY_DIR})
- add_dependencies(omptarget.devicertl.${target_name} ${bclib_target_name})
-
- set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES
${bclib_name} ${LIBOMPTARGET_LIBRARY_DIR}/${bclib_name})
-
- # Install bitcode library under the lib destination folder.
- install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name} DESTINATION
"${OFFLOAD_INSTALL_LIBDIR}")
-
- set(target_feature "")
- if("${target_triple}" STREQUAL "nvptx64-nvidia-cuda")
- set(target_feature "feature=+ptx63")
- endif()
-
- # Package the bitcode in the bitcode and embed it in an ELF for the static
library
- add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/packaged_${bclib_name}
- COMMAND ${PACKAGER_TOOL} -o
${CMAKE_CURRENT_BINARY_DIR}/packaged_${bclib_name}
-
"--image=file=${CMAKE_CURRENT_BINARY_DIR}/${bclib_name},${target_feature},triple=${target_triple},arch=generic,kind=openmp"
- DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name}
- COMMENT "Packaging LLVM offloading binary ${bclib_name}.out"
- )
- if(TARGET clang-offload-packager)
- add_custom_command(OUTPUT
${CMAKE_CURRENT_BINARY_DIR}/packaged_${bclib_name}
- DEPENDS clang-offload-packager
- APPEND)
- endif()
-
- set(output_name "${CMAKE_CURRENT_BINARY_DIR}/devicertl-${target_name}.o")
- add_custom_command(OUTPUT ${output_name}
- COMMAND ${CLANG_TOOL} --std=c++17 -c -nostdlib
- -Xclang
-fembed-offload-object=${CMAKE_CURRENT_BINARY_DIR}/packaged_${bclib_name}
- -o ${output_name}
- ${source_directory}/Stub.cpp
- DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/packaged_${bclib_name}
${source_directory}/Stub.cpp
- COMMENT "Embedding LLVM offloading binary in devicertl-${target_name}.o"
- VERBATIM
- )
- if(TARGET clang)
- add_custom_command(OUTPUT ${output_name}
- DEPENDS clang
- APPEND)
- endif()
-
- set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES
${output_name})
- set_property(TARGET omptarget.devicertl.all_objs APPEND PROPERTY
IMPORTED_OBJECTS ${output_name})
+ target_link_libraries(omp.${target_name} PRIVATE omp.${target_name}.all_objs)
+
+ install(TARGETS omp.${target_name}
+ ARCHIVE DESTINATION "lib${LLVM_LIBDIR_SUFFIX}/${target_triple}")
+
+ # Trick to combine these into a bitcode file via the linker's LTO pass. This
+ # is used to provide the legacy `libomptarget-<name>.bc` files.
+ add_executable(libomptarget-${target_name} ${obj_files})
+ set_target_properties(libomptarget-${target_name} PROPERTIES
+ RUNTIME_OUTPUT_DIRECTORY ${LIBOMPTARGET_LLVM_LIBRARY_INTDIR}
+ LINKER_LANGUAGE CXX
+ RUNTIME_OUTPUT_NAME libomptarget-${target_name}.bc)
+ target_compile_options(libomptarget-${target_name} PRIVATE
"--target=${target_triple}")
+ target_link_options(libomptarget-${target_name} PRIVATE
"--target=${target_triple}"
+ "-r" "-nostdlib" "-flto" "-Wl,--lto-emit-llvm")
+ install(TARGETS libomptarget-${target_name}
+ PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ
+ DESTINATION ${OFFLOAD_INSTALL_LIBDIR})
if (CMAKE_EXPORT_COMPILE_COMMANDS)
set(ide_target_name omptarget-ide-${target_name})
@@ -254,18 +176,6 @@ function(compileDeviceRTLLibrary target_name target_triple)
endif()
endfunction()
-add_custom_target(omptarget.devicertl.amdgpu)
compileDeviceRTLLibrary(amdgpu amdgcn-amd-amdhsa -Xclang
-mcode-object-version=none)
-add_custom_target(omptarget.devicertl.nvptx)
compileDeviceRTLLibrary(nvptx nvptx64-nvidia-cuda --cuda-feature=+ptx63)
-
-# Archive all the object files generated above into a static library
-add_library(omptarget.devicertl STATIC)
-set_target_properties(omptarget.devicertl PROPERTIES
- ARCHIVE_OUTPUT_DIRECTORY "${LIBOMPTARGET_LLVM_LIBRARY_INTDIR}"
- LINKER_LANGUAGE CXX
-)
-target_link_libraries(omptarget.devicertl PRIVATE omptarget.devicertl.all_objs)
-
-install(TARGETS omptarget.devicertl ARCHIVE DESTINATION
${OFFLOAD_INSTALL_LIBDIR})
diff --git a/offload/DeviceRTL/src/exports b/offload/DeviceRTL/src/exports
deleted file mode 100644
index 01667e7aba827a3..000000000000000
--- a/offload/DeviceRTL/src/exports
+++ /dev/null
@@ -1,19 +0,0 @@
-omp_*
-ompx_*
-*llvm_*
-__kmpc_*
-
-__omp_rtl_debug_kind
-__omp_rtl_assume_no_thread_state
-__omp_rtl_assume_no_nested_parallelism
-
-_ZN4ompx*
-
-IsSPMDMode
-
-malloc
-free
-memcmp
-printf
-__llvm_rpc_client
-__assert_fail
diff --git a/offload/test/lit.cfg b/offload/test/lit.cfg
index 658ae5f9653ba90..92ef16446a1c5a6 100644
--- a/offload/test/lit.cfg
+++ b/offload/test/lit.cfg
@@ -164,7 +164,7 @@ else: # Unices
if config.cuda_libdir:
config.test_flags += " -Wl,-rpath," + config.cuda_libdir
if config.libomptarget_current_target.startswith('nvptx'):
- config.test_flags_clang += " --libomptarget-nvptx-bc-path=" +
config.library_dir + '/DeviceRTL'
+ config.test_flags_clang += " --libomptarget-nvptx-bc-path=" +
config.llvm_library_intdir
if config.libomptarget_current_target.endswith('-LTO'):
config.test_flags += " -foffload-lto"
if config.libomptarget_current_target.endswith('-JIT-LTO') and
evaluate_bool_env(
@@ -183,11 +183,11 @@ def remove_suffix_if_present(name):
def add_libraries(source):
if config.libomptarget_has_libc:
- return source + " -Xoffload-linker " + "-lc " + \
- "-Xoffload-linker " + "-lm " + \
- config.llvm_library_intdir + "/libomptarget.devicertl.a"
+ return source + " -Xoffload-linker -lc " + \
+ "-Xoffload-linker -lm " + \
+ "-Xoffload-linker -lomp "
else:
- return source + " " + config.llvm_library_intdir +
"/libomptarget.devicertl.a"
+ return source + " " + "-Xoffload-lnker -lomp"
# Add platform targets
host_targets = [
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
