commit: bb46b0f6b4260c8aca367dc153b10a385d9728eb Author: Alexey Chernov <4ernov <AT> gmail <DOT> com> AuthorDate: Mon Feb 10 18:51:53 2020 +0000 Commit: Benda XU <heroxbd <AT> gentoo <DOT> org> CommitDate: Wed Feb 19 14:35:57 2020 +0000 URL: https://gitweb.gentoo.org/proj/sci.git/commit/?id=bb46b0f6
sci-libs/pytorch: initial commit. This is CPU version only. ROCm and CUDA options will be added later. Reference: https://github.com/aclex/pytorch-ebuild Bug: https://github.com/gentoo/sci/pull/941 Package-Manager: Portage-2.3.88, Repoman-2.3.18 Signed-off-by: Benda Xu <heroxbd <AT> gentoo.org> ...-t-build-libtorch-again-for-PyTorch-1.4.0.patch | 30 +++ ...path-to-caffe2-build-dir-made-by-libtorch.patch | 46 ++++ ...rpath-of-Caffe2-library-for-system-wide-i.patch | 104 ++++++++ ...ibrary-directory-according-to-CMake-build.patch | 31 +++ ...conversion-ambiguity-in-ternary-operators.patch | 38 +++ sci-libs/pytorch/files/pytorch-1.2.0-setup.patch | 24 ++ sci-libs/pytorch/files/pytorch-1.4.0-eigen.patch | 13 + .../files/pytorch-1.4.0-nccl-nvccflags.patch | 28 +++ .../pytorch/files/pytorch-1.4.0-skip-tests.patch | 28 +++ sci-libs/pytorch/files/pytorch-1.4.0-sleef.patch | 15 ++ sci-libs/pytorch/metadata.xml | 35 +++ sci-libs/pytorch/pytorch-1.4.0_p0.ebuild | 272 +++++++++++++++++++++ 12 files changed, 664 insertions(+) diff --git a/sci-libs/pytorch/files/0002-Don-t-build-libtorch-again-for-PyTorch-1.4.0.patch b/sci-libs/pytorch/files/0002-Don-t-build-libtorch-again-for-PyTorch-1.4.0.patch new file mode 100644 index 000000000..de9fe9a0d --- /dev/null +++ b/sci-libs/pytorch/files/0002-Don-t-build-libtorch-again-for-PyTorch-1.4.0.patch @@ -0,0 +1,30 @@ +From e5003b88ad3a504eb04c0ba66c2f439149de2ee3 Mon Sep 17 00:00:00 2001 +From: Alexey Chernov <[email protected]> +Date: Tue, 5 Nov 2019 00:58:17 +0300 +Subject: [PATCH 2/7] Don't build `libtorch` again for PyTorch + +If starting with building of `libtorch` using CMake, +don't build it again for the sake of running `setup.py` +to build Python version (PyTorch). +--- + setup.py | 7 ------- + 1 file changed, 7 deletions(-) + +Index: pytorch-1.4.0_p0/setup.py +=================================================================== +--- pytorch-1.4.0_p0.orig/setup.py ++++ pytorch-1.4.0_p0/setup.py +@@ -303,13 +303,6 @@ def build_deps(): + check_pydep('yaml', 'pyyaml') + check_pydep('typing', 'typing') + +- build_caffe2(version=version, +- cmake_python_library=cmake_python_library, +- build_python=True, +- rerun_cmake=RERUN_CMAKE, +- cmake_only=CMAKE_ONLY, +- cmake=cmake) +- + version_path = os.path.join(cwd, 'torch', 'version.py') + with open(version_path, 'w') as f: + f.write("__version__ = '{}'\n".format(version)) diff --git a/sci-libs/pytorch/files/0003-Change-path-to-caffe2-build-dir-made-by-libtorch.patch b/sci-libs/pytorch/files/0003-Change-path-to-caffe2-build-dir-made-by-libtorch.patch new file mode 100644 index 000000000..f71ba17bc --- /dev/null +++ b/sci-libs/pytorch/files/0003-Change-path-to-caffe2-build-dir-made-by-libtorch.patch @@ -0,0 +1,46 @@ +From dbb0bac77986016217cdfe779499a2ccd0e10570 Mon Sep 17 00:00:00 2001 +From: Alexey Chernov <[email protected]> +Date: Wed, 6 Nov 2019 00:16:24 +0300 +Subject: [PATCH 3/4] Change path to `caffe2` build dir made by `libtorch` + +Setup and use properly `caffe2` build dir made while +`libtorch` building with CMake, so that it was used +by the Python installation. +--- + setup.py | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/setup.py b/setup.py +index f4bfdc99cd..60502b6add 100644 +--- a/setup.py ++++ b/setup.py +@@ -236,7 +236,7 @@ else: + cwd = os.path.dirname(os.path.abspath(__file__)) + lib_path = os.path.join(cwd, "torch", "lib") + third_party_path = os.path.join(cwd, "third_party") +-caffe2_build_dir = os.path.join(cwd, "build") ++caffe2_build_dir = os.environ.get("CMAKE_BUILD_DIR", os.path.join(cwd, "build")) + # lib/pythonx.x/site-packages + rel_site_packages = distutils.sysconfig.get_python_lib(prefix='') + # full absolute path to the dir above +@@ -279,7 +279,7 @@ elif sha != 'Unknown': + version += '+' + sha[:7] + report("Building wheel {}-{}".format(package_name, version)) + +-cmake = CMake() ++cmake = CMake(caffe2_build_dir) + + # all the work we need to do _before_ setup runs + def build_deps(): +@@ -445,7 +445,7 @@ class build_ext(setuptools.command.build_ext.build_ext): + filename = self.get_ext_filename(fullname) + report("\nCopying extension {}".format(ext.name)) + +- src = os.path.join("torch", rel_site_packages, filename) ++ src = os.path.join(caffe2_build_dir, filename) + if not os.path.exists(src): + report("{} does not exist".format(src)) + del self.extensions[i] +-- +2.23.0 + diff --git a/sci-libs/pytorch/files/0004-Don-t-fill-rpath-of-Caffe2-library-for-system-wide-i.patch b/sci-libs/pytorch/files/0004-Don-t-fill-rpath-of-Caffe2-library-for-system-wide-i.patch new file mode 100644 index 000000000..c8f57d9e6 --- /dev/null +++ b/sci-libs/pytorch/files/0004-Don-t-fill-rpath-of-Caffe2-library-for-system-wide-i.patch @@ -0,0 +1,104 @@ +From a0ace7295b6895954d2a0f8d6fe311a34a7c4e85 Mon Sep 17 00:00:00 2001 +From: Alexey Chernov <[email protected]> +Date: Tue, 12 Nov 2019 02:01:36 +0300 +Subject: [PATCH 4/4] Don't fill rpath of Caffe2 library for system-wide + install + +Don't fill in runtime path of Caffe2 library for the case of +building for system-wide installation. +--- + CMakeLists.txt | 3 +++ + caffe2/CMakeLists.txt | 18 +++++++++++++++--- + cmake/Dependencies.cmake | 19 +++++++++++-------- + 3 files changed, 29 insertions(+), 11 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 20923c0544..45a882ae0e 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -193,6 +193,9 @@ option(USE_TBB "Use TBB" OFF) + # Used when building Caffe2 through setup.py + option(BUILDING_WITH_TORCH_LIBS "Tell cmake if Caffe2 is being built alongside torch libs" ON) + ++# Used when building Caffe2 for further system-wide installation ++option(BUILDING_SYSTEM_WIDE "Tell cmake if Caffe2 is being built for system-wide installation" OFF) ++ + # /Z7 override option + # When generating debug symbols, CMake default to use the flag /Zi. + # However, it is not compatible with sccache. So we rewrite it off. +diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt +index 1f7d037243..30d5615cdb 100644 +--- a/caffe2/CMakeLists.txt ++++ b/caffe2/CMakeLists.txt +@@ -1235,7 +1235,11 @@ if (BUILD_PYTHON) + if (MSVC AND BUILD_SHARED_LIBS) + install(FILES $<TARGET_PDB_FILE:caffe2_pybind11_state> DESTINATION "${PYTHON_LIB_REL_PATH}/caffe2/python" OPTIONAL) + endif() +- set_target_properties(caffe2_pybind11_state PROPERTIES INSTALL_RPATH "${caffe2_pybind11_rpath}") ++ if (NOT BUILDING_SYSTEM_WIDE) ++ set_target_properties(caffe2_pybind11_state PROPERTIES INSTALL_RPATH "${caffe2_pybind11_rpath}") ++else() ++ set_target_properties(caffe2_pybind11_state PROPERTIES SKIP_BUILD_RPATH ON) ++ endif() + + if(USE_CUDA) + add_library(caffe2_pybind11_state_gpu MODULE ${Caffe2_GPU_PYTHON_SRCS}) +@@ -1261,7 +1265,11 @@ if (BUILD_PYTHON) + if (MSVC AND BUILD_SHARED_LIBS) + install(FILES $<TARGET_PDB_FILE:caffe2_pybind11_state_gpu> DESTINATION "${PYTHON_LIB_REL_PATH}/caffe2/python" OPTIONAL) + endif() +- set_target_properties(caffe2_pybind11_state_gpu PROPERTIES INSTALL_RPATH "${caffe2_pybind11_rpath}") ++ if (NOT BUILDING_SYSTEM_WIDE) ++ set_target_properties(caffe2_pybind11_state_gpu PROPERTIES INSTALL_RPATH "${caffe2_pybind11_rpath}") ++ else() ++ set_target_properties(caffe2_pybind11_state_gpu PROPERTIES SKIP_BUILD_RPATH ON) ++ endif() + endif() + + if(USE_ROCM) +@@ -1284,7 +1292,11 @@ if (BUILD_PYTHON) + caffe2_pybind11_state_hip PROPERTIES LIBRARY_OUTPUT_DIRECTORY + ${CMAKE_BINARY_DIR}/caffe2/python) + install(TARGETS caffe2_pybind11_state_hip DESTINATION "${PYTHON_LIB_REL_PATH}/caffe2/python") +- set_target_properties(caffe2_pybind11_state_hip PROPERTIES INSTALL_RPATH "${caffe2_pybind11_rpath}") ++ if (NOT BUILDING_SYSTEM_WIDE) ++ set_target_properties(caffe2_pybind11_state_hip PROPERTIES INSTALL_RPATH "${caffe2_pybind11_rpath}") ++ else() ++ set_target_properties(caffe2_pybind11_state_hip PROPERTIES SKIP_BUILD_RPATH ON) ++ endif() + endif() + + if (MSVC AND CMAKE_GENERATOR MATCHES "Visual Studio") +diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake +index a8e9769536..6ef9d7b0e2 100644 +--- a/cmake/Dependencies.cmake ++++ b/cmake/Dependencies.cmake +@@ -6,14 +6,17 @@ if (APPLE) + else() + set(_rpath_portable_origin $ORIGIN) + endif(APPLE) +-# Use separate rpaths during build and install phases +-set(CMAKE_SKIP_BUILD_RPATH FALSE) +-# Don't use the install-rpath during the build phase +-set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) +-set(CMAKE_INSTALL_RPATH "${_rpath_portable_origin}") +-# Automatically add all linked folders that are NOT in the build directory to +-# the rpath (per library?) +-set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) ++ ++if (NOT BUILDING_SYSTEM_WIDE) ++ # Use separate rpaths during build and install phases ++ set(CMAKE_SKIP_BUILD_RPATH FALSE) ++ # Don't use the install-rpath during the build phase ++ set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) ++ set(CMAKE_INSTALL_RPATH "${_rpath_portable_origin}") ++ # Automatically add all linked folders that are NOT in the build directory to ++ # the rpath (per library?) ++ set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) ++endif() + + # UBSAN triggers when compiling protobuf, so we need to disable it. + set(UBSAN_FLAG "-fsanitize=undefined") +-- +2.23.0 + diff --git a/sci-libs/pytorch/files/0005-Change-library-directory-according-to-CMake-build.patch b/sci-libs/pytorch/files/0005-Change-library-directory-according-to-CMake-build.patch new file mode 100644 index 000000000..324b8f682 --- /dev/null +++ b/sci-libs/pytorch/files/0005-Change-library-directory-according-to-CMake-build.patch @@ -0,0 +1,31 @@ +From 52019a3f395e5fa97b26d424152d91f73b400f8e Mon Sep 17 00:00:00 2001 +From: Alexey Chernov <[email protected]> +Date: Wed, 13 Nov 2019 23:44:12 +0300 +Subject: [PATCH 5/5] Change library directory according to CMake build + +Change `lib_path` in favour of out-of-tree CMake build +directory, so that all the C++ libraries be found. +--- + setup.py | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/setup.py b/setup.py +index 60502b6add..cbced52192 100644 +--- a/setup.py ++++ b/setup.py +@@ -234,9 +234,10 @@ else: + + # Constant known variables used throughout this file + cwd = os.path.dirname(os.path.abspath(__file__)) +-lib_path = os.path.join(cwd, "torch", "lib") ++cmake_build_dir = os.environ.get("CMAKE_BUILD_DIR", os.path.join(cwd, "build")) ++lib_path = os.path.join(cmake_build_dir, "lib") + third_party_path = os.path.join(cwd, "third_party") +-caffe2_build_dir = os.environ.get("CMAKE_BUILD_DIR", os.path.join(cwd, "build")) ++caffe2_build_dir = cmake_build_dir + # lib/pythonx.x/site-packages + rel_site_packages = distutils.sysconfig.get_python_lib(prefix='') + # full absolute path to the dir above +-- +2.23.0 + diff --git a/sci-libs/pytorch/files/0010-Remove-conversion-ambiguity-in-ternary-operators.patch b/sci-libs/pytorch/files/0010-Remove-conversion-ambiguity-in-ternary-operators.patch new file mode 100644 index 000000000..49835d201 --- /dev/null +++ b/sci-libs/pytorch/files/0010-Remove-conversion-ambiguity-in-ternary-operators.patch @@ -0,0 +1,38 @@ +From 0f3a0d9a948fdd8481a5f7751a255ecd9ed4d37a Mon Sep 17 00:00:00 2001 +From: Alexey Chernov <[email protected]> +Date: Mon, 20 Jan 2020 01:21:22 +0300 +Subject: [PATCH 3/3] Remove conversion ambiguity in ternary operators + +It fails to compile with recent versions of Clang +(namely, version 10 used behind HCC compiler when +being built with ROCm support) with the +`conditional expression is ambiguous` error. +--- + caffe2/operators/relu_op.cu | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/caffe2/operators/relu_op.cu b/caffe2/operators/relu_op.cu +index f6edf7105e..ab49540fc9 100644 +--- a/caffe2/operators/relu_op.cu ++++ b/caffe2/operators/relu_op.cu +@@ -51,7 +51,7 @@ __global__ void ReluCUDAKernel<half2>(const int N, const half2* X, half2* Y) { + #else + const float2 xx = __half22float2(X[i]); + Y[i] = +- __floats2half2_rn(xx.x > 0.0f ? xx.x : 0.0f, xx.y > 0.0f ? xx.y : 0.0f); ++ __floats2half2_rn(xx.x > 0.0f ? float(xx.x) : 0.0f, xx.y > 0.0f ? float(xx.y) : 0.0f); + #endif + } + } +@@ -101,7 +101,7 @@ __global__ void ReluGradientCUDAKernel<half2>( + const float2 dy = __half22float2(dY[i]); + const float2 yy = __half22float2(Y[i]); + dX[i] = +- __floats2half2_rn(yy.x > 0.0f ? dy.x : 0.0f, yy.y > 0.0f ? dy.y : 0.0f); ++ __floats2half2_rn(yy.x > 0.0f ? float(dy.x) : 0.0f, yy.y > 0.0f ? float(dy.y) : 0.0f); + #endif + } + } +-- +2.24.1 + diff --git a/sci-libs/pytorch/files/pytorch-1.2.0-setup.patch b/sci-libs/pytorch/files/pytorch-1.2.0-setup.patch new file mode 100644 index 000000000..85e796a60 --- /dev/null +++ b/sci-libs/pytorch/files/pytorch-1.2.0-setup.patch @@ -0,0 +1,24 @@ +Index: pytorch-1.4.0_p0/setup.py +=================================================================== +--- pytorch-1.4.0_p0.orig/setup.py ++++ pytorch-1.4.0_p0/setup.py +@@ -288,9 +288,7 @@ def build_deps(): + sys.exit(1) + + check_file(os.path.join(third_party_path, "gloo", "CMakeLists.txt")) +- check_file(os.path.join(third_party_path, "pybind11", "CMakeLists.txt")) + check_file(os.path.join(third_party_path, 'cpuinfo', 'CMakeLists.txt')) +- check_file(os.path.join(third_party_path, 'tbb', 'Makefile')) + check_file(os.path.join(third_party_path, 'onnx', 'CMakeLists.txt')) + check_file(os.path.join(third_party_path, 'foxi', 'CMakeLists.txt')) + check_file(os.path.join(third_party_path, 'QNNPACK', 'CMakeLists.txt')) +@@ -341,9 +339,6 @@ def build_deps(): + if not same: + shutil.copyfile(orig_file, sym_file) + +- dir_util.copy_tree('third_party/pybind11/include/pybind11/', +- 'torch/include/pybind11') +- + ################################################################################ + # Building dependent libraries + ################################################################################ diff --git a/sci-libs/pytorch/files/pytorch-1.4.0-eigen.patch b/sci-libs/pytorch/files/pytorch-1.4.0-eigen.patch new file mode 100644 index 000000000..54ea3a5e0 --- /dev/null +++ b/sci-libs/pytorch/files/pytorch-1.4.0-eigen.patch @@ -0,0 +1,13 @@ +Index: pytorch-1.4.0/CMakeLists.txt +=================================================================== +--- pytorch-1.4.0.orig/CMakeLists.txt ++++ pytorch-1.4.0/CMakeLists.txt +@@ -173,7 +173,7 @@ option(USE_REDIS "Use Redis" OFF) + option(USE_ROCKSDB "Use RocksDB" OFF) + option(USE_SNPE "Use Qualcomm's SNPE library" OFF) + option(USE_SYSTEM_EIGEN_INSTALL +- "Use system Eigen instead of the one under third_party" OFF) ++ "Use system Eigen instead of the one under third_party" ON) + option(USE_TENSORRT "Using Nvidia TensorRT library" OFF) + option(USE_ZMQ "Use ZMQ" OFF) + option(USE_ZSTD "Use ZSTD" OFF) diff --git a/sci-libs/pytorch/files/pytorch-1.4.0-nccl-nvccflags.patch b/sci-libs/pytorch/files/pytorch-1.4.0-nccl-nvccflags.patch new file mode 100644 index 000000000..165e79dd4 --- /dev/null +++ b/sci-libs/pytorch/files/pytorch-1.4.0-nccl-nvccflags.patch @@ -0,0 +1,28 @@ +Index: nccl-7c72dee660e4d055b81721dd6b03e4e1c0a983cf/makefiles/common.mk +=================================================================== +--- nccl-7c72dee660e4d055b81721dd6b03e4e1c0a983cf.orig/makefiles/common.mk ++++ nccl-7c72dee660e4d055b81721dd6b03e4e1c0a983cf/makefiles/common.mk +@@ -46,7 +46,7 @@ endif + CXXFLAGS := -DCUDA_MAJOR=$(CUDA_MAJOR) -DCUDA_MINOR=$(CUDA_MINOR) -fPIC -fvisibility=hidden + CXXFLAGS += -Wall -Wno-unused-function -Wno-sign-compare -std=c++11 -Wvla + CXXFLAGS += -I $(CUDA_INC) +-NVCUFLAGS := -ccbin $(CXX) $(NVCC_GENCODE) -lineinfo -std=c++11 -Xptxas -maxrregcount=96 -Xfatbin -compress-all ++NVCUFLAGS := $(NVCCFLAGS) $(NVCC_GENCODE) -lineinfo -std=c++11 -Xptxas -maxrregcount=96 -Xfatbin -compress-all + # Use addprefix so that we can specify more than one path + NVLDFLAGS := -L${CUDA_LIB} -lcudart -lrt + +@@ -60,14 +60,6 @@ NVLDFLAGS += ${GCOV_FLAGS:%=-Xcompiler + # $(warning GCOV_FLAGS=${GCOV_FLAGS}) + ########## GCOV ########## + +-ifeq ($(DEBUG), 0) +-NVCUFLAGS += -O3 +-CXXFLAGS += -O3 -g +-else +-NVCUFLAGS += -O0 -G -g +-CXXFLAGS += -O0 -g -ggdb3 +-endif +- + ifneq ($(VERBOSE), 0) + NVCUFLAGS += -Xptxas -v -Xcompiler -Wall,-Wextra,-Wno-unused-parameter + CXXFLAGS += -Wall -Wextra diff --git a/sci-libs/pytorch/files/pytorch-1.4.0-skip-tests.patch b/sci-libs/pytorch/files/pytorch-1.4.0-skip-tests.patch new file mode 100644 index 000000000..3682469d7 --- /dev/null +++ b/sci-libs/pytorch/files/pytorch-1.4.0-skip-tests.patch @@ -0,0 +1,28 @@ +Disable expected test failures. + +TCPStoreTest is too slow to run. + +Reference: https://github.com/pytorch/pytorch/issues/13746 +Reference: https://github.com/pytorch/pytorch/issues/21873 + +Index: pytorch-1.4.0_p0/torch/lib/c10d/test/CMakeLists.txt +=================================================================== +--- pytorch-1.4.0_p0.orig/torch/lib/c10d/test/CMakeLists.txt ++++ pytorch-1.4.0_p0/torch/lib/c10d/test/CMakeLists.txt +@@ -14,7 +14,6 @@ function(c10d_add_test test_src) + endfunction() + + c10d_add_test(FileStoreTest.cpp c10d) +-c10d_add_test(TCPStoreTest.cpp c10d) + + if(USE_CUDA) + if(USE_C10D_GLOO) +Index: pytorch-1.4.0_p0/caffe2/share/contrib/depthwise/CMakeLists.txt +=================================================================== +--- pytorch-1.4.0_p0.orig/caffe2/share/contrib/depthwise/CMakeLists.txt ++++ pytorch-1.4.0_p0/caffe2/share/contrib/depthwise/CMakeLists.txt +@@ -6,4 +6,3 @@ set(Caffe2_CONTRIB_DEPTHWISE3x3_TEST_CPU + ) + + set(Caffe2_CPU_SRCS ${Caffe2_CPU_SRCS} ${Caffe2_CONTRIB_DEPTHWISE3x3_CPU_SRC} PARENT_SCOPE) +-set(Caffe2_CPU_TEST_SRCS ${Caffe2_CPU_TEST_SRCS} ${Caffe2_CONTRIB_DEPTHWISE3x3_TEST_CPU_SRC} PARENT_SCOPE) diff --git a/sci-libs/pytorch/files/pytorch-1.4.0-sleef.patch b/sci-libs/pytorch/files/pytorch-1.4.0-sleef.patch new file mode 100644 index 000000000..222a0f73e --- /dev/null +++ b/sci-libs/pytorch/files/pytorch-1.4.0-sleef.patch @@ -0,0 +1,15 @@ +Give the location of sleef.h. + +Reference: https://github.com/pytorch/pytorch/issues/26555 +Index: pytorch-1.4.0/caffe2/CMakeLists.txt +=================================================================== +--- pytorch-1.4.0.orig/caffe2/CMakeLists.txt ++++ pytorch-1.4.0/caffe2/CMakeLists.txt +@@ -1062,6 +1062,7 @@ if (BUILD_TEST) + add_executable(${test_name} "${test_src}") + target_link_libraries(${test_name} ${Caffe2_MAIN_LIBS} gtest_main) + target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>) ++ target_include_directories(${test_name} PRIVATE $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/include>) + target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE}) + add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>) + if (INSTALL_TEST) diff --git a/sci-libs/pytorch/metadata.xml b/sci-libs/pytorch/metadata.xml new file mode 100644 index 000000000..78e46cc35 --- /dev/null +++ b/sci-libs/pytorch/metadata.xml @@ -0,0 +1,35 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE pkgmetadata SYSTEM "http://www.gentoo.org/dtd/metadata.dtd";> +<pkgmetadata> + <maintainer type="person"> + <email>[email protected]</email> + <name>Alexey Chernov</name> + </maintainer> + <use> + <flag name="asan">Build with address sanitizer (ASAN)</flag> + <flag name="atlas">Use ATLAS as BLAS implementation</flag> + <flag name="cuda">Enable support for nVidia CUDA</flag> + <flag name="eigen">Use Eigen as BLAS implementation</flag> + <flag name="fbgemm">Enable FBGEMM (Facebook GEneral Matrix Multiplication) support</flag> + <flag name="gflags">Build with gflags</flag> + <flag name="glog">Build with glog support</flag> + <flag name="gloo">Build gloo distributed communication backend support</flag> + <flag name="leveldb">Enable support for LevelDB</flag> + <flag name="lmdb">Enable support for LMDB</flag> + <flag name="mkl">Use Intel MKL as BLAS implementation</flag> + <flag name="mkldnn">Build with Intel MKL-DNN support</flag> + <flag name="namedtensor">Enable support for named tensors</flag> + <flag name="nnpack">Enable NNPACK support</flag> + <flag name="numa">Enable support for NUMA</flag> + <flag name="numpy">Enable NumPy support</flag> + <flag name="observers">Build observers</flag> + <flag name="openblas">Use OpenBLAS as BLAS implementation</flag> + <flag name="opencv">Enable support for OpenCV</flag> + <flag name="opencl">Enable support for OpenCL</flag> + <flag name="qnnpack">Enable QNNPACK support</flag> + <flag name="redis">Enable support for Redis</flag> + <flag name="tbb">Build with Intel TBB enabled</flag> + <flag name="tools">Build binary tools</flag> + <flag name="zeromq">Enable support for ZeroMQ</flag> + </use> +</pkgmetadata> diff --git a/sci-libs/pytorch/pytorch-1.4.0_p0.ebuild b/sci-libs/pytorch/pytorch-1.4.0_p0.ebuild new file mode 100644 index 000000000..06ee528f2 --- /dev/null +++ b/sci-libs/pytorch/pytorch-1.4.0_p0.ebuild @@ -0,0 +1,272 @@ +# Copyright 1999-2020 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +EAPI=7 + +PYTHON_COMPAT=( python3_6 ) + +inherit distutils-r1 cmake-utils cuda + +MPV=${PV/_p/a} + +DESCRIPTION="Tensors and Dynamic neural networks in Python with strong GPU acceleration" +HOMEPAGE="https://pytorch.org/"; +SRC_URI="https://github.com/pytorch/${PN}/archive/v${MPV}.tar.gz -> ${P}.tar.gz +https://github.com/facebookincubator/gloo/archive/ca528e32.tar.gz -> gloo-ca528e32.tar.gz +https://github.com/google/benchmark/archive/505be96a.tar.gz -> benchmark-505be96a.tar.gz +https://github.com/google/gemmlowp/archive/3fb5c176.tar.gz -> gemmlowp-3fb5c176.tar.gz +https://github.com/google/googletest/archive/2fe3bd99.tar.gz -> googletest-2fe3bd99.tar.gz +https://github.com/houseroad/foxi/archive/8f74bc4d.tar.gz -> foxi-8f74bc4d.tar.gz +https://github.com/intel/ideep/archive/78eafa5d.tar.gz -> ideep-78eafa5d.tar.gz +https://github.com/Maratyszcza/FP16/archive/febbb1c1.tar.gz -> FP16-febbb1c1.tar.gz +https://github.com/Maratyszcza/FXdiv/archive/b742d114.tar.gz -> FXdiv-b742d114.tar.gz +https://github.com/Maratyszcza/NNPACK/archive/c039579a.tar.gz -> NNPACK-c039579a.tar.gz +https://github.com/Maratyszcza/PeachPy/archive/07d8fde8.tar.gz -> PeachPy-07d8fde8.tar.gz +https://github.com/Maratyszcza/psimd/archive/90a938f3.tar.gz -> psimd-90a938f3.tar.gz +https://github.com/Maratyszcza/pthreadpool/archive/13da0b4c.tar.gz -> pthreadpool-13da0b4c.tar.gz +cuda? ( https://github.com/NVIDIA/nccl/archive/7c72dee6.tar.gz -> nccl-7c72dee6.tar.gz ) +https://github.com/NVlabs/cub/archive/285aeeba.tar.gz -> cub-285aeeba.tar.gz +https://github.com/onnx/onnx/archive/2891e145.tar.gz -> onnx-2891e145.tar.gz +https://github.com/onnx/onnx-tensorrt/archive/cb3d8066.tar.gz -> onnx-tensorrt-cb3d8066.tar.gz +https://github.com/pytorch/cpuinfo/archive/89fe1695.tar.gz -> cpuinfo-89fe1695.tar.gz +https://github.com/pytorch/fbgemm/archive/82d259da.tar.gz -> fbgemm-82d259da.tar.gz +https://github.com/pytorch/QNNPACK/archive/7d2a4e99.tar.gz -> QNNPACK-7d2a4e99.tar.gz +https://github.com/shibatch/sleef/archive/7f523de6.tar.gz -> sleef-7f523de6.tar.gz +https://github.com/asmjit/asmjit/archive/17556b2d.tar.gz -> asmjit-17556b2d.tar.gz + " + +LICENSE="BSD" +SLOT="0" +KEYWORDS="~amd64" + +IUSE="asan atlas cuda doc eigen +fbgemm ffmpeg gflags glog +gloo leveldb lmdb mkl mkldnn mpi namedtensor +nnpack numa +numpy +observers +openblas opencl opencv +openmp +python +qnnpack redis static tbb test tools zeromq" + +REQUIRED_USE=" + python? ( ${PYTHON_REQUIRED_USE} ) + numpy? ( python ) + ^^ ( atlas eigen mkl openblas ) +" + +DEPEND=" + dev-libs/protobuf + dev-python/pyyaml[${PYTHON_USEDEP}] + virtual/python-typing[${PYTHON_USEDEP}] + atlas? ( sci-libs/atlas ) + cuda? ( dev-util/nvidia-cuda-toolkit:0= ) + doc? ( dev-python/pytorch-sphinx-theme[${PYTHON_USEDEP}] ) + ffmpeg? ( virtual/ffmpeg ) + gflags? ( dev-cpp/gflags ) + glog? ( dev-cpp/glog ) + leveldb? ( dev-libs/leveldb ) + lmdb? ( dev-db/lmdb ) + mkl? ( sci-libs/mkl ) + mpi? ( virtual/mpi ) + numpy? ( dev-python/numpy[${PYTHON_USEDEP}] ) + openblas? ( sci-libs/openblas ) + opencl? ( dev-libs/clhpp virtual/opencl ) + opencv? ( media-libs/opencv ) + python? ( ${PYTHON_DEPS} + dev-python/pybind11[${PYTHON_USEDEP}] + ) + redis? ( dev-db/redis ) + zeromq? ( net-libs/zeromq ) + eigen? ( dev-cpp/eigen + cuda? ( dev-cpp/eigen[cuda] ) ) +" +RDEPEND="${DEPEND}" +BDEPEND=" + doc? ( app-doc/doxygen ) +" + +DEPEND=" + test? ( dev-python/pytest[${PYTHON_USEDEP}] ) + dev-python/setuptools[${PYTHON_USEDEP}] + dev-cpp/tbb + app-arch/zstd + dev-python/protobuf-python[${PYTHON_USEDEP}] + dev-python/pybind11[${PYTHON_USEDEP}] + sys-fabric/libibverbs + sys-process/numactl +" + +PATCHES=( + "${FILESDIR}"/${PN}-1.2.0-setup.patch + "${FILESDIR}"/${PN}-1.4.0-eigen.patch + "${FILESDIR}"/${PN}-1.4.0-sleef.patch + "${FILESDIR}"/${PN}-1.4.0-skip-tests.patch + "${FILESDIR}"/0002-Don-t-build-libtorch-again-for-PyTorch-1.4.0.patch + "${FILESDIR}"/0003-Change-path-to-caffe2-build-dir-made-by-libtorch.patch + "${FILESDIR}"/0005-Change-library-directory-according-to-CMake-build.patch +) + +src_unpack() { + default + + mv -v ${PN}-${MPV} ${P} || die +} + +src_prepare() { + cmake-utils_src_prepare + + mv -v third_party/miniz-* ../ || die + rm -r third_party || die + ln -s .. third_party || die + cd .. || die + for d in *; do + case ${d} in + ${PN}* | miniz-*) continue ;; + PeachPy-*) mv -v ${d} python-peachpy || die ;; + *) mv -v ${d} ${d%-*} || die ;; + esac + done + + mv -v FBGEMM fbgemm || die + cd fbgemm || die + rm -r third_party || die + ln -s .. third_party || die + + cd ../onnx || die + rm -r third_party || die + ln -s .. third_party || die + + if use cuda; then + cd ../nccl || die + eapply "${FILESDIR}"/${PN}-1.4.0-nccl-nvccflags.patch + ln -s . nccl || die + + cuda_src_prepare + export CUDAHOSTCXX=$(cuda_gccdir)/g++ + fi +} + +src_configure() { + local blas="Eigen" + + if use atlas; then + blas="ATLAS" + elif use mkl; then + blas="MKL" + elif use openblas; then + blas="OpenBLAS" + fi + + local mycmakeargs=( + -DTORCH_BUILD_VERSION=${PV} + -DTORCH_INSTALL_LIB_DIR=$(get_libdir) + -DBUILD_BINARY=$(usex tools ON OFF) + -DBUILD_CUSTOM_PROTOBUF=OFF + -DBUILD_DOCS=$(usex doc ON OFF) + -DBUILD_PYTHON=$(usex python ON OFF) + -DBUILD_SHARED_LIBS=$(usex static OFF ON) + -DBUILD_TEST=$(usex test ON OFF) + -DUSE_ASAN=$(usex asan ON OFF) + -DUSE_CUDA=$(usex cuda ON OFF) + -DUSE_NCCL=$(usex cuda ON OFF) + -DUSE_ROCM=OFF + -DUSE_FBGEMM=$(usex fbgemm ON OFF) + -DUSE_FFMPEG=$(usex ffmpeg ON OFF) + -DUSE_GFLAGS=$(usex gflags ON OFF) + -DUSE_GLOG=$(usex glog ON OFF) + -DUSE_LEVELDB=$(usex leveldb ON OFF) + -DUSE_LITE_PROTO=OFF + -DUSE_LMDB=$(usex lmdb ON OFF) + -DCAFFE2_USE_MKL=$(usex mkl ON OFF) + -DUSE_MKLDNN=$(usex mkldnn ON OFF) + -DUSE_MKLDNN_CBLAS=OFF + -DUSE_NCCL=OFF + -DUSE_NNPACK=$(usex nnpack ON OFF) + -DUSE_NUMPY=$(usex numpy ON OFF) + -DUSE_NUMA=$(usex numa ON OFF) + -DUSE_OBSERVERS=$(usex observers ON OFF) + -DUSE_OPENCL=$(usex opencl ON OFF) + -DUSE_OPENCV=$(usex opencv ON OFF) + -DUSE_OPENMP=$(usex openmp ON OFF) + -DUSE_TBB=OFF + -DUSE_PROF=OFF + -DUSE_QNNPACK=$(usex qnnpack ON OFF) + -DUSE_REDIS=$(usex redis ON OFF) + -DUSE_ROCKSDB=OFF + -DUSE_ZMQ=$(usex zeromq ON OFF) + -DUSE_MPI=$(usex mpi ON OFF) + -DUSE_GLOO=$(usex gloo ON OFF) + -DBUILD_NAMEDTENSOR=$(usex namedtensor ON OFF) + -DBLAS=${blas} + -DBUILDING_SYSTEM_WIDE=ON # to remove insecure DT_RUNPATH header + ) + + cmake-utils_src_configure + + if use python; then + CMAKE_BUILD_DIR=${BUILD_DIR} distutils-r1_src_configure + fi +} + +src_compile() { + cmake-utils_src_compile + + if use python; then + CMAKE_BUILD_DIR=${BUILD_DIR} distutils-r1_src_compile + fi +} + +src_install() { + cmake-utils_src_install + + local multilib_failing_files=( + libc10.so + libtbb.so + libcaffe2_observers.so + libshm.so + libcaffe2_detectron_ops.so + ) + + local LIB=$(get_libdir) + if [[ ${LIB} != lib ]]; then + for file in ${multilib_failing_files[@]}; do + mv -fv "${ED}/usr/lib/$file" "${ED}/usr/${LIB}" + done + fi + + rm -rfv "${ED}/torch" + rm -rfv "${ED}/var" + rm -rfv "${ED}/usr/lib" + + rm -fv "${ED}/usr/include/*.{h,hpp}" + rm -rfv "${ED}/usr/include/asmjit" + rm -rfv "${ED}/usr/include/c10d" + rm -rfv "${ED}/usr/include/fbgemm" + rm -rfv "${ED}/usr/include/fp16" + rm -rfv "${ED}/usr/include/gloo" + rm -rfv "${ED}/usr/include/include" + rm -rfv "${ED}/usr/include/var" + + cp -rv "${WORKDIR}/${P}/third_party/pybind11/include/pybind11" "${ED}/usr/include/" + + rm -fv "${ED}/usr/lib64/libtbb.so" + rm -rfv "${ED}/usr/lib64/cmake" + + rm -rfv "${ED}/usr/share/doc/mkldnn" + + if use python; then + install_shm_manager() { + TORCH_BIN_DIR="${ED}/usr/lib64/${EPYTHON}/site-packages/torch/bin" + + mkdir -pv ${TORCH_BIN_DIR} + cp -v "${ED}/usr/bin/torch_shm_manager" "${TORCH_BIN_DIR}" + } + + python_foreach_impl install_shm_manager + + scanelf -r --fix "${BUILD_DIR}/caffe2/python" + CMAKE_BUILD_DIR=${BUILD_DIR} distutils-r1_src_install + + python_foreach_impl python_optimize + fi + + find "${ED}/usr/lib64" -name "*.a" -exec rm -fv {} \; + + use test && rm -rfv "${ED}/usr/test" "${ED}"/usr/bin/test_{api,jit} + + # Remove the empty directories by CMake Python: + find "${ED}" -type d -empty -delete || die +}
