From 5a59f075b4424c7cb4c807bf1c13437f204da577 Mon Sep 17 00:00:00 2001 From: Marco de Abreu Date: Wed, 28 Nov 2018 20:28:26 +0100 Subject: [PATCH] Add direct softlinks for compiler in local bin --- .gitignore | 4 - Makefile | 6 +- ci/docker/install/centos7_adduser.sh | 5 ++ ci/docker/install/ubuntu_adduser.sh | 5 ++ ci/docker/runtime_functions.sh | 129 ++++++++++++++------------- 5 files changed, 81 insertions(+), 68 deletions(-) diff --git a/.gitignore b/.gitignore index c8a813649bb8..7b8ae55e5dd9 100644 --- a/.gitignore +++ b/.gitignore @@ -167,10 +167,6 @@ python/.eggs tests/Makefile tests/mxnet_unit_tests -# generated wrappers for ccache -cc -cxx - # Code coverage related .coverage *.gcov diff --git a/Makefile b/Makefile index 16ea59f3d585..11f2c7953bc9 100644 --- a/Makefile +++ b/Makefile @@ -463,7 +463,7 @@ build/src/%.o: src/%.cc | mkldnn build/src/%_gpu.o: src/%.cu | mkldnn @mkdir -p $(@D) - $(NVCC) $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS)" -M -MT build/src/$*_gpu.o $< >build/src/$*_gpu.d + $(NVCC) $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS)" --generate-dependencies -MT build/src/$*_gpu.o $< >build/src/$*_gpu.d $(NVCC) -c -o $@ $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS)" $< # A nvcc bug cause it to generate "generic/xxx.h" dependencies from torch headers. @@ -479,7 +479,7 @@ build/plugin/%.o: plugin/%.cc %_gpu.o: %.cu @mkdir -p $(@D) - $(NVCC) $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS) -Isrc/operator" -M -MT $*_gpu.o $< >$*_gpu.d + $(NVCC) $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS) -Isrc/operator" --generate-dependencies -MT $*_gpu.o $< >$*_gpu.d $(NVCC) -c -o $@ $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS) -Isrc/operator" $< %.o: %.cc $(CORE_INC) @@ -660,7 +660,7 @@ jnilint: rclean: $(RM) -r R-package/src/image_recordio.h R-package/NAMESPACE R-package/man R-package/R/mxnet_generated.R \ - R-package/inst R-package/src/*.o R-package/src/*.so mxnet_*.tar.gz + R-package/inst R-package/src/*.o R-package/src/*.so mxnet_*.tar.gz deps ifneq ($(EXTRA_OPERATORS),) clean: rclean cyclean $(EXTRA_PACKAGES_CLEAN) diff --git a/ci/docker/install/centos7_adduser.sh b/ci/docker/install/centos7_adduser.sh index ba72c9b92281..f9d2402c9554 100755 --- a/ci/docker/install/centos7_adduser.sh +++ b/ci/docker/install/centos7_adduser.sh @@ -34,4 +34,9 @@ then mkdir /work/mxnet mkdir /work/build chown -R jenkins_slave /work/ + + # Later on, we have to override the links because underlying build systems ignore our compiler settings. Thus, + # we have to give the process the proper permission to these files. This is hacky, but unfortunately + # there's no better way to do this without patching all our submodules. + chown -R jenkins_slave /usr/local/bin fi diff --git a/ci/docker/install/ubuntu_adduser.sh b/ci/docker/install/ubuntu_adduser.sh index 515a80f63b07..a7668bac2ab6 100755 --- a/ci/docker/install/ubuntu_adduser.sh +++ b/ci/docker/install/ubuntu_adduser.sh @@ -40,4 +40,9 @@ then mkdir /work/mxnet mkdir /work/build chown -R jenkins_slave /work/ + + # Later on, we have to override the links because underlying build systems ignore our compiler settings. Thus, + # we have to give the process the proper permission to these files. This is hacky, but unfortunately + # there's no better way to do this without patching all our submodules. + chown -R jenkins_slave /usr/local/bin fi diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 1fc10bf0e085..654f78e2fa66 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -36,35 +36,37 @@ clean_repo() { git submodule update --init --recursive } -build_ccache_wrappers() { - set -ex - - rm -f cc - rm -f cxx - - touch cc - touch cxx - - if [ -z ${CC+x} ]; then - echo "No \$CC set, defaulting to gcc"; - export CC=gcc - fi - - if [ -z ${CXX+x} ]; then - echo "No \$CXX set, defaulting to g++"; - export CXX=g++ - fi - - # this function is nessesary for cuda enabled make based builds, since nvcc needs just an executable for -ccbin - - echo -e "#!/bin/sh\n/usr/local/bin/ccache ${CC} \"\$@\"\n" >> cc - echo -e "#!/bin/sh\n/usr/local/bin/ccache ${CXX} \"\$@\"\n" >> cxx - - chmod +x cc - chmod +x cxx - - export CC=`pwd`/cc - export CXX=`pwd`/cxx +install_ccache_wrappers() { + set -ex + + # Recommended by CCache: https://ccache.samba.org/manual.html#_run_modes + # Add to the beginning of path to ensure this redirection is picked up instead + # of the original ones. Especially CUDA/NVCC appends itself to the beginning of the + # path and thus this redirect is ignored. This change fixes this problem + # This hacky approach with symbolic links is required because underlying build + # systems of our submodules ignore our CMake settings. If they use Makefile, + # we can't influence them at all in general and NVCC also prefers to hardcode their + # compiler instead of respecting the settings. Thus, we take this brutal approach + # and just redirect everything of this installer has been called. + # In future, we could do these links during image build time of the container. + # But in the beginning, we'll make this opt-in. In future, loads of processes like + # the scala make step or numpy compilation and other pip package generations + # could be heavily sped up by using ccache as well. + export PATH=/usr/local/bin:$PATH + ln -s ccache /usr/local/bin/gcc + ln -s ccache /usr/local/bin/g++ + ln -s ccache /usr/local/bin/g++-8 + ln -s ccache /usr/local/bin/gcc-8 + ln -s ccache /usr/local/bin/nvcc + ln -s ccache /usr/local/bin/clang++-3.9 + ln -s ccache /usr/local/bin/clang-3.9 + ln -s ccache /usr/local/bin/clang++-5.0 + ln -s ccache /usr/local/bin/clang-5.0 + ln -s ccache /usr/local/bin/clang++-6.0 + ln -s ccache /usr/local/bin/clang-6.0 + + # TODO: Remove + export CCACHE_LOGFILE=/work/mxnet/ccache-log } build_wheel() { @@ -128,7 +130,6 @@ build_armv6() { # to be linked additionally. # We do not need OpenMP, since most armv6 systems have only 1 core - cmake \ -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ @@ -227,6 +228,8 @@ build_android_armv8() { cd /work/build cmake\ -DANDROID=ON \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DUSE_CUDA=OFF\ -DUSE_SSE=OFF\ -DUSE_LAPACK=OFF\ @@ -242,9 +245,8 @@ build_android_armv8() { build_centos7_cpu() { set -ex cd /work/mxnet - export CC="ccache gcc" - export CXX="ccache g++" + install_ccache_wrappers make \ DEV=1 \ USE_LAPACK=1 \ @@ -257,6 +259,7 @@ build_centos7_cpu() { build_amzn_linux_cpu() { cd /work/build + install_ccache_wrappers cmake \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ @@ -277,9 +280,8 @@ build_amzn_linux_cpu() { build_centos7_mkldnn() { set -ex cd /work/mxnet - export CC="ccache gcc" - export CXX="ccache g++" - + + install_ccache_wrappers make \ DEV=1 \ ENABLE_TESTCOVERAGE=1 \ @@ -293,8 +295,7 @@ build_centos7_mkldnn() { build_centos7_gpu() { set -ex cd /work/mxnet - # unfortunately this build has problems in 3rdparty dependencies with ccache and make - # build_ccache_wrappers + install_ccache_wrappers make \ DEV=1 \ ENABLE_TESTCOVERAGE=1 \ @@ -315,8 +316,7 @@ build_ubuntu_cpu() { build_ubuntu_cpu_openblas() { set -ex - export CC="ccache gcc" - export CXX="ccache g++" + install_ccache_wrappers make \ DEV=1 \ ENABLE_TESTCOVERAGE=1 \ @@ -330,6 +330,7 @@ build_ubuntu_cpu_cmake_debug() { set -ex pushd . cd /work/build + install_ccache_wrappers cmake \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ @@ -351,6 +352,8 @@ build_ubuntu_cpu_cmake_asan() { pushd . cd /work/build + install_ccache_wrappers + # TODO: Check if we can remove the g++-8 override cmake \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ @@ -377,10 +380,10 @@ build_ubuntu_cpu_cmake_asan() { build_ubuntu_cpu_clang39() { set -ex - export CXX=clang++-3.9 + export CXX=clang++-3.9 export CC=clang-3.9 - build_ccache_wrappers - make \ + install_ccache_wrappers + make \ ENABLE_TESTCOVERAGE=1 \ USE_CPP_PACKAGE=1 \ USE_BLAS=openblas \ @@ -395,7 +398,7 @@ build_ubuntu_cpu_clang60() { export CXX=clang++-6.0 export CC=clang-6.0 - build_ccache_wrappers + install_ccache_wrappers make \ ENABLE_TESTCOVERAGE=1 \ @@ -415,6 +418,7 @@ build_ubuntu_cpu_clang_tidy() { pushd . cd /work/build + install_ccache_wrappers cmake \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ @@ -438,8 +442,7 @@ build_ubuntu_cpu_clang39_mkldnn() { export CXX=clang++-3.9 export CC=clang-3.9 - build_ccache_wrappers - + install_ccache_wrappers make \ ENABLE_TESTCOVERAGE=1 \ USE_CPP_PACKAGE=1 \ @@ -455,8 +458,7 @@ build_ubuntu_cpu_clang60_mkldnn() { export CXX=clang++-6.0 export CC=clang-6.0 - build_ccache_wrappers - + install_ccache_wrappers make \ ENABLE_TESTCOVERAGE=1 \ USE_CPP_PACKAGE=1 \ @@ -469,8 +471,7 @@ build_ubuntu_cpu_clang60_mkldnn() { build_ubuntu_cpu_mkldnn() { set -ex - build_ccache_wrappers - + install_ccache_wrappers make \ DEV=1 \ ENABLE_TESTCOVERAGE=1 \ @@ -488,7 +489,7 @@ build_ubuntu_gpu_tensorrt() { set -ex - build_ccache_wrappers + install_ccache_wrappers # Build ONNX pushd . @@ -498,6 +499,8 @@ build_ubuntu_gpu_tensorrt() { mkdir -p build cd build cmake \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_FLAGS=-I/usr/include/python${PYVER}\ -DBUILD_SHARED_LIBS=ON ..\ -G Ninja @@ -512,7 +515,10 @@ build_ubuntu_gpu_tensorrt() { cd 3rdparty/onnx-tensorrt/ mkdir -p build cd build - cmake .. + cmake \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + .. make -j$(nproc) export LIBRARY_PATH=`pwd`:$LIBRARY_PATH popd @@ -543,8 +549,7 @@ build_ubuntu_gpu_tensorrt() { build_ubuntu_gpu_mkldnn() { set -ex - build_ccache_wrappers - + install_ccache_wrappers make \ DEV=1 \ ENABLE_TESTCOVERAGE=1 \ @@ -561,8 +566,7 @@ build_ubuntu_gpu_mkldnn() { build_ubuntu_gpu_mkldnn_nocudnn() { set -ex - build_ccache_wrappers - + install_ccache_wrappers make \ DEV=1 \ ENABLE_TESTCOVERAGE=1 \ @@ -577,8 +581,7 @@ build_ubuntu_gpu_mkldnn_nocudnn() { build_ubuntu_gpu_cuda91_cudnn7() { set -ex - # unfortunately this build has problems in 3rdparty dependencies with ccache and make - # build_ccache_wrappers + install_ccache_wrappers make \ DEV=1 \ ENABLE_TESTCOVERAGE=1 \ @@ -595,6 +598,7 @@ build_ubuntu_gpu_cuda91_cudnn7() { build_ubuntu_amalgamation() { set -ex # Amalgamation can not be run with -j nproc + install_ccache_wrappers make -C amalgamation/ clean make -C amalgamation/ \ USE_BLAS=openblas \ @@ -604,6 +608,7 @@ build_ubuntu_amalgamation() { build_ubuntu_amalgamation_min() { set -ex # Amalgamation can not be run with -j nproc + install_ccache_wrappers make -C amalgamation/ clean make -C amalgamation/ \ USE_BLAS=openblas \ @@ -614,9 +619,10 @@ build_ubuntu_amalgamation_min() { build_ubuntu_gpu_cmake_mkldnn() { set -ex cd /work/build + install_ccache_wrappers cmake \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DENABLE_TESTCOVERAGE=ON \ -DUSE_CUDA=1 \ -DUSE_CUDNN=1 \ @@ -637,9 +643,10 @@ build_ubuntu_gpu_cmake_mkldnn() { build_ubuntu_gpu_cmake() { set -ex cd /work/build + install_ccache_wrappers cmake \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DENABLE_TESTCOVERAGE=ON \ -DUSE_CUDA=1 \ -DUSE_CUDNN=1 \