From 6ee88af1c19adecc2a21c279bf449426e112817f Mon Sep 17 00:00:00 2001 From: Kellen Sunderland Date: Sun, 21 Oct 2018 14:41:42 -0700 Subject: [PATCH] [MXNET-674] Speed up GPU builds in CI (#12782) * [MXNET-674] Speed up GPU builds in CI * [MXNET-674] Refactor SMs into shell variable * [MXNET-674] Build CMake GPU CI jobs without PTX --- ci/docker/runtime_functions.sh | 138 ++++++++++++++++++--------------- 1 file changed, 74 insertions(+), 64 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 5c05027db4d0..43006f23974d 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -23,6 +23,8 @@ set -ex NOSE_COVERAGE_ARGUMENTS="--with-coverage --cover-inclusive --cover-xml --cover-branches --cover-package=mxnet" +CI_CUDA_COMPUTE_CAPABILITIES="-gencode=arch=compute_52,code=sm_52 -gencode=arch=compute_70,code=sm_70" +CI_CMAKE_CUDA_ARCH_BIN="52,70" clean_repo() { set -ex @@ -293,15 +295,16 @@ build_centos7_gpu() { # unfortunately this build has problems in 3rdparty dependencies with ccache and make # build_ccache_wrappers make \ - DEV=1 \ - ENABLE_TESTCOVERAGE=1 \ - USE_LAPACK=1 \ - USE_LAPACK_PATH=/usr/lib64/liblapack.so \ - USE_BLAS=openblas \ - USE_CUDA=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_CUDNN=1 \ - USE_DIST_KVSTORE=1 \ + DEV=1 \ + ENABLE_TESTCOVERAGE=1 \ + USE_LAPACK=1 \ + USE_LAPACK_PATH=/usr/lib64/liblapack.so \ + USE_BLAS=openblas \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=1 \ + USE_DIST_KVSTORE=1 \ + CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ -j$(nproc) } @@ -520,19 +523,19 @@ build_ubuntu_gpu_tensorrt() { rm -rf build make \ - DEV=1 \ - ENABLE_TESTCOVERAGE=1 \ - USE_BLAS=openblas \ - USE_CUDA=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_CUDNN=1 \ - USE_OPENCV=0 \ - USE_DIST_KVSTORE=0 \ - USE_TENSORRT=1 \ - USE_JEMALLOC=0 \ - USE_GPERFTOOLS=0 \ - ONNX_NAMESPACE=onnx \ - CUDA_ARCH="-gencode arch=compute_70,code=compute_70"\ + DEV=1 \ + ENABLE_TESTCOVERAGE=1 \ + USE_BLAS=openblas \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=1 \ + USE_OPENCV=0 \ + USE_DIST_KVSTORE=0 \ + USE_TENSORRT=1 \ + USE_JEMALLOC=0 \ + USE_GPERFTOOLS=0 \ + ONNX_NAMESPACE=onnx \ + CUDA_ARCH="-gencode arch=compute_70,code=compute_70" \ -j$(nproc) } @@ -542,14 +545,15 @@ build_ubuntu_gpu_mkldnn() { build_ccache_wrappers make \ - DEV=1 \ - ENABLE_TESTCOVERAGE=1 \ - USE_CPP_PACKAGE=1 \ - USE_BLAS=openblas \ - USE_MKLDNN=1 \ - USE_CUDA=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_CUDNN=1 \ + DEV=1 \ + ENABLE_TESTCOVERAGE=1 \ + USE_CPP_PACKAGE=1 \ + USE_BLAS=openblas \ + USE_MKLDNN=1 \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=1 \ + CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ -j$(nproc) } @@ -559,13 +563,14 @@ build_ubuntu_gpu_mkldnn_nocudnn() { build_ccache_wrappers make \ - DEV=1 \ - ENABLE_TESTCOVERAGE=1 \ - USE_BLAS=openblas \ - USE_MKLDNN=1 \ - USE_CUDA=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_CUDNN=0 \ + DEV=1 \ + ENABLE_TESTCOVERAGE=1 \ + USE_BLAS=openblas \ + USE_MKLDNN=1 \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=0 \ + CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ -j$(nproc) } @@ -574,14 +579,15 @@ build_ubuntu_gpu_cuda91_cudnn7() { # unfortunately this build has problems in 3rdparty dependencies with ccache and make # build_ccache_wrappers make \ - DEV=1 \ - ENABLE_TESTCOVERAGE=1 \ - USE_BLAS=openblas \ - USE_CUDA=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_CUDNN=1 \ - USE_CPP_PACKAGE=1 \ - USE_DIST_KVSTORE=1 \ + DEV=1 \ + ENABLE_TESTCOVERAGE=1 \ + USE_BLAS=openblas \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=1 \ + USE_CPP_PACKAGE=1 \ + USE_DIST_KVSTORE=1 \ + CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ -j$(nproc) } @@ -608,15 +614,17 @@ build_ubuntu_gpu_cmake_mkldnn() { set -ex cd /work/build cmake \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DENABLE_TESTCOVERAGE=ON \ - -DUSE_CUDA=1 \ - -DUSE_CUDNN=1 \ - -DUSE_MKLML_MKL=1 \ - -DUSE_MKLDNN=1 \ - -DCMAKE_BUILD_TYPE=Release \ - -G Ninja \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DENABLE_TESTCOVERAGE=ON \ + -DUSE_CUDA=1 \ + -DUSE_CUDNN=1 \ + -DUSE_MKLML_MKL=1 \ + -DUSE_MKLDNN=1 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCUDA_ARCH_NAME=Manual \ + -DCUDA_ARCH_BIN=$CI_CMAKE_CUDA_ARCH_BIN \ + -G Ninja \ /work/mxnet ninja -v @@ -629,16 +637,18 @@ build_ubuntu_gpu_cmake() { set -ex cd /work/build cmake \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DENABLE_TESTCOVERAGE=ON \ - -DUSE_CUDA=1 \ - -DUSE_CUDNN=1 \ - -DUSE_MKLML_MKL=0 \ - -DUSE_MKLDNN=0 \ - -DUSE_DIST_KVSTORE=1 \ - -DCMAKE_BUILD_TYPE=Release \ - -G Ninja \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DENABLE_TESTCOVERAGE=ON \ + -DUSE_CUDA=1 \ + -DUSE_CUDNN=1 \ + -DUSE_MKLML_MKL=0 \ + -DUSE_MKLDNN=0 \ + -DUSE_DIST_KVSTORE=1 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCUDA_ARCH_NAME=Manual \ + -DCUDA_ARCH_BIN=$CI_CMAKE_CUDA_ARCH_BIN \ + -G Ninja \ /work/mxnet ninja -v