Skip to content

Commit

Permalink
[MXNET-674] Speed up GPU builds in CI (apache#12782)
Browse files Browse the repository at this point in the history 
* [MXNET-674] Speed up GPU builds in CI

* [MXNET-674] Refactor SMs into shell variable

* [MXNET-674] Build CMake GPU CI jobs without PTX
  • Loading branch information
@KellenSunderland @ChaiBapchya
KellenSunderland authored and ChaiBapchya committed Oct 30, 2018
1 parent 4e9d6be commit 6ee88af
Showing 1 changed file with 74 additions and 64 deletions.
138 changes: 74 additions & 64 deletions ci/docker/runtime_functions.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
set -ex

NOSE_COVERAGE_ARGUMENTS="--with-coverage --cover-inclusive --cover-xml --cover-branches --cover-package=mxnet"
CI_CUDA_COMPUTE_CAPABILITIES="-gencode=arch=compute_52,code=sm_52 -gencode=arch=compute_70,code=sm_70"
CI_CMAKE_CUDA_ARCH_BIN="52,70"

clean_repo() {
set -ex
Expand Down Expand Up @@ -293,15 +295,16 @@ build_centos7_gpu() {
# unfortunately this build has problems in 3rdparty dependencies with ccache and make
# build_ccache_wrappers
make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_LAPACK=1 \
USE_LAPACK_PATH=/usr/lib64/liblapack.so \
USE_BLAS=openblas \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
USE_DIST_KVSTORE=1 \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_LAPACK=1 \
USE_LAPACK_PATH=/usr/lib64/liblapack.so \
USE_BLAS=openblas \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
USE_DIST_KVSTORE=1 \
CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
-j$(nproc)
}

Expand Down Expand Up @@ -520,19 +523,19 @@ build_ubuntu_gpu_tensorrt() {

rm -rf build
make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_BLAS=openblas \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
USE_OPENCV=0 \
USE_DIST_KVSTORE=0 \
USE_TENSORRT=1 \
USE_JEMALLOC=0 \
USE_GPERFTOOLS=0 \
ONNX_NAMESPACE=onnx \
CUDA_ARCH="-gencode arch=compute_70,code=compute_70"\
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_BLAS=openblas \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
USE_OPENCV=0 \
USE_DIST_KVSTORE=0 \
USE_TENSORRT=1 \
USE_JEMALLOC=0 \
USE_GPERFTOOLS=0 \
ONNX_NAMESPACE=onnx \
CUDA_ARCH="-gencode arch=compute_70,code=compute_70" \
-j$(nproc)
}

Expand All @@ -542,14 +545,15 @@ build_ubuntu_gpu_mkldnn() {
build_ccache_wrappers

make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
USE_MKLDNN=1 \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
USE_MKLDNN=1 \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
-j$(nproc)
}

Expand All @@ -559,13 +563,14 @@ build_ubuntu_gpu_mkldnn_nocudnn() {
build_ccache_wrappers

make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_BLAS=openblas \
USE_MKLDNN=1 \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=0 \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_BLAS=openblas \
USE_MKLDNN=1 \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=0 \
CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
-j$(nproc)
}

Expand All @@ -574,14 +579,15 @@ build_ubuntu_gpu_cuda91_cudnn7() {
# unfortunately this build has problems in 3rdparty dependencies with ccache and make
# build_ccache_wrappers
make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_BLAS=openblas \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
USE_CPP_PACKAGE=1 \
USE_DIST_KVSTORE=1 \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_BLAS=openblas \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
USE_CPP_PACKAGE=1 \
USE_DIST_KVSTORE=1 \
CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
-j$(nproc)
}

Expand All @@ -608,15 +614,17 @@ build_ubuntu_gpu_cmake_mkldnn() {
set -ex
cd /work/build
cmake \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DENABLE_TESTCOVERAGE=ON \
-DUSE_CUDA=1 \
-DUSE_CUDNN=1 \
-DUSE_MKLML_MKL=1 \
-DUSE_MKLDNN=1 \
-DCMAKE_BUILD_TYPE=Release \
-G Ninja \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DENABLE_TESTCOVERAGE=ON \
-DUSE_CUDA=1 \
-DUSE_CUDNN=1 \
-DUSE_MKLML_MKL=1 \
-DUSE_MKLDNN=1 \
-DCMAKE_BUILD_TYPE=Release \
-DCUDA_ARCH_NAME=Manual \
-DCUDA_ARCH_BIN=$CI_CMAKE_CUDA_ARCH_BIN \
-G Ninja \
/work/mxnet

ninja -v
Expand All @@ -629,16 +637,18 @@ build_ubuntu_gpu_cmake() {
set -ex
cd /work/build
cmake \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DENABLE_TESTCOVERAGE=ON \
-DUSE_CUDA=1 \
-DUSE_CUDNN=1 \
-DUSE_MKLML_MKL=0 \
-DUSE_MKLDNN=0 \
-DUSE_DIST_KVSTORE=1 \
-DCMAKE_BUILD_TYPE=Release \
-G Ninja \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DENABLE_TESTCOVERAGE=ON \
-DUSE_CUDA=1 \
-DUSE_CUDNN=1 \
-DUSE_MKLML_MKL=0 \
-DUSE_MKLDNN=0 \
-DUSE_DIST_KVSTORE=1 \
-DCMAKE_BUILD_TYPE=Release \
-DCUDA_ARCH_NAME=Manual \
-DCUDA_ARCH_BIN=$CI_CMAKE_CUDA_ARCH_BIN \
-G Ninja \
/work/mxnet

ninja -v
Expand Down

0 comments on commit 6ee88af

Please sign in to comment.