diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu index 7c7e2240ee61..2df9f5887f54 100644 --- a/ci/docker/Dockerfile.build.ubuntu_cpu +++ b/ci/docker/Dockerfile.build.ubuntu_cpu @@ -54,6 +54,9 @@ RUN /work/ubuntu_clang.sh COPY install/ubuntu_gcc8.sh /work/ RUN /work/ubuntu_gcc8.sh +COPY install/ubuntu_mkl.sh /work/ +RUN /work/ubuntu_mkl.sh + COPY install/ubuntu_mklml.sh /work/ RUN /work/ubuntu_mklml.sh diff --git a/ci/docker/install/ubuntu_mkl.sh b/ci/docker/install/ubuntu_mkl.sh new file mode 100755 index 000000000000..36fc7b07ffdc --- /dev/null +++ b/ci/docker/install/ubuntu_mkl.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +set -ex + +apt-get update || true +# Install Intel Math Kernel Library (latest major release) +# https://software.intel.com/en-us/articles/installing-intel-free-libs-and-python-apt-repo +wget -O - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB | apt-key add - && \ + sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \ + apt-get update && \ + apt-get install -y intel-mkl-2019.1-053 diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 1fc10bf0e085..6dd5bb6f239d 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -326,6 +326,20 @@ build_ubuntu_cpu_openblas() { -j$(nproc) } +build_ubuntu_cpu_mkl() { + set -ex + export CC="ccache gcc" + export CXX="ccache g++" + make \ + DEV=1 \ + ENABLE_TESTCOVERAGE=1 \ + USE_CPP_PACKAGE=1 \ + USE_BLAS=mkl \ + USE_INTEL_PATH=/opt/intel \ + USE_DIST_KVSTORE=1 \ + -j$(nproc) +} + build_ubuntu_cpu_cmake_debug() { set -ex pushd . @@ -480,6 +494,20 @@ build_ubuntu_cpu_mkldnn() { -j$(nproc) } +build_ubuntu_cpu_mkldnn_mkl() { + set -ex + + build_ccache_wrappers + + make \ + DEV=1 \ + ENABLE_TESTCOVERAGE=1 \ + USE_CPP_PACKAGE=1 \ + USE_BLAS=mkl \ + USE_MKLDNN=1 \ + -j$(nproc) +} + build_ubuntu_gpu() { build_ubuntu_gpu_cuda91_cudnn7 } diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index f48a26737308..74bde1eee211 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -121,6 +121,20 @@ def compile_unix_openblas_debug_cpu() { }] } +def compile_unix_mkl_cpu() { + return ['CPU: MKL': { + node(NODE_LINUX_CPU) { + ws('workspace/build-cpu-mkl') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git() + utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkl', false) + utils.pack_lib('cpu_mkl', mx_dist_lib, true) + } + } + } + }] +} + def compile_unix_mkldnn_cpu() { return ['CPU: MKLDNN': { node(NODE_LINUX_CPU) { @@ -135,6 +149,20 @@ def compile_unix_mkldnn_cpu() { }] } +def compile_unix_mkldnn_mkl_cpu() { + return ['CPU: MKLDNN_MKL': { + node(NODE_LINUX_CPU) { + ws('workspace/build-mkldnn-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git() + utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkldnn_mkl', false) + utils.pack_lib('mkldnn_mkl_cpu', mx_mkldnn_lib, true) + } + } + } + }] +} + def compile_unix_mkldnn_gpu() { return ['GPU: MKLDNN': { node(NODE_LINUX_CPU) { @@ -580,6 +608,23 @@ def test_unix_python3_cpu() { }] } +def test_unix_python3_mkl_cpu() { + return ['Python3: MKL-CPU': { + node(NODE_LINUX_CPU) { + ws('workspace/ut-python3-cpu') { + try { + utils.unpack_and_init('cpu_mkl', mx_lib, true) + python3_ut('ubuntu_cpu') + utils.publish_test_coverage() + } finally { + utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_cpu_unittest.xml') + utils.collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python3_cpu_quantization.xml') + } + } + } + }] +} + def test_unix_python3_gpu() { return ['Python3: GPU': { node(NODE_LINUX_GPU) { @@ -665,6 +710,23 @@ def test_unix_python3_mkldnn_cpu() { }] } +def test_unix_python3_mkldnn_mkl_cpu() { + return ['Python3: MKLDNN-MKL-CPU': { + node(NODE_LINUX_CPU) { + ws('workspace/ut-python3-mkldnn-mkl-cpu') { + try { + utils.unpack_and_init('mkldnn_mkl_cpu', mx_mkldnn_lib, true) + python3_ut_mkldnn('ubuntu_cpu') + utils.publish_test_coverage() + } finally { + utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_mkldnn_cpu_unittest.xml') + utils.collect_test_results_unix('nosetests_mkl.xml', 'nosetests_python3_mkldnn_cpu_mkl.xml') + } + } + } + }] +} + def test_unix_python3_mkldnn_gpu() { return ['Python3: MKLDNN-GPU': { node(NODE_LINUX_GPU) { diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu index 9c9a41503772..e581bcf65dc5 100644 --- a/ci/jenkins/Jenkinsfile_unix_cpu +++ b/ci/jenkins/Jenkinsfile_unix_cpu @@ -36,15 +36,19 @@ core_logic: { utils.parallel_stage('Build', [ custom_steps.compile_unix_cpu_openblas(), custom_steps.compile_unix_openblas_debug_cpu(), - custom_steps.compile_unix_mkldnn_cpu() + custom_steps.compile_unix_mkl_cpu(), + custom_steps.compile_unix_mkldnn_cpu(), + custom_steps.compile_unix_mkldnn_mkl_cpu() ]) utils.parallel_stage('Tests', [ custom_steps.test_unix_python2_cpu(), custom_steps.test_unix_python3_cpu(), custom_steps.test_unix_python3_debug_cpu(), + custom_steps.test_unix_python3_mkl_cpu(), custom_steps.test_unix_python2_mkldnn_cpu(), custom_steps.test_unix_python3_mkldnn_cpu(), + custom_steps.test_unix_python3_mkldnn_mkl_cpu(), custom_steps.test_unix_scala_cpu(), custom_steps.test_unix_clojure_cpu(), custom_steps.test_unix_r_cpu(),