From 4a1a078cdcc9fc693737478e17f487d8cef2be11 Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Wed, 5 Dec 2018 10:57:34 -0800 Subject: [PATCH] Fix testing pipeline --- ci/docker/Dockerfile.build.centos7_cpu | 2 -- ci/docker/Dockerfile.build.centos7_gpu | 2 -- .../Dockerfile.publish.test.ubuntu1804_cpu | 2 ++ .../Dockerfile.publish.test.ubuntu1804_gpu | 2 ++ ci/docker/install/centos7_scala.sh | 8 ++++++- ci/docker/install/ubuntu_core.sh | 1 + ci/docker/install/ubuntu_scala.sh | 5 ++++ scala-package/dev/build.sh | 1 - scala-package/dev/fullDeploy.sh | 23 +++++++++++++++++++ scala-package/dev/test.sh | 13 ++++------- scala-package/packageTest/pom.xml | 12 ---------- 11 files changed, 45 insertions(+), 26 deletions(-) create mode 100644 scala-package/dev/fullDeploy.sh diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/docker/Dockerfile.build.centos7_cpu index d456f1c80be4..e2802aa2fb2b 100644 --- a/ci/docker/Dockerfile.build.centos7_cpu +++ b/ci/docker/Dockerfile.build.centos7_cpu @@ -24,8 +24,6 @@ WORKDIR /work/deps COPY install/centos7_core.sh /work/ RUN /work/centos7_core.sh -COPY install/centos7_build.sh /work/ -RUN /work/centos7_build.sh COPY install/centos7_ccache.sh /work/ RUN /work/centos7_ccache.sh COPY install/centos7_python.sh /work/ diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu index b962f7be8b53..8bf2442731af 100644 --- a/ci/docker/Dockerfile.build.centos7_gpu +++ b/ci/docker/Dockerfile.build.centos7_gpu @@ -24,8 +24,6 @@ WORKDIR /work/deps COPY install/centos7_core.sh /work/ RUN /work/centos7_core.sh -COPY install/centos7_build.sh /work/ -RUN /work/centos7_build.sh COPY install/centos7_ccache.sh /work/ RUN /work/centos7_ccache.sh COPY install/centos7_python.sh /work/ diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu b/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu index a3359a8f685f..c4126ab41e92 100644 --- a/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu +++ b/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu @@ -22,6 +22,8 @@ FROM ubuntu:18.04 WORKDIR /work/deps +ENV DEBIAN_FRONTEND noninteractive + COPY install/ubuntu_core.sh /work/ RUN /work/ubuntu_core.sh diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu b/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu index 210937b5d654..970906f865eb 100644 --- a/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu +++ b/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu @@ -22,6 +22,8 @@ FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu18.04 WORKDIR /work/deps +ENV DEBIAN_FRONTEND noninteractive + COPY install/ubuntu_core.sh /work/ RUN /work/ubuntu_core.sh diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/install/centos7_scala.sh index 90979b286b05..5c43f011cbf1 100755 --- a/ci/docker/install/centos7_scala.sh +++ b/ci/docker/install/centos7_scala.sh @@ -26,8 +26,14 @@ yum install -y java-1.8.0-openjdk-devel export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk export PATH=$JAVA_HOME/bin:$PATH # Build from source with Maven -wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz +wget -q http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz tar xzf apache-maven-3.3.9-bin.tar.gz mkdir /usr/local/maven mv apache-maven-3.3.9/ /usr/local/maven/ alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 + +echo "export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk" >> /etc/profile.d/maven.sh +echo "export M3_HOME=/usr/local/src/apache-maven" >> /etc/profile.d/maven.sh +echo "export PATH=$M3_HOME/bin:$JAVA_HOME/bin:$PATH" >> /etc/profile.d/maven.sh +chmod +x /etc/profile.d/maven.sh +source /etc/profile.d/maven.sh diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh index 64f8af3e0444..8300eaa688b1 100755 --- a/ci/docker/install/ubuntu_core.sh +++ b/ci/docker/install/ubuntu_core.sh @@ -36,6 +36,7 @@ apt-get install -y \ libopenblas-dev \ libopencv-dev \ libzmq3-dev \ + libgfortran3 \ ninja-build \ software-properties-common \ sudo \ diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/docker/install/ubuntu_scala.sh index 4934f685a505..f1da174e152d 100755 --- a/ci/docker/install/ubuntu_scala.sh +++ b/ci/docker/install/ubuntu_scala.sh @@ -41,3 +41,8 @@ apt-get install -y \ gnupg2 \ gnupg-agent \ scala + +# Ubuntu 14.04 +if [[ $(lsb_release -r | grep 14.04) ]]; then + sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java +fi diff --git a/scala-package/dev/build.sh b/scala-package/dev/build.sh index 013ee5739f84..807da714b729 100755 --- a/scala-package/dev/build.sh +++ b/scala-package/dev/build.sh @@ -32,5 +32,4 @@ make scalaintegrationtest CI=1 # Compile tests for discovery later export GPG_TTY=$(tty) make scalatestcompile CI=1 -# make scalainstall CI=1 make scaladeploylocal CI=1 diff --git a/scala-package/dev/fullDeploy.sh b/scala-package/dev/fullDeploy.sh new file mode 100644 index 000000000000..ea2aa97bb2f1 --- /dev/null +++ b/scala-package/dev/fullDeploy.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +./scala-package/dev/build.sh +./scala-package/dev/test.sh +./scala-package/dev/deploy.sh diff --git a/scala-package/dev/test.sh b/scala-package/dev/test.sh index 129c67e0c702..5cef35ca3c2b 100755 --- a/scala-package/dev/test.sh +++ b/scala-package/dev/test.sh @@ -18,14 +18,11 @@ set -ex -# Install -# make scalainstall -# MXNET_FULL=$(find assembly/ -name "mxnet-full*-SNAPSHOT.jar" | grep -v "archive-tmp") -# MXNET_EXAMPLES=$(find . -name "mxnet-examples*-SNAPSHOT.jar") -# mvn install:install-file -Dfile=$MXNET_FULL -# mvn install:install-file -Dfile=$MXNET_EXAMPLES -# make scalainstallonly CI=1 +if [ -z "$JAVA_HOME" ]; then + source /etc/profile +fi # Test cd scala-package/packageTest -make scalaintegrationtestsnapshot CI=1 +# make testlocal CI=1 +make testsnapshot UNIT=1 CI=1 diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml index 101de33146e0..9c5c11cf2779 100644 --- a/scala-package/packageTest/pom.xml +++ b/scala-package/packageTest/pom.xml @@ -29,11 +29,7 @@ apache-snapshots -<<<<<<< HEAD ${mxnet.repo} -======= - https://repository.apache.org/content/repositories/snapshots ->>>>>>> 5503e4e79... Seperate Build, Test, and Deploy Stages with parallel default true @@ -64,14 +60,6 @@ org.apache.mxnet mxnet-full_${mxnet.scalaprofile}-${mxnet.profile} ${mxnet.version} -<<<<<<< HEAD -======= - ->>>>>>> 5503e4e79... Seperate Build, Test, and Deploy Stages with parallel org.scala-lang