From ebb3df3ff0f67d379e8b24e3cda658618db229a7 Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Tue, 8 Jan 2019 11:31:35 -0800 Subject: [PATCH] [MXNET-862] Basic maven jenkins pipeline (#13450) * Jenkins Publish Nightly Maven Progress * Seperate Build, Test, and Deploy Stages with parallel --- ci/docker/install/ubuntu_scala.sh | 3 +++ scala-package/dev/build.sh | 34 ++++++++++++++++++++++++ scala-package/dev/buildkey.py | 28 +++++++++++++++----- scala-package/dev/deploy.sh | 44 +++++++++++++++++++++++++++++++ scala-package/dev/test.sh | 23 ++++++++++++++++ 5 files changed, 125 insertions(+), 7 deletions(-) create mode 100755 scala-package/dev/build.sh create mode 100755 scala-package/dev/deploy.sh create mode 100755 scala-package/dev/test.sh diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/docker/install/ubuntu_scala.sh index 8b23d614b490..5bade47463e2 100755 --- a/ci/docker/install/ubuntu_scala.sh +++ b/ci/docker/install/ubuntu_scala.sh @@ -36,6 +36,9 @@ apt-get install -y \ openjdk-8-jdk \ openjdk-8-jre \ software-properties-common \ + gnupg \ + gnupg2 \ + gnupg-agent \ scala # Ubuntu 14.04 diff --git a/scala-package/dev/build.sh b/scala-package/dev/build.sh new file mode 100755 index 000000000000..c336fd84e5e2 --- /dev/null +++ b/scala-package/dev/build.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +# Setup Environment Variables +# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu +# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu + +bash scala-package/dev/compile-mxnet-backend.sh $MAVEN_PUBLISH_OS_TYPE ./ + +# Scala steps to deploy +make scalapkg CI=1 + +# Compile tests for discovery later +export GPG_TTY=$(tty) +make scalatestcompile CI=1 +# make scalainstall CI=1 +make scaladeploylocal CI=1 diff --git a/scala-package/dev/buildkey.py b/scala-package/dev/buildkey.py index a070f8477ea6..8a1b7bf63286 100644 --- a/scala-package/dev/buildkey.py +++ b/scala-package/dev/buildkey.py @@ -76,22 +76,36 @@ def importASC(key, gpgPassphrase): filename = os.path.join(KEY_PATH, "key.asc") with open(filename, 'w') as f: f.write(key) - subprocess.run(['gpg2', '--batch', '--yes', + subprocess.check_output(['gpg2', '--batch', '--yes', '--passphrase-fd', '0', "--import", "{}".format(filename)], input=str.encode(gpgPassphrase)) def encryptMasterPSW(password): - result = subprocess.run(['mvn', '--encrypt-master-password'], - stdout=subprocess.PIPE, input=str.encode(password)) - return str(result.stdout)[2:-3] + filename = os.path.join(KEY_PATH, "encryptMasterPassword.exp") + with open(filename, 'w') as f: + f.write(''' + spawn mvn --encrypt-master-password + expect -exact "Master password: " + send -- "{}\r" + expect eof + '''.format(password)) + result = subprocess.check_output(['expect', filename]) + return str(result).split('\r\n')[-1][2:-3] def encryptPSW(password): - result = subprocess.run(['mvn', '--encrypt-password'], - stdout=subprocess.PIPE, input=str.encode(password)) - return str(result.stdout)[2:-3] + filename = os.path.join(KEY_PATH, "encryptPassword.exp") + with open(filename, 'w') as f: + f.write(''' + spawn mvn --encrypt-password + expect -exact "Password: " + send -- "{}\r" + expect eof + '''.format(password)) + result = subprocess.check_output(['expect', filename]) + return str(result).split('\r\n')[-1][2:-3] def masterPSW(password): diff --git a/scala-package/dev/deploy.sh b/scala-package/dev/deploy.sh new file mode 100755 index 000000000000..6f845ba5d78f --- /dev/null +++ b/scala-package/dev/deploy.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +# Setup Environment Variables +# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu +# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu + +# Run python to configure keys +python3 $PWD/scala-package/dev/buildkey.py + +# Updating cache +mkdir -p ~/.gnupg +echo "default-cache-ttl 14400" > ~/.gnupg/gpg-agent.conf +echo "max-cache-ttl 14400" >> ~/.gnupg/gpg-agent.conf +echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf +echo "pinentry-mode loopback" >> ~/.gnupg/gpg-agent.conf +export GPG_TTY=$(tty) + +cd scala-package +VERSION=$(mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec) +cd .. + +# echo "\n\n$VERSION\n" | make scalarelease-dryrun +make scaladeploy CI=1 + +# Clear all password .xml files, exp files, and gpg key files +rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp diff --git a/scala-package/dev/test.sh b/scala-package/dev/test.sh new file mode 100755 index 000000000000..03810fbf8d55 --- /dev/null +++ b/scala-package/dev/test.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +# Test +cd scala-package/packageTest +make scalaintegrationtestlocal CI=1