diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0a6c1e9c..00000000 --- a/.travis.yml +++ /dev/null @@ -1,176 +0,0 @@ -matrix: - include: - - name: "Build C++/Java and Upload to S3 on AL2012" - os: linux - services: docker - dist: focal - before_install: - - echo "################################################################################" - - echo "Installing Dependencies for Linux/x86_64" - - echo "################################################################################" - - sudo apt-get update - - sudo apt-get install -y ca-certificates - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - - unzip awscliv2.zip - - sudo ./aws/install - - export PATH=~/bin:$PATH - - mkdir -p ~/.aws - - | - cat > ~/.aws/credentials << EOF - [default] - aws_access_key_id = $AWS_ACCESS_KEY_ID - aws_secret_access_key = $AWS_SECRET_ACCESS_KEY - - cat <<'EOF' >> ~/.aws/credentials - - echo "################################################################################" - - echo "SUCCESS Installed Dependencies for Linux/x86_64" - - echo "################################################################################" - script: - - echo "################################################################################" - - echo "Starting build phase for Linux/x86_64" - - echo "################################################################################" - - aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 056543101242.dkr.ecr.us-west-2.amazonaws.com - - echo "Getting Docker image for AL2012 build of KPL..." - - docker pull 056543101242.dkr.ecr.us-west-2.amazonaws.com/kinesis-producer-library-builder:latest - - ls - - df -h - - echo "Building KPL in current directory in the Container..." - - docker run --memory 9g -v $(pwd):/kpl -it 056543101242.dkr.ecr.us-west-2.amazonaws.com/kinesis-producer-library-builder:latest /bin/bash -c "export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID && cd /kpl && ls && ./bootstrap.sh" - - cd java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/$TRAVIS_OS_NAME-`uname -m` - - ls - - sudo zip kinesis-producer.zip kinesis_producer - - echo "Uploading build artifacts for KPL built on Linux..." - - aws s3 cp kinesis-producer.zip s3://kpl-build-kinesis-internal/$KPL_VERSION-$HASH/$TRAVIS_OS_NAME-`uname -m`/kinesis-producer.zip - - echo "################################################################################" - - echo "SUCCESS Build phase for Linux/x86_64, complete. Check for any errors reported" - - echo "################################################################################" - - name: "Build C++/Java and Upload to S3 on AL2/Arm64" - os: linux - arch: arm64-graviton2 - virt: vm - services: docker - dist: focal - group: edge - before_install: - - echo "################################################################################" - - echo "Installing Dependencies for Linux/arm64" - - echo "################################################################################" - - sudo apt-get update - - sudo apt-get install -y ca-certificates - - sudo apt-get install python3 - - sudo apt-get -y install python3-pip - - pip3 --version - - pip3 install --user awscli - - mkdir -p ~/.aws - - | - cat > ~/.aws/credentials << EOF - [default] - aws_access_key_id = $AWS_ACCESS_KEY_ID - aws_secret_access_key = $AWS_SECRET_ACCESS_KEY - - cat <<'EOF' >> ~/.aws/credentials - - echo "################################################################################" - - echo "SUCCESS Installed Dependencies for Linux/arm64" - - echo "################################################################################" - script: - - echo "################################################################################" - - echo "Starting build phase for Linux/arm64" - - echo "################################################################################" - - $(aws ecr get-login --no-include-email --region us-west-2) - - echo "Getting Docker image for AL2/Arm64 build of KPL..." - - docker pull amazonlinux:2 - - ls - - df -h - - echo "Building KPL in current directory in the Container..." - - docker run --memory 9g -v $(pwd):/kpl -it amazonlinux:2 /bin/bash -c "export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID && yum install -y -q cmake3 gcc gcc-c++ make file tar gzip which perl git libuuid libuuid-devel maven zlib zlib-devel zlib-static wget && cd /kpl && ls && ./bootstrap.sh" - - cd java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/$TRAVIS_OS_NAME-`uname -m` - - ls - - sudo zip kinesis-producer.zip kinesis_producer - - echo "Uploading build artifacts for KPL built on Linux Arm64..." - - aws s3 cp kinesis-producer.zip s3://kpl-build-kinesis-internal/$KPL_VERSION-$HASH/$TRAVIS_OS_NAME-`uname -m`/kinesis-producer.zip - - echo "################################################################################" - - echo "SUCCESS Build phase for Linux/arm64, complete. Check for any errors reported" - - echo "################################################################################" - - name: "Build C++/Java and Upload to S3 on OSX" - os: osx - compiler: clang - osx_image: xcode9.3 - script: - - curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg" - - sudo installer -pkg AWSCLIV2.pkg -target / - - java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' - - export JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}') - - DIR=$(pwd) - - ./bootstrap.sh - - cd java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/$TRAVIS_OS_NAME - - zip kinesis-producer.zip kinesis_producer - - aws s3 cp kinesis-producer.zip s3://kpl-build-kinesis-internal/$KPL_VERSION-$HASH/$TRAVIS_OS_NAME/kinesis-producer.zip - - echo "################################################################################" - - echo "Finished building KPL on OSX...Starting Bundling of Linux/x86_64 with OSX binaries" - - echo "################################################################################" - - test -e $DIR/java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/osx/kinesis_producer - - aws s3 cp s3://kpl-build-kinesis-internal/$KPL_VERSION-$HASH/linux-x86_64/kinesis-producer.zip linux-x86_64-binaries.zip - - mkdir -p java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/linux-x86_64/ - - unzip linux-x86_64-binaries.zip -d $DIR/java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/linux-x86_64/ - - cd $DIR/java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/linux-x86_64/ - - pwd - - ls - - test -e $DIR/java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/linux-x86_64/kinesis_producer - - cd $DIR - - pwd - - ls - - echo "################################################################################" - - echo "Starting Bundling of Linux/arm64 with OSX binaries" - - echo "################################################################################" - - aws s3 cp s3://kpl-build-kinesis-internal/$KPL_VERSION-$HASH/linux-aarch64/kinesis-producer.zip linux-aarch64-binaries.zip - - mkdir -p java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/linux-aarch64/ - - unzip linux-aarch64-binaries.zip -d $DIR/java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/linux-aarch64/ - - cd $DIR/java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/linux-aarch64/ - - pwd - - ls - - test -e $DIR/java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/linux-aarch64/kinesis_producer - - cd $DIR - - pwd - - ls - - echo "################################################################################" - - echo "Starting the build phase for amazon-kinesis-producer and amazon-kinesis-producer-sample $KPL_VERSION" - - echo "################################################################################" - - cd $DIR/java/amazon-kinesis-producer/ - - pwd - - ls - - mvn clean package source:jar javadoc:jar install - - cd $DIR - - pwd - - ls - - cd $DIR/java/amazon-kinesis-producer-sample/ - - mvn clean package - - cd $DIR - - pwd - - ls - - cd $DIR/java/amazon-kinesis-producer/target - - pwd - - ls - - zip -R amazon-kinesis-producer-jars.zip '*.jar' - - aws s3 cp $DIR/java/amazon-kinesis-producer/target/amazon-kinesis-producer-jars.zip s3://kpl-build-kinesis-internal/$KPL_VERSION-$HASH/release/amazon-kinesis-producer-jars.zip - - cd $DIR - - pwd - - ls - - cd $DIR/java/amazon-kinesis-producer-sample/target - - pwd - - ls - - aws s3 cp $DIR/java/amazon-kinesis-producer-sample/target/amazon-kinesis-producer-sample-1.0.0.jar s3://kpl-build-kinesis-internal/$KPL_VERSION-$HASH/release/amazon-kinesis-producer-sample-1.0.0.jar - - cd $DIR - - pwd - - ls - - echo "################################################################################" - - echo "SUCCESS, Uploaded Zipped Release Artifacts" - - echo "################################################################################" -before_script: - - cd java/amazon-kinesis-producer - - HASH=$(git rev-parse HEAD) - - KPL_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - - echo "Found KPL version $KPL_VERSION in current directory $PWD ... " - - echo "Starting Packaging of KPL.$KPL_VERSION" - - cd ../.. -after_success: - - sleep 10 -after_failure: - - sleep 10 diff --git a/bootstrap.sh b/bootstrap.sh index 657e6a87..79436417 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -13,8 +13,8 @@ silence() { } OPENSSL_VERSION="1.1.1s" -BOOST_VERSION="1.80.0" -BOOST_VERSION_UNDERSCORED="${BOOST_VERSION//\./_}" # convert from 1.80.0 to 1_80_0 +BOOST_VERSION="1.76.0" +BOOST_VERSION_UNDERSCORED="${BOOST_VERSION//\./_}" # convert from 1.76.0 to 1_76_0 ZLIB_VERSION="1.2.13" PROTOBUF_VERSION="3.11.4" CURL_VERSION="7.86.0" diff --git a/buildspec-aarch64.yml b/buildspec-aarch64.yml new file mode 100644 index 00000000..20dfe6b5 --- /dev/null +++ b/buildspec-aarch64.yml @@ -0,0 +1,60 @@ +version: 0.2 +env: + variables: + CODEBUILD_OS_NAME: "linux" + MAVEN_VERSION: "3.8.7" + MAVEN_DOWNLOAD_LINK: "https://archive.apache.org/dist/maven/maven-3/3.8.7/binaries/apache-maven-3.8.7-bin.tar.gz" + +phases: + install: + run-as: root + runtime-versions: + java: corretto8 + commands: + - nohup /usr/local/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 --storage-driver=overlay2 & + - timeout 15 sh -c "until docker info; do echo .; sleep 1; done" + pre_build: + run-as: root + commands: + - echo "################################################################################" + - echo "Installing Dependencies for Linux/arm64" + - echo "################################################################################" + - yum install -y sudo + - sudo yum -y update + - sudo yum install -y ca-certificates + - echo "Install aws cli v2" + - curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" + - unzip awscliv2.zip + - sudo ./aws/install --bin-dir /usr/local/bin --install-dir /usr/local/aws-cli --update + - export PATH=~/bin:$PATH + - echo "Aws cli has been installed successfully" + - echo "Assuming the builder role to get credentials for building KPL" + - BUILDER_ROLE=$(aws sts assume-role --role-arn arn:aws:iam::056543101242:role/KPLCodeBuildAdminRole --role-session-name kpl-build-session) + - export BUILDER_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.SessionToken') + - echo "Pulled the credentials for KPL builders" + - echo "################################################################################" + - echo "SUCCESS Installed Dependencies for Linux/arm64" + - echo "################################################################################" + build: + commands: + - echo "################################################################################" + - echo "Starting build phase for Linux/arm64" + - echo "################################################################################" + - aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 056543101242.dkr.ecr.us-west-2.amazonaws.com + - echo "Getting Docker image for AL2/Arm64 build of KPL..." + - docker pull public.ecr.aws/amazonlinux/amazonlinux:2 + - ls + - df -h + - echo "Building KPL in current directory in the Container..." + - docker run --rm -m 15g --privileged -v $(pwd):/kpl public.ecr.aws/amazonlinux/amazonlinux:2 /bin/bash -c "export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID && export MAVEN_DOWNLOAD_LINK=$MAVEN_DOWNLOAD_LINK && export MAVEN_VERSION=$MAVEN_VERSION && cd /kpl && ls && yum install -y -q sudo cmake3 gcc gcc-c++ make file tar gzip which perl git libuuid libuuid-devel zlib zlib-devel zlib-static wget && sudo amazon-linux-extras enable corretto8 && sudo yum install -y java-1.8.0-amazon-corretto-devel && export JAVA_HOME=/usr/lib/jvm/java-1.8.0-amazon-corretto.aarch64 && java -version && echo \"Install Maven from $MAVEN_DOWNLOAD_LINK...\" && mkdir -p /usr/local/maven && wget $MAVEN_DOWNLOAD_LINK && tar xvf apache-maven-$MAVEN_VERSION-bin.tar.gz -C /usr/local/maven --strip-components=1 && ln -s /usr/local/maven/bin/mvn /usr/bin/mvn && echo \"Maven has been installed with $(mvn --version)\" && cd /kpl && ls && ./bootstrap.sh" + - cd java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/$CODEBUILD_OS_NAME-`uname -m` + - ls + - sudo zip kinesis-producer.zip kinesis_producer + - echo "Uploading build artifacts for KPL built on Linux arm64..." + - aws s3 cp kinesis-producer.zip s3://kpl-build-kinesis-internal/$KPL_VERSION-$HASH/$CODEBUILD_OS_NAME-`uname -m`/kinesis-producer.zip + - echo "################################################################################" + - echo "SUCCESS Build phase for Linux/arm64, complete. Check for any errors reported" + - echo "################################################################################" \ No newline at end of file diff --git a/buildspec-x86_64.yml b/buildspec-x86_64.yml new file mode 100644 index 00000000..c188b654 --- /dev/null +++ b/buildspec-x86_64.yml @@ -0,0 +1,57 @@ +version: 0.2 +env: + variables: + CODEBUILD_OS_NAME: "linux" + +phases: + install: + run-as: root + runtime-versions: + java: corretto8 + commands: + - nohup /usr/local/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 --storage-driver=overlay2 & + - timeout 15 sh -c "until docker info; do echo .; sleep 1; done" + pre_build: + run-as: root + commands: + - echo "################################################################################" + - echo "Installing Dependencies for Linux/x86_64" + - echo "################################################################################" + - sudo apt update + - sudo apt install -y ca-certificates + - echo "Install aws cli v2" + - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + - unzip awscliv2.zip + - sudo ./aws/install --bin-dir /usr/local/bin --install-dir /usr/local/aws-cli --update + - export PATH=~/bin:$PATH + - echo "Aws cli has been installed successfully" + - echo "Assuming the builder role to get credentials for building KPL" + - BUILDER_ROLE=$(aws sts assume-role --role-arn arn:aws:iam::056543101242:role/KPLCodeBuildAdminRole --role-session-name kpl-build-session) + - export BUILDER_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.SessionToken') + - echo "Pulled the credentials for KPL builders" + - echo "################################################################################" + - echo "SUCCESS Installed Dependencies for Linux/x86_64" + - echo "################################################################################" + build: + commands: + - echo "################################################################################" + - echo "Starting build phase for Linux/x86_64" + - echo "################################################################################" + - aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 056543101242.dkr.ecr.us-west-2.amazonaws.com + - echo "Getting Docker image for AL2012 build of KPL..." + - docker pull 056543101242.dkr.ecr.us-west-2.amazonaws.com/kinesis-producer-library-builder:latest + - ls + - df -h + - echo "Building KPL in current directory in the Container..." + - docker run --rm -m 15g -v $(pwd):/kpl 056543101242.dkr.ecr.us-west-2.amazonaws.com/kinesis-producer-library-builder:latest /bin/bash -c "export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID && cd /kpl && ls && ./bootstrap.sh" + - cd java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/$CODEBUILD_OS_NAME-`uname -m` + - ls + - sudo zip kinesis-producer.zip kinesis_producer + - echo "Uploading build artifacts for KPL built on Linux x86_64..." + - aws s3 cp kinesis-producer.zip s3://kpl-build-kinesis-internal/$KPL_VERSION-$HASH/$CODEBUILD_OS_NAME-`uname -m`/kinesis-producer.zip + - echo "################################################################################" + - echo "SUCCESS Build phase for Linux/x86_64, complete. Check for any errors reported" + - echo "################################################################################" \ No newline at end of file