Use AWS CodeBuild to compile C++ binary

awslabs · Jan 11, 2023 · 76d7c1b · 76d7c1b
1 parent c140347
commit 76d7c1b
Show file tree

Hide file tree

Showing 4 changed files with 119 additions and 178 deletions.
diff --git a/.travis.yml b/.travis.yml
diff --git a/bootstrap.sh b/bootstrap.sh
@@ -13,8 +13,8 @@ silence() {
 }
 
 OPENSSL_VERSION="1.1.1s"
-BOOST_VERSION="1.80.0"
-BOOST_VERSION_UNDERSCORED="${BOOST_VERSION//\./_}" # convert from 1.80.0 to 1_80_0
+BOOST_VERSION="1.76.0"
+BOOST_VERSION_UNDERSCORED="${BOOST_VERSION//\./_}" # convert from 1.76.0 to 1_76_0
 ZLIB_VERSION="1.2.13"
 PROTOBUF_VERSION="3.11.4"
 CURL_VERSION="7.86.0"

diff --git a/buildspec-aarch64.yml b/buildspec-aarch64.yml
@@ -0,0 +1,60 @@
+version: 0.2
+env:
+  variables:
+    CODEBUILD_OS_NAME: "linux"
+    MAVEN_VERSION: "3.8.7"
+    MAVEN_DOWNLOAD_LINK: "https://archive.apache.org/dist/maven/maven-3/3.8.7/binaries/apache-maven-3.8.7-bin.tar.gz"
+
+phases:
+  install:
+    run-as: root
+    runtime-versions:
+      java: corretto8
+    commands:
+      - nohup /usr/local/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 --storage-driver=overlay2 &
+      - timeout 15 sh -c "until docker info; do echo .; sleep 1; done"
+  pre_build:
+    run-as: root
+    commands:
+      - echo "################################################################################"
+      - echo "Installing Dependencies for Linux/arm64"
+      - echo "################################################################################"
+      - yum install -y sudo
+      - sudo yum -y update
+      - sudo yum install -y ca-certificates
+      - echo "Install aws cli v2"
+      - curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip"
+      - unzip awscliv2.zip
+      - sudo ./aws/install --bin-dir /usr/local/bin --install-dir /usr/local/aws-cli --update
+      - export PATH=~/bin:$PATH
+      - echo "Aws cli has been installed successfully"
+      - echo "Assuming the builder role to get credentials for building KPL"
+      - BUILDER_ROLE=$(aws sts assume-role --role-arn arn:aws:iam::056543101242:role/KPLCodeBuildAdminRole --role-session-name kpl-build-session)
+      - export BUILDER_ROLE
+      - export AWS_ACCESS_KEY_ID=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.SessionToken')
+      - echo "Pulled the credentials for KPL builders"
+      - echo "################################################################################"
+      - echo "SUCCESS Installed Dependencies for Linux/arm64"
+      - echo "################################################################################"
+  build:
+    commands:
+      - echo "################################################################################"
+      - echo "Starting build phase for Linux/arm64"
+      - echo "################################################################################"
+      - aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 056543101242.dkr.ecr.us-west-2.amazonaws.com
+      - echo "Getting Docker image for AL2/Arm64 build of KPL..."
+      - docker pull public.ecr.aws/amazonlinux/amazonlinux:2
+      - ls
+      - df -h
+      - echo "Building KPL in current directory in the Container..."
+      - docker run --rm -m 15g --privileged -v $(pwd):/kpl public.ecr.aws/amazonlinux/amazonlinux:2 /bin/bash -c "export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID && export MAVEN_DOWNLOAD_LINK=$MAVEN_DOWNLOAD_LINK && export MAVEN_VERSION=$MAVEN_VERSION && cd /kpl && ls && yum install -y -q sudo cmake3 gcc gcc-c++ make file tar gzip which perl git libuuid libuuid-devel zlib zlib-devel zlib-static wget && sudo amazon-linux-extras enable corretto8 && sudo yum install -y java-1.8.0-amazon-corretto-devel && export JAVA_HOME=/usr/lib/jvm/java-1.8.0-amazon-corretto.aarch64 && java -version && echo \"Install Maven from $MAVEN_DOWNLOAD_LINK...\" && mkdir -p /usr/local/maven && wget $MAVEN_DOWNLOAD_LINK && tar xvf apache-maven-$MAVEN_VERSION-bin.tar.gz -C /usr/local/maven --strip-components=1 && ln -s /usr/local/maven/bin/mvn /usr/bin/mvn && echo \"Maven has been installed with $(mvn --version)\" && cd /kpl && ls && ./bootstrap.sh"
+      - cd java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/$CODEBUILD_OS_NAME-`uname -m`
+      - ls
+      - sudo zip kinesis-producer.zip kinesis_producer
+      - echo "Uploading build artifacts for KPL built on Linux arm64..."
+      - aws s3 cp kinesis-producer.zip s3://kpl-build-kinesis-internal/$KPL_VERSION-$HASH/$CODEBUILD_OS_NAME-`uname -m`/kinesis-producer.zip
+      - echo "################################################################################"
+      - echo "SUCCESS Build phase for Linux/arm64, complete. Check for any errors reported"
+      - echo "################################################################################"
diff --git a/buildspec-x86_64.yml b/buildspec-x86_64.yml
@@ -0,0 +1,57 @@
+version: 0.2
+env:
+  variables:
+    CODEBUILD_OS_NAME: "linux"
+
+phases:
+  install:
+    run-as: root
+    runtime-versions:
+      java: corretto8
+    commands:
+      - nohup /usr/local/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 --storage-driver=overlay2 &
+      - timeout 15 sh -c "until docker info; do echo .; sleep 1; done"
+  pre_build:
+    run-as: root
+    commands:
+      - echo "################################################################################"
+      - echo "Installing Dependencies for Linux/x86_64"
+      - echo "################################################################################"
+      - sudo apt update
+      - sudo apt install -y ca-certificates
+      - echo "Install aws cli v2"
+      - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+      - unzip awscliv2.zip
+      - sudo ./aws/install --bin-dir /usr/local/bin --install-dir /usr/local/aws-cli --update
+      - export PATH=~/bin:$PATH
+      - echo "Aws cli has been installed successfully"
+      - echo "Assuming the builder role to get credentials for building KPL"
+      - BUILDER_ROLE=$(aws sts assume-role --role-arn arn:aws:iam::056543101242:role/KPLCodeBuildAdminRole --role-session-name kpl-build-session)
+      - export BUILDER_ROLE
+      - export AWS_ACCESS_KEY_ID=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.SessionToken')
+      - echo "Pulled the credentials for KPL builders"
+      - echo "################################################################################"
+      - echo "SUCCESS Installed Dependencies for Linux/x86_64"
+      - echo "################################################################################"
+  build:
+    commands:
+      - echo "################################################################################"
+      - echo "Starting build phase for Linux/x86_64"
+      - echo "################################################################################"
+      - aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 056543101242.dkr.ecr.us-west-2.amazonaws.com
+      - echo "Getting Docker image for AL2012 build of KPL..."
+      - docker pull 056543101242.dkr.ecr.us-west-2.amazonaws.com/kinesis-producer-library-builder:latest
+      - ls
+      - df -h
+      - echo "Building KPL in current directory in the Container..."
+      - docker run --rm -m 15g -v $(pwd):/kpl 056543101242.dkr.ecr.us-west-2.amazonaws.com/kinesis-producer-library-builder:latest /bin/bash -c "export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID && cd /kpl && ls && ./bootstrap.sh"
+      - cd java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/$CODEBUILD_OS_NAME-`uname -m`
+      - ls
+      - sudo zip kinesis-producer.zip kinesis_producer
+      - echo "Uploading build artifacts for KPL built on Linux x86_64..."
+      - aws s3 cp kinesis-producer.zip s3://kpl-build-kinesis-internal/$KPL_VERSION-$HASH/$CODEBUILD_OS_NAME-`uname -m`/kinesis-producer.zip
+      - echo "################################################################################"
+      - echo "SUCCESS Build phase for Linux/x86_64, complete. Check for any errors reported"
+      - echo "################################################################################"