From 9b7b8e570950d62c9a9ce9b6482106f129ab0fa9 Mon Sep 17 00:00:00 2001 From: Bryan Moffatt Date: Mon, 22 Aug 2022 12:45:22 -0700 Subject: [PATCH] Add instructions for executing the AWS CodeBuild workflows locally --- CONTRIBUTING.md | 18 ++++ ci/codebuild_build.sh | 205 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+) create mode 100755 ci/codebuild_build.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e8c3aa5..d6b300a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,6 +43,24 @@ GitHub provides additional document on [forking a repository](https://help.githu ## Finding contributions to work on Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](/~https://github.com/awslabs/aws-lambda-cpp-runtime/labels/help%20wanted) issues is a great place to start. +## Running the Integration Tests Locally + +The integration testing for the project creates, invokes, and deletes, Lambda functions. +These tests typically run AWS CodeBuild, but may also be executed locally + +Prerequisites: +* install Docker +* configure AWS credentials, need at least permissions to Create, Delete, and Invoke Lambda functions +* an IAM role, named exactly `integration-tests`, must exist in the account. + * The role must also be assumable by Lambda. + * (optional) attach AWSLambdaBasicExecutionRole managed policy to the role, so that the test function logs are saved to CloudWatch + +Then, to iterate on a single workflow: +``` +docker build -t lambda-cpp-amazon-linux-2 -f ./ci/docker/amazon-linux-2 . +./ci/codebuild_build.sh -c -a /tmp -i lambda-cpp-amazon-linux-2 -b ./ci/codebuild/amazon-linux-2.yml +``` + ## Code of Conduct This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). diff --git a/ci/codebuild_build.sh b/ci/codebuild_build.sh new file mode 100755 index 0000000..b2f75bc --- /dev/null +++ b/ci/codebuild_build.sh @@ -0,0 +1,205 @@ +#!/bin/bash + +function allOSRealPath() { + if isOSWindows + then + path="" + case $1 in + .* ) path="$PWD/${1#./}" ;; + /* ) path="$1" ;; + * ) path="/$1" ;; + esac + + echo "/$path" | sed -e 's/\\/\//g' -e 's/://' -e 's/./\U&/3' + else + case $1 in + /* ) echo "$1"; exit;; + * ) echo "$PWD/${1#./}"; exit;; + esac + fi +} + +function isOSWindows() { + if [ $OSTYPE == "msys" ] + then + return 0 + else + return 1 + fi +} + +function usage { + echo "usage: codebuild_build.sh [-i image_name] [-a artifact_output_directory] [options]" + echo "Required:" + echo " -i Used to specify the customer build container image." + echo " -a Used to specify an artifact output directory." + echo "Options:" + echo " -l IMAGE Used to override the default local agent image." + echo " -r Used to specify a report output directory." + echo " -s Used to specify source information. Defaults to the current working directory for primary source." + echo " * First (-s) is for primary source" + echo " * Use additional (-s) in : format for secondary source" + echo " * For sourceIdentifier, use a value that is fewer than 128 characters and contains only alphanumeric characters and underscores" + echo " -c Use the AWS configuration and credentials from your local host. This includes ~/.aws and any AWS_* environment variables." + echo " -p Used to specify the AWS CLI Profile." + echo " -b FILE Used to specify a buildspec override file. Defaults to buildspec.yml in the source directory." + echo " -m Used to mount the source directory to the customer build container directly." + echo " -d Used to run the build container in docker privileged mode." + echo " -e FILE Used to specify a file containing environment variables." + echo " (-e) File format expectations:" + echo " * Each line is in VAR=VAL format" + echo " * Lines beginning with # are processed as comments and ignored" + echo " * Blank lines are ignored" + echo " * File can be of type .env or .txt" + echo " * There is no special handling of quotation marks, meaning they will be part of the VAL" + exit 1 +} + +image_flag=false +artifact_flag=false +awsconfig_flag=false +mount_src_dir_flag=false +docker_privileged_mode_flag=false + +while getopts "cmdi:a:r:s:b:e:l:p:h" opt; do + case $opt in + i ) image_flag=true; image_name=$OPTARG;; + a ) artifact_flag=true; artifact_dir=$OPTARG;; + r ) report_dir=$OPTARG;; + b ) buildspec=$OPTARG;; + c ) awsconfig_flag=true;; + m ) mount_src_dir_flag=true;; + d ) docker_privileged_mode_flag=true;; + s ) source_dirs+=("$OPTARG");; + e ) environment_variable_file=$OPTARG;; + l ) local_agent_image=$OPTARG;; + p ) aws_profile=$OPTARG;; + h ) usage; exit;; + \? ) echo "Unknown option: -$OPTARG" >&2; exit 1;; + : ) echo "Missing option argument for -$OPTARG" >&2; exit 1;; + * ) echo "Invalid option: -$OPTARG" >&2; exit 1;; + esac +done + +if ! $image_flag +then + echo "The image name flag (-i) must be included for a build to run" >&2 +fi + +if ! $artifact_flag +then + echo "The artifact directory (-a) must be included for a build to run" >&2 +fi + +if ! $image_flag || ! $artifact_flag +then + exit 1 +fi + +docker_command="docker run -it " +if isOSWindows +then + docker_command+="-v //var/run/docker.sock:/var/run/docker.sock -e " +else + docker_command+="-v /var/run/docker.sock:/var/run/docker.sock -e " +fi + +docker_command+="\"IMAGE_NAME=$image_name\" -e \ + \"ARTIFACTS=$(allOSRealPath "$artifact_dir")\"" + +if [ -n "$report_dir" ] +then + docker_command+=" -e \"REPORTS=$(allOSRealPath "$report_dir")\"" +fi + +if [ -z "$source_dirs" ] +then + docker_command+=" -e \"SOURCE=$(allOSRealPath "$PWD")\"" +else + for index in "${!source_dirs[@]}"; do + if [ $index -eq 0 ] + then + docker_command+=" -e \"SOURCE=$(allOSRealPath "${source_dirs[$index]}")\"" + else + identifier=${source_dirs[$index]%%:*} + src_dir=$(allOSRealPath "${source_dirs[$index]#*:}") + + docker_command+=" -e \"SECONDARY_SOURCE_$index=$identifier:$src_dir\"" + fi + done +fi + +if [ -n "$buildspec" ] +then + docker_command+=" -e \"BUILDSPEC=$(allOSRealPath "$buildspec")\"" +fi + +if [ -n "$environment_variable_file" ] +then + environment_variable_file_path=$(allOSRealPath "$environment_variable_file") + environment_variable_file_dir=$(dirname "$environment_variable_file_path") + environment_variable_file_basename=$(basename "$environment_variable_file") + docker_command+=" -v \"$environment_variable_file_dir:/LocalBuild/envFile/\" -e \"ENV_VAR_FILE=$environment_variable_file_basename\"" +fi + +if [ -n "$local_agent_image" ] +then + docker_command+=" -e \"LOCAL_AGENT_IMAGE_NAME=$local_agent_image\"" +fi + +if $awsconfig_flag +then + if [ -d "$HOME/.aws" ] + then + configuration_file_path=$(allOSRealPath "$HOME/.aws") + docker_command+=" -e \"AWS_CONFIGURATION=$configuration_file_path\"" + else + docker_command+=" -e \"AWS_CONFIGURATION=NONE\"" + fi + + if [ -n "$aws_profile" ] + then + docker_command+=" -e \"AWS_PROFILE=$aws_profile\"" + fi + + docker_command+="$(env | grep ^AWS_ | while read -r line; do echo " -e \"$line\""; done )" +fi + +if $mount_src_dir_flag +then + docker_command+=" -e \"MOUNT_SOURCE_DIRECTORY=TRUE\"" +fi + +if $docker_privileged_mode_flag +then + docker_command+=" -e \"DOCKER_PRIVILEGED_MODE=TRUE\"" +fi + +if isOSWindows +then + docker_command+=" -e \"INITIATOR=$USERNAME\"" +else + docker_command+=" -e \"INITIATOR=$USER\"" +fi + +if [ -n "$local_agent_image" ] +then + docker_command+=" $local_agent_image" +else + docker_command+=" public.ecr.aws/codebuild/local-builds:latest" +fi + +# Note we do not expose the AWS_SECRET_ACCESS_KEY or the AWS_SESSION_TOKEN +exposed_command=$docker_command +secure_variables=( "AWS_SECRET_ACCESS_KEY=" "AWS_SESSION_TOKEN=") +for variable in "${secure_variables[@]}" +do + exposed_command="$(echo $exposed_command | sed "s/\($variable\)[^ ]*/\1********\"/")" +done + +echo "Build Command:" +echo "" +echo $exposed_command +echo "" + +eval $docker_command