diff --git a/Makefile b/Makefile index a60f96c..b248675 100644 --- a/Makefile +++ b/Makefile @@ -10,13 +10,24 @@ CURRENT_DIR := $(shell pwd) TEST_REPORT_DIR ?= $(CURRENT_DIR)/tests/report TEST_REPORT_FILE ?= nose2-junit.xml +# Docker metadata +GIT_HASH = `git rev-parse HEAD` +GIT_HASH_SHORT = `git rev-parse --short HEAD` +GIT_BRANCH = `git symbolic-ref HEAD --short 2>/dev/null` +GIT_DIRTY = `git status --porcelain` +GIT_TAG = `git describe --tags || echo "no version info"` +AUTHOR = $(USER) + # general targets timestamps TIMESTAMPS = .timestamps -REQUIREMENTS_TIMESTAMP = $(TIMESTAMPS)/.requirements.timestamp -DEV_REQUIREMENTS_TIMESTAMP = $(TIMESTAMPS)/.dev-requirements.timestamps +REQUIREMENTS := $(TIMESTAMPS) ${VOLUMES_MINIO} $(LOGS_DIR) $(PIP_FILE) $(PIP_FILE_LOCK) # Docker variables -DOCKER_IMG_LOCAL_TAG = swisstopo/$(SERVICE_NAME):local +DOCKER_REGISTRY = 974517877189.dkr.ecr.eu-central-1.amazonaws.com +DOCKER_IMG_LOCAL_TAG := $(DOCKER_REGISTRY)/$(SERVICE_NAME):local-$(USER)-$(GIT_HASH_SHORT) + +# AWS variables +AWS_DEFAULT_REGION = eu-central-1 # Find all python files that are not inside a hidden directory (directory starting with .) PYTHON_FILES := $(shell find ./* -type f -name "*.py" -print) @@ -38,13 +49,6 @@ ISORT := $(PIPENV_RUN) isort NOSE := $(PIPENV_RUN) nose2 PYLINT := $(PIPENV_RUN) pylint -# Docker metadata -GIT_HASH := `git rev-parse HEAD` -GIT_BRANCH := `git symbolic-ref HEAD --short 2>/dev/null` -GIT_DIRTY := `git status --porcelain` -GIT_TAG := `git describe --tags || echo "no version info"` -AUTHOR := $(USER) - all: help @@ -68,8 +72,10 @@ help: @echo -e " \033[1mLOCAL SERVER TARGETS\033[0m " @echo "- serve Run the project using the flask debug server. Port can be set by Env variable HTTP_PORT (default: 5000)" @echo "- gunicornserve Run the project using the gunicorn WSGI server. Port can be set by Env variable DEBUG_HTTP_PORT (default: 5000)" - @echo "- serve-spec Serve the spec using Redoc on localhost:8080" + @echo "- serve-spec-redoc Serve the spec using Redoc on localhost:8080" + @echo "- serve-spec-swagger Serve the spec using Redoc on localhost:8080/swagger" @echo -e " \033[1mDocker TARGETS\033[0m " + @echo "- dockerlogin Login to the AWS ECR registery for pulling/pushing docker images" @echo "- dockerbuild Build the project localy (with tag := $(DOCKER_IMG_LOCAL_TAG)) using the gunicorn WSGI server inside a container" @echo "- dockerpush Build and push the project localy (with tag := $(DOCKER_IMG_LOCAL_TAG))" @echo "- dockerrun Run the project using the gunicorn WSGI server inside a container (exposed port: 5000)" @@ -82,16 +88,18 @@ help: # Build targets. Calling setup is all that is needed for the local files to be installed as needed. .PHONY: dev -dev: $(DEV_REQUIREMENTS_TIMESTAMP) +dev: $(REQUIREMENTS) + pipenv install --dev pipenv shell .PHONY: setup -setup: $(REQUIREMENTS_TIMESTAMP) +setup: $(REQUIREMENTS) + pipenv install pipenv shell .PHONY: ci -ci: $(TIMESTAMPS) $(PIP_FILE) $(PIP_FILE_LOCK) +ci: $(REQUIREMENTS) # Create virtual env with all packages for development using the Pipfile.lock pipenv sync --dev @@ -99,44 +107,33 @@ ci: $(TIMESTAMPS) $(PIP_FILE) $(PIP_FILE_LOCK) # linting target, calls upon yapf to make sure your code is easier to read and respects some conventions. .PHONY: format -format: $(DEV_REQUIREMENTS_TIMESTAMP) +format: $(YAPF) -p -i --style .style.yapf $(PYTHON_FILES) $(ISORT) $(PYTHON_FILES) -.PHONY: lint -lint: $(DEV_REQUIREMENTS_TIMESTAMP) - $(PYLINT) $(PYTHON_FILES) - - -.PHONY: format-lint -format-lint: format lint - - .PHONY: ci-check-format ci-check-format: format @if [[ -n `git status --porcelain` ]]; then \ - >&2 echo "ERROR: Code was not formatted correctly"; \ + >&2 echo "ERROR: the following files are not formatted correctly:"; \ + >&2 git status --porcelain; \ exit 1; \ fi -# Spec targets -.PHONY: lint-spec -lint-spec: - docker run --volume "$(PWD)":/data jamescooke/openapi-validator -e openapi.yml +.PHONY: lint +lint: + $(PYLINT) $(PYTHON_FILES) -.PHONY: serve-spec -serve-spec: - docker run -it --rm -p 8080:80 \ - -v "$(PWD)/openapi.yml":/usr/share/nginx/html/openapi.yml \ - -e SPEC_URL=openapi.yml redocly/redoc +.PHONY: format-lint +format-lint: format lint + # Test target .PHONY: test -test: $(DEV_REQUIREMENTS_TIMESTAMP) +test: $(TEST_REPORT_DIR) mkdir -p $(TEST_REPORT_DIR) $(NOSE) -c tests/unittest.cfg --verbose --junit-xml-path $(TEST_REPORT_DIR)/$(TEST_REPORT_FILE) -s tests/ @@ -144,17 +141,22 @@ test: $(DEV_REQUIREMENTS_TIMESTAMP) # Serve targets. Using these will run the application on your local machine. You can either serve with a wsgi front (like it would be within the container), or without. .PHONY: serve -serve: $(REQUIREMENTS_TIMESTAMP) +serve: FLASK_APP=service_launcher FLASK_DEBUG=1 $(FLASK) run --host=0.0.0.0 --port=$(HTTP_PORT) .PHONY: gunicornserve -gunicornserve: $(REQUIREMENTS_TIMESTAMP) +gunicornserve: $(PYTHON) wsgi.py # Docker related functions. +.PHONY: dockerlogin +dockerlogin: + aws --profile swisstopo-bgdi-builder ecr get-login-password --region $(AWS_DEFAULT_REGION) | docker login --username AWS --password-stdin $(DOCKER_REGISTRY) + + .PHONY: dockerbuild dockerbuild: docker build \ @@ -180,6 +182,31 @@ shutdown: HTTP_PORT=$(HTTP_PORT) SERVICE_NAME=$(SERVICE_NAME) docker-compose down +# Spec targets + +.PHONY: lint-spec +lint-spec: + docker run --volume "$(PWD)":/data jamescooke/openapi-validator -e openapi.yml + + +.PHONY: serve-spec +serve-spec-redoc: + docker run -it --rm -p 8080:80 \ + -v "$(PWD)/openapi.yml":/usr/share/nginx/html/openapi.yml \ + -e SPEC_URL=openapi.yml redocly/redoc + + +.PHONY: serve-spec-swagger +serve-spec-swagger: + echo "SWAGGER UI on http://localhost:8080/swagger" + docker run -p 8080:8080 \ + -e BASE_URL=/swagger -e SWAGGER_JSON=/openapi.yaml \ + -v ${PWD}/openapi.yml:/openapi.yaml \ + swaggerapi/swagger-ui + + +# Clean targets + .PHONY: clean_venv clean_venv: pipenv --rm @@ -189,7 +216,6 @@ clean_venv: clean: clean_venv @# clean python cache files find . -name __pycache__ -type d -print0 | xargs -I {} -0 rm -rf "{}" - rm -rf $(PYTHON_LOCAL_DIR) rm -rf $(TEST_REPORT_DIR) rm -rf $(TIMESTAMPS) @@ -199,11 +225,3 @@ clean: clean_venv $(TIMESTAMPS): mkdir -p $(TIMESTAMPS) -$(REQUIREMENTS_TIMESTAMP): $(TIMESTAMPS) $(PIP_FILE) $(PIP_FILE_LOCK) - pipenv install - @touch $(REQUIREMENTS_TIMESTAMP) - - -$(DEV_REQUIREMENTS_TIMESTAMP): $(TIMESTAMPS) $(PIP_FILE) $(PIP_FILE_LOCK) - pipenv install --dev - @touch $(DEV_REQUIREMENTS_TIMESTAMP) diff --git a/buildspec.yml b/buildspec.yml index a4524e9..71fd2ab 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -2,28 +2,32 @@ version: 0.2 env: variables: - IMAGE_BASE_NAME: "swisstopo/service-name" + IMAGE_BASE_NAME: "service-name" + REGISTRY: "974517877189.dkr.ecr.eu-central-1.amazonaws.com" SHELL: /bin/bash AWS_DEFAULT_REGION: eu-central-1 USER: "aws_code_build" TEST_REPORT_DIR: "./tests/report" TEST_REPORT_FILE: "nose2-junit.xml" PIPENV_NOSPIN: 1 - parameter-store: - CI_DOCKERHUB_USER: "/dockerhub/user" - CI_DOCKERHUB_PASSWORD: "/dockerhub/password" phases: install: runtime-versions: docker: 18 commands: - - echo "Installing necessary softwares" - - docker login -u ${CI_DOCKERHUB_USER} -p ${CI_DOCKERHUB_PASSWORD} - - apt-get update && apt-get install -y docker-compose python3-pip + - echo "Installing necessary dependencies" + - apt-get update && apt-get install -y docker-compose python3-pip python3-venv pass gnupg2 + - echo "Install aws cli v2 for docker login to ECR registry" + - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + - unzip awscliv2.zip + - ./aws/install + - aws --version + - echo "Login to AWS ECR docker registry" + - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${REGISTRY} pre_build: commands: - - echo "export of the image tag for build and push purposes" + - echo "Export of the image tag for build and push purposes" # Reading git branch (the utility in the deploy script is unable to read it automatically on CodeBuild) # see https://stackoverflow.com/questions/47657423/get-github-git-branch-for-aws-codebuild - export GITHUB_BRANCH="$(git symbolic-ref HEAD --short 2>/dev/null)" @@ -35,16 +39,22 @@ phases: - export GITHUB_COMMIT=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7) - export GITHUB_TAG="$(git describe --tags 2>/dev/null)" - echo "GITHUB_BRANCH=${GITHUB_BRANCH} GITHUB_COMMIT=${GITHUB_COMMIT} GITHUB_TAG=${GITHUB_TAG} DOCKER_IMG_TAG=${DOCKER_IMG_TAG}" - - echo "creating a clean environment" + - echo "Creating a clean environment" - make ci + - echo "Check code formatting" + - make ci-check-format + - echo "Linting code" + - make lint + - echo "Linting openapi spec" + - make lint-spec build: commands: - echo Build started on $(date) - - export DOCKER_IMG_TAG=${IMAGE_BASE_NAME}:${GITHUB_TAG} - - export DOCKER_IMG_TAG_LATEST=${IMAGE_BASE_NAME}:${GITHUB_BRANCH}.latest + - export DOCKER_IMG_TAG=${REGISTRY}/${IMAGE_BASE_NAME}:${GITHUB_TAG} + - export DOCKER_IMG_TAG_LATEST=${REGISTRY}/${IMAGE_BASE_NAME}:${GITHUB_BRANCH}.latest - |- if [ "${GITHUB_TAG}" = "" ] ; then - export DOCKER_IMG_TAG=${IMAGE_BASE_NAME}:${GITHUB_BRANCH}.${GITHUB_COMMIT} + export DOCKER_IMG_TAG=${REGISTRY}/${IMAGE_BASE_NAME}:${GITHUB_BRANCH}.${GITHUB_COMMIT} export GITHUB_TAG=${GITHUB_COMMIT} fi - echo "Building docker image with tags ${DOCKER_IMG_TAG} and ${DOCKER_IMG_TAG_LATEST}" @@ -58,12 +68,6 @@ phases: post_build: commands: - - echo "Linting code..." - - make lint - - echo "Linting spec..." - - make lint-spec - - echo "Checking code style..." - - make ci-check-format - echo "Unit testing..." - mkdir -p ${TEST_REPORT_DIR} - TEST_REPORT_DIR=${TEST_REPORT_DIR} TEST_REPORT_FILE=${TEST_REPORT_FILE} make test