Added ECR docker registry and small other improvements

Makefile

@@ -10,13 +10,24 @@ CURRENT_DIR := $(shell pwd)
 TEST_REPORT_DIR ?= $(CURRENT_DIR)/tests/report
 TEST_REPORT_FILE ?= nose2-junit.xml
 
+# Docker metadata
+GIT_HASH = `git rev-parse HEAD`
+GIT_HASH_SHORT = `git rev-parse --short HEAD`
+GIT_BRANCH = `git symbolic-ref HEAD --short 2>/dev/null`
+GIT_DIRTY = `git status --porcelain`
+GIT_TAG = `git describe --tags || echo "no version info"`
+AUTHOR = $(USER)
+
 # general targets timestamps
 TIMESTAMPS = .timestamps
-REQUIREMENTS_TIMESTAMP = $(TIMESTAMPS)/.requirements.timestamp
-DEV_REQUIREMENTS_TIMESTAMP = $(TIMESTAMPS)/.dev-requirements.timestamps
+REQUIREMENTS := $(TIMESTAMPS) ${VOLUMES_MINIO} $(LOGS_DIR) $(PIP_FILE) $(PIP_FILE_LOCK)
 
 # Docker variables
-DOCKER_IMG_LOCAL_TAG = swisstopo/$(SERVICE_NAME):local
+DOCKER_REGISTRY = 974517877189.dkr.ecr.eu-central-1.amazonaws.com
+DOCKER_IMG_LOCAL_TAG := $(DOCKER_REGISTRY)/$(SERVICE_NAME):local-$(USER)-$(GIT_HASH_SHORT)
+
+# AWS variables
+AWS_DEFAULT_REGION = eu-central-1
 
 # Find all python files that are not inside a hidden directory (directory starting with .)
 PYTHON_FILES := $(shell find ./* -type f -name "*.py" -print)
@@ -38,13 +49,6 @@ ISORT := $(PIPENV_RUN) isort
 NOSE := $(PIPENV_RUN) nose2
 PYLINT := $(PIPENV_RUN) pylint
 
-# Docker metadata
-GIT_HASH := `git rev-parse HEAD`
-GIT_BRANCH := `git symbolic-ref HEAD --short 2>/dev/null`
-GIT_DIRTY := `git status --porcelain`
-GIT_TAG := `git describe --tags || echo "no version info"`
-AUTHOR := $(USER)
-
 
 all: help
 
@@ -68,8 +72,10 @@ help:
 	@echo -e " \033[1mLOCAL SERVER TARGETS\033[0m "
 	@echo "- serve              Run the project using the flask debug server. Port can be set by Env variable HTTP_PORT (default: 5000)"
 	@echo "- gunicornserve      Run the project using the gunicorn WSGI server. Port can be set by Env variable DEBUG_HTTP_PORT (default: 5000)"
-	@echo "- serve-spec         Serve the spec using Redoc on localhost:8080"
+	@echo "- serve-spec-redoc   Serve the spec using Redoc on localhost:8080"
+	@echo "- serve-spec-swagger Serve the spec using Redoc on localhost:8080/swagger"
 	@echo -e " \033[1mDocker TARGETS\033[0m "
+	@echo "- dockerlogin        Login to the AWS ECR registery for pulling/pushing docker images"
 	@echo "- dockerbuild        Build the project localy (with tag := $(DOCKER_IMG_LOCAL_TAG)) using the gunicorn WSGI server inside a container"
 	@echo "- dockerpush         Build and push the project localy (with tag := $(DOCKER_IMG_LOCAL_TAG))"
 	@echo "- dockerrun          Run the project using the gunicorn WSGI server inside a container (exposed port: 5000)"
@@ -82,79 +88,75 @@ help:
 # Build targets. Calling setup is all that is needed for the local files to be installed as needed.
 
 .PHONY: dev
-dev: $(DEV_REQUIREMENTS_TIMESTAMP)
+dev: $(REQUIREMENTS)
+	pipenv install --dev
 	pipenv shell
 
 
 .PHONY: setup
-setup: $(REQUIREMENTS_TIMESTAMP)
+setup: $(REQUIREMENTS)
+	pipenv install
 	pipenv shell
 
 .PHONY: ci
-ci: $(TIMESTAMPS) $(PIP_FILE) $(PIP_FILE_LOCK)
+ci: $(REQUIREMENTS)
 	# Create virtual env with all packages for development using the Pipfile.lock
 	pipenv sync --dev
 
 
 # linting target, calls upon yapf to make sure your code is easier to read and respects some conventions.
 
 .PHONY: format
-format: $(DEV_REQUIREMENTS_TIMESTAMP)
+format:
 	$(YAPF) -p -i --style .style.yapf $(PYTHON_FILES)
 	$(ISORT) $(PYTHON_FILES)
 
 
-.PHONY: lint
-lint: $(DEV_REQUIREMENTS_TIMESTAMP)
-	$(PYLINT) $(PYTHON_FILES)
-
-
-.PHONY: format-lint
-format-lint: format lint
-
-
 .PHONY: ci-check-format
 ci-check-format: format
 	@if [[ -n `git status --porcelain` ]]; then \
-	 	>&2 echo "ERROR: Code was not formatted correctly"; \
+		>&2 echo "ERROR: the following files are not formatted correctly:"; \
+		>&2 git status --porcelain; \
 		exit 1; \
 	fi
 
-# Spec targets
 
-.PHONY: lint-spec
-lint-spec:
-	docker run --volume "$(PWD)":/data jamescooke/openapi-validator -e openapi.yml
+.PHONY: lint
+lint:
+	$(PYLINT) $(PYTHON_FILES)
 
 
-.PHONY: serve-spec
-serve-spec:
-	docker run -it --rm -p 8080:80 \
-		-v "$(PWD)/openapi.yml":/usr/share/nginx/html/openapi.yml \
-		-e SPEC_URL=openapi.yml redocly/redoc
+.PHONY: format-lint
+format-lint: format lint
+
 
 # Test target
 
 .PHONY: test
-test: $(DEV_REQUIREMENTS_TIMESTAMP)
+test: $(TEST_REPORT_DIR)
 	mkdir -p $(TEST_REPORT_DIR)
 	$(NOSE) -c tests/unittest.cfg --verbose --junit-xml-path $(TEST_REPORT_DIR)/$(TEST_REPORT_FILE) -s tests/
 
 
 # Serve targets. Using these will run the application on your local machine. You can either serve with a wsgi front (like it would be within the container), or without.
 
 .PHONY: serve
-serve: $(REQUIREMENTS_TIMESTAMP)
+serve:
 	FLASK_APP=service_launcher FLASK_DEBUG=1 $(FLASK) run --host=0.0.0.0 --port=$(HTTP_PORT)
 
 
 .PHONY: gunicornserve
-gunicornserve: $(REQUIREMENTS_TIMESTAMP)
+gunicornserve:
 	$(PYTHON) wsgi.py
 
 
 # Docker related functions.
 
+.PHONY: dockerlogin
+dockerlogin:
+	aws --profile swisstopo-bgdi-builder ecr get-login-password --region $(AWS_DEFAULT_REGION) | docker login --username AWS --password-stdin $(DOCKER_REGISTRY)
+
+
 .PHONY: dockerbuild
 dockerbuild:
 	docker build \
@@ -180,6 +182,31 @@ shutdown:
 	HTTP_PORT=$(HTTP_PORT) SERVICE_NAME=$(SERVICE_NAME) docker-compose down
 
 
+# Spec targets
+
+.PHONY: lint-spec
+lint-spec:
+	docker run --volume "$(PWD)":/data jamescooke/openapi-validator -e openapi.yml
+
+
+.PHONY: serve-spec
+serve-spec-redoc:
+	docker run -it --rm -p 8080:80 \
+		-v "$(PWD)/openapi.yml":/usr/share/nginx/html/openapi.yml \
+		-e SPEC_URL=openapi.yml redocly/redoc
+
+
+.PHONY: serve-spec-swagger
+serve-spec-swagger:
+	echo "SWAGGER UI on http://localhost:8080/swagger"
+	docker run -p 8080:8080 \
+		-e BASE_URL=/swagger -e SWAGGER_JSON=/openapi.yaml \
+		-v ${PWD}/openapi.yml:/openapi.yaml \
+		swaggerapi/swagger-ui
+
+
+# Clean targets
+
 .PHONY: clean_venv
 clean_venv:
 	pipenv --rm
@@ -189,7 +216,6 @@ clean_venv:
 clean: clean_venv
 	@# clean python cache files
 	find . -name __pycache__ -type d -print0 | xargs -I {} -0 rm -rf "{}"
-	rm -rf $(PYTHON_LOCAL_DIR)
 	rm -rf $(TEST_REPORT_DIR)
 	rm -rf $(TIMESTAMPS)
 
@@ -199,11 +225,3 @@ clean: clean_venv
 $(TIMESTAMPS):
 	mkdir -p $(TIMESTAMPS)
 
-$(REQUIREMENTS_TIMESTAMP): $(TIMESTAMPS) $(PIP_FILE) $(PIP_FILE_LOCK)
-	pipenv install
-	@touch $(REQUIREMENTS_TIMESTAMP)
-
-
-$(DEV_REQUIREMENTS_TIMESTAMP): $(TIMESTAMPS) $(PIP_FILE) $(PIP_FILE_LOCK)
-	pipenv install --dev
-	@touch $(DEV_REQUIREMENTS_TIMESTAMP)

buildspec.yml

@@ -2,28 +2,32 @@ version: 0.2
 
 env:
   variables:
-    IMAGE_BASE_NAME: "swisstopo/service-name"
+    IMAGE_BASE_NAME: "service-name"
+    REGISTRY: "974517877189.dkr.ecr.eu-central-1.amazonaws.com"
     SHELL: /bin/bash
     AWS_DEFAULT_REGION: eu-central-1
     USER: "aws_code_build"
     TEST_REPORT_DIR: "./tests/report"
     TEST_REPORT_FILE: "nose2-junit.xml"
     PIPENV_NOSPIN: 1
-  parameter-store:
-    CI_DOCKERHUB_USER: "/dockerhub/user"
-    CI_DOCKERHUB_PASSWORD: "/dockerhub/password"
 
 phases:
   install:
     runtime-versions:
       docker: 18
     commands:
-      - echo "Installing necessary softwares"
-      - docker login -u ${CI_DOCKERHUB_USER} -p ${CI_DOCKERHUB_PASSWORD}
-      - apt-get update && apt-get install -y docker-compose python3-pip
+      - echo "Installing necessary dependencies"
+      - apt-get update && apt-get install -y docker-compose python3-pip python3-venv pass gnupg2
+      - echo "Install aws cli v2 for docker login to ECR registry"
+      - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+      - unzip awscliv2.zip
+      - ./aws/install
+      - aws --version
+      - echo "Login to AWS ECR docker registry"
+      - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${REGISTRY}
   pre_build:
     commands:
-      - echo "export of the image tag for build and push purposes"
+      - echo "Export of the image tag for build and push purposes"
       # Reading git branch (the utility in the deploy script is unable to read it automatically on CodeBuild)
       # see https://stackoverflow.com/questions/47657423/get-github-git-branch-for-aws-codebuild
       - export GITHUB_BRANCH="$(git symbolic-ref HEAD --short 2>/dev/null)"
@@ -35,16 +39,22 @@ phases:
       - export GITHUB_COMMIT=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7)
       - export GITHUB_TAG="$(git describe --tags 2>/dev/null)"
       - echo "GITHUB_BRANCH=${GITHUB_BRANCH} GITHUB_COMMIT=${GITHUB_COMMIT} GITHUB_TAG=${GITHUB_TAG} DOCKER_IMG_TAG=${DOCKER_IMG_TAG}"
-      - echo "creating a clean environment"
+      - echo "Creating a clean environment"
       - make ci
+      - echo "Check code formatting"
+      - make ci-check-format
+      - echo "Linting code"
+      - make lint
+      - echo "Linting openapi spec"
+      - make lint-spec
   build:
     commands:
       - echo Build started on $(date)
-      - export DOCKER_IMG_TAG=${IMAGE_BASE_NAME}:${GITHUB_TAG}
-      - export DOCKER_IMG_TAG_LATEST=${IMAGE_BASE_NAME}:${GITHUB_BRANCH}.latest
+      - export DOCKER_IMG_TAG=${REGISTRY}/${IMAGE_BASE_NAME}:${GITHUB_TAG}
+      - export DOCKER_IMG_TAG_LATEST=${REGISTRY}/${IMAGE_BASE_NAME}:${GITHUB_BRANCH}.latest
       - |-
         if [ "${GITHUB_TAG}" = "" ] ; then
-          export DOCKER_IMG_TAG=${IMAGE_BASE_NAME}:${GITHUB_BRANCH}.${GITHUB_COMMIT}
+          export DOCKER_IMG_TAG=${REGISTRY}/${IMAGE_BASE_NAME}:${GITHUB_BRANCH}.${GITHUB_COMMIT}
           export GITHUB_TAG=${GITHUB_COMMIT}
         fi
       - echo "Building docker image with tags ${DOCKER_IMG_TAG} and ${DOCKER_IMG_TAG_LATEST}"
@@ -58,12 +68,6 @@ phases:
 
   post_build:
     commands:
-      - echo "Linting code..."
-      - make lint
-      - echo "Linting spec..."
-      - make lint-spec
-      - echo "Checking code style..."
-      - make ci-check-format
       - echo "Unit testing..."
       - mkdir -p ${TEST_REPORT_DIR}
       - TEST_REPORT_DIR=${TEST_REPORT_DIR} TEST_REPORT_FILE=${TEST_REPORT_FILE} make test