Merge pull request #13 from PizzaFactory/pr-update-to-the-upstream

Update to the upstream

.github/CODEOWNERS

@@ -1,2 +1,2 @@
 # Global Owners
-* @vparfonov @l0rd @rhopp @skabashnyuk @amisevsk @nickboldt
+* @vparfonov @l0rd @rhopp @skabashnyuk @amisevsk @nickboldt @ibuziuk

README.md

@@ -11,7 +11,10 @@ This repository holds ready-to-use Devfiles for different languages and technolo
 
 Execute
 ```shell
-docker build --no-cache -t quay.io/eclipse/che-devfile-registry:nightly .
+docker build --no-cache -t quay.io/eclipse/che-devfile-registry:nightly --target registry .
+
+# or to use & create a RHEL-based image
+docker build --no-cache -t quay.io/eclipse/che-devfile-registry:nightly -f build/dockerfiles/rhel.Dockerfile --target registry.
 ```
 Where `--no-cache` is needed to prevent usage of cached layers with devfile registry files.
 Useful when you change devfile files and rebuild the image.
@@ -21,6 +24,14 @@ Though you may also just provide the image to the older versions of Docker (ex.
 
 `quay.io/eclipse/che-devfile-registry:nightly` image would be rebuilt after each commit in master.
 
+### Offline registry
+
+The default docker build has multiple targets:
+- `--target registry` is used to build the default devfile registry, where projects in devfiles refer to publically hosted git repos
+- `--target offline-registry` is used to build a devfile registry which self-hosts projects as zip files.
+
+The offline registry build will, during the docker build, pull zips from all projects hosted on github and store them in the `/resources` path. This registry should be deployed with environment variable `CHE_DEVFILE_REGISTRY_URL` set to the URL of the route/endpoint that exposes the devfile registry, as devfiles need to be rewritten to point to internally hosted zip files.
+
 ## OpenShift
 You can deploy Che devfile registry on Openshift with command.
 ```

VERSION

@@ -1 +1 @@
-7.3.0-SNAPSHOT
+7.5.0-SNAPSHOT

arbitrary-users-patch/base_images

@@ -1,6 +1,5 @@
 che-cpp-rhel7           registry.access.redhat.com/devtools/llvm-toolset-rhel7
 che-dotnet-2.2          mcr.microsoft.com/dotnet/core/sdk:2.2-stretch
-che-golang-1.10         golang:1.10.7-stretch
 che-golang-1.12         golang:1.12-stretch
 che-java11-gradle       gradle:5.2.1-jdk11
 che-java11-maven        maven:3.6.0-jdk-11

arbitrary-users-patch/happy-path/Dockerfile

@@ -0,0 +1,15 @@
+ARG TAG
+FROM quay.io/eclipse/che-java11-maven:${TAG}
+
+USER root
+
+RUN cd / && \
+    git clone /~https://github.com/spring-projects/spring-petclinic && \
+    cd /spring-petclinic && \
+    mvn clean package && \
+    mkdir -p /home/user/.m2/repository && \
+    cp -r /root/.m2/repository/* /home/user/.m2/repository && \
+    rm -rf spring-petclinic/ /root/.m2/repository/* && \
+    chmod -R g=u /home/user
+
+USER 10001

arbitrary-users-patch/happy-path/build_happy_path_image.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Copyright (c) 2012-2019 Red Hat, Inc.
+# This program and the accompanying materials are made
+# available under the terms of the Eclipse Public License 2.0
+# which is available at https://www.eclipse.org/legal/epl-2.0/
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+
+set -e
+
+SCRIPT_DIR=$(cd "$(dirname "$0")"; pwd)
+
+DEFAULT_REGISTRY="quay.io"
+DEFAULT_ORGANIZATION="eclipse"
+DEFAULT_TAG="nightly"
+
+REGISTRY=${REGISTRY:-${DEFAULT_REGISTRY}}
+ORGANIZATION=${ORGANIZATION:-${DEFAULT_ORGANIZATION}}
+TAG=${TAG:-${DEFAULT_TAG}}
+
+NAME_FORMAT="${REGISTRY}/${ORGANIZATION}"
+
+PUSH_IMAGES=false
+if [ "$1" == "--push" ]; then
+  PUSH_IMAGES=true
+fi
+
+# Build image for happy-path tests with precashed mvn dependencies
+docker build -t "${NAME_FORMAT}/happy-path:${TAG}" --no-cache --build-arg TAG="${TAG}" "${SCRIPT_DIR}"/
+if ${PUSH_IMAGES}; then
+    echo "Pushing ${NAME_FORMAT}/happy-path:${TAG}" to remote registry
+    docker push "${NAME_FORMAT}/happy-path:${TAG}"
+fi

build.sh

@@ -1,26 +1,85 @@
-#!/bin/sh
+#!/bin/bash
 #
-# Copyright (c) 2012-2018 Red Hat, Inc.
+# Copyright (c) 2019 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/
 #
 # SPDX-License-Identifier: EPL-2.0
 #
-# Build Che devfile registry image. Note that this script will read the version
-# in ./VERSION; if it is *-SNAPSHOT, devfiles in the registry will use nightly-tagged
-# images with the arbitrary user IDs patch. If ./VERSION contains otherwise,
-# the devfiles in the registry will instead use the value in ./VERSION.
-#
 
+set -e
+
+REGISTRY="quay.io"
+ORGANIZATION="eclipse"
+TAG="nightly"
+TARGET="registry"
+DOCKERFILE="./build/dockerfiles/Dockerfile"
+
+USAGE="
+Usage: ./build.sh [OPTIONS]
+Options:
+    --help
+        Print this message.
+    --tag, -t [TAG]
+        Docker image tag to be used for image; default: 'nightly'
+    --registry, -r [REGISTRY]
+        Docker registry to be used for image; default 'quay.io'
+    --organization, -o [ORGANIZATION]
+        Docker image organization to be used for image; default: 'eclipse'
+    --offline
+        Build offline version of registry, with all sample projects
+        cached in the registry; disabled by default.
+    --rhel
+        Build registry using UBI images instead of default
+"
+
+function print_usage() {
+    echo -e "$USAGE"
+}
+
+function parse_arguments() {
+    while [[ $# -gt 0 ]]; do
+        key="$1"
+        case $key in
+            -t|--tag)
+            TAG="$2"
+            shift; shift;
+            ;;
+            -r|--registry)
+            REGISTRY="$2"
+            shift; shift;
+            ;;
+            -o|--organization)
+            ORGANIZATION="$2"
+            shift; shift;
+            ;;
+            --offline)
+            TARGET="offline-registry"
+            shift
+            ;;
+            --rhel)
+            DOCKERFILE="./build/dockerfiles/rhel.Dockerfile"
+            shift
+            ;;
+            *)
+            print_usage
+            exit 0
+        esac
+    done
+}
+
+parse_arguments "$@"
+
+IMAGE="${REGISTRY}/${ORGANIZATION}/che-devfile-registry:${TAG}"
 VERSION=$(head -n 1 VERSION)
 case $VERSION in
   *SNAPSHOT)
     echo "Snapshot version (${VERSION}) specified in $(find . -name VERSION): building nightly plugin registry."
-    docker build -t "quay.io/eclipse/che-devfile-registry:nightly" -f ./build/dockerfiles/Dockerfile .
+    docker build -t "${IMAGE}" -f ${DOCKERFILE} --target ${TARGET} .
     ;;
   *)
     echo "Release version specified in $(find . -name VERSION): Building plugin registry for release ${VERSION}."
-    docker build -t "quay.io/eclipse/che-devfile-registry:${VERSION}" -f ./build/dockerfiles/Dockerfile . --build-arg "PATCHED_IMAGES_TAG=${VERSION}"
+    docker build -t "${IMAGE}" -f ${DOCKERFILE} --target ${TARGET} --build-arg "PATCHED_IMAGES_TAG=${VERSION}" .
     ;;
 esac

build/dockerfiles/Dockerfile

@@ -7,7 +7,7 @@
 # SPDX-License-Identifier: EPL-2.0
 #
 FROM alpine:3.10 AS builder
-RUN apk add --no-cache py-pip jq bash && pip install yq
+RUN apk add --no-cache py-pip jq bash wget git && pip install yq
 
 # Registry, organization, and tag to use for base images in dockerfiles. Devfiles
 # will be rewritten during build to use these values for base images.
@@ -16,18 +16,33 @@ ARG PATCHED_IMAGES_ORG="eclipse"
 ARG PATCHED_IMAGES_TAG="nightly"
 
 COPY ./build/scripts ./arbitrary-users-patch/base_images /build/
-COPY /devfiles /build/devfiles
+COPY ./devfiles /build/devfiles
 WORKDIR /build/
 RUN TAG=${PATCHED_IMAGES_TAG} \
     ORGANIZATION=${PATCHED_IMAGES_ORG} \
     REGISTRY=${PATCHED_IMAGES_REG} \
     ./update_devfile_patched_image_tags.sh
 RUN ./check_mandatory_fields.sh devfiles
 RUN ./index.sh > /build/devfiles/index.json
+RUN ./list_referenced_images.sh devfiles > /build/devfiles/external_images.txt
+RUN chmod -R g+rwX /build/devfiles
 
-FROM registry.centos.org/centos/httpd-24-centos7
+FROM registry.centos.org/centos/httpd-24-centos7 AS registry
 RUN mkdir /var/www/html/devfiles
 COPY .htaccess README.md /var/www/html/
 COPY --from=builder /build/devfiles /var/www/html/devfiles
-USER 0
-RUN chmod -R g+rwX /var/www/html/devfiles
+COPY ./build/dockerfiles/entrypoint.sh /usr/bin/
+ENTRYPOINT ["/usr/bin/entrypoint.sh"]
+CMD ["/usr/bin/run-httpd"]
+
+
+# Offline registry: download project zips and place them in /build/resources
+FROM builder AS offline-builder
+RUN ./cache_projects.sh devfiles resources && \
+    ./cache_images.sh devfiles resources && \
+    chmod -R g+rwX /build
+
+# Offline registry: copy updated devfile.yamls and cached projects
+FROM registry AS offline-registry
+COPY --from=offline-builder /build/devfiles /var/www/html/devfiles
+COPY --from=offline-builder /build/resources /var/www/html/resources

build/dockerfiles/content_sets_epel7.repo

@@ -0,0 +1,5 @@
+[epel-7]
+name=epel-7
+baseurl=https://download.fedoraproject.org/pub/epel/7/x86_64/
+enabled=1
+gpgcheck=0

build/dockerfiles/entrypoint.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+#
+# Copyright (c) 2012-2018 Red Hat, Inc.
+# This program and the accompanying materials are made
+# available under the terms of the Eclipse Public License 2.0
+# which is available at https://www.eclipse.org/legal/epl-2.0/
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+# Updates plugin runner images to point a registry defined by environment
+# variables
+#     CHE_DEVFILE_IMAGES_REGISTRY_URL
+#     CHE_DEVFILE_IMAGES_REGISTRY_ORGANIZATION
+#     CHE_DEVFILE_IMAGES_REGISTRY_TAG
+#
+# By default, this script will operate on the `/var/www/html/devfiles` directory.
+# This can be overridden by the environment variable $DEVFILES_DIR
+#
+# In addition, this script will perform the necessary set up for the offline
+# devfile registry, replacing placeholders in all devfiles based off environment
+# variable
+#     CHE_DEVFILE_REGISTRY_URL
+# which should be set to the public endpoint for this registry.
+#
+# Will execute any arguments on completion (`exec $@`)
+
+set -e
+
+REGISTRY=${CHE_DEVFILE_IMAGES_REGISTRY_URL}
+ORGANIZATION=${CHE_DEVFILE_IMAGES_REGISTRY_ORGANIZATION}
+TAG=${CHE_DEVFILE_IMAGES_REGISTRY_TAG}
+PUBLIC_URL=${CHE_DEVFILE_REGISTRY_URL}
+
+DEFAULT_DEVFILES_DIR="/var/www/html/devfiles"
+DEVFILES_DIR="${DEVFILES_DIR:-${DEFAULT_DEVFILES_DIR}}"
+INDEX_JSON="${DEVFILES_DIR}/index.json"
+
+# Regex used to break an image reference into groups:
+#   \1 - Whitespace and (optional) quotation preceding image reference
+#   \2 - Registry portion of image, e.g. (quay.io)/eclipse/che-theia:tag
+#   \3 - Organization portion of image, e.g. quay.io/(eclipse)/che-theia:tag
+#   \4 - Image name portion of image, e.g. quay.io/eclipse/(che-theia):tag
+#   \5 - Tag of image, e.g. quay.io/eclipse/che-theia:(tag)
+#   \6 - Optional quotation following image reference
+IMAGE_REGEX='([[:space:]]*"?)([._:a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*):([._a-zA-Z0-9-]*)("?)'
+
+# We can't use the `-d` option for readarray because
+# registry.centos.org/centos/httpd-24-centos7 ships with Bash 4.2
+# The below command will fail if any path contains whitespace
+readarray -t devfiles < <(find "${DEVFILES_DIR}" -name 'devfile.yaml')
+readarray -t metas < <(find "${DEVFILES_DIR}" -name 'meta.yaml')
+for devfile in "${devfiles[@]}"; do
+  echo "Checking devfile $devfile"
+  # Need to update each field separately in case they are not defined.
+  # Defaults don't work because registry and tags may be different.
+  if [ -n "$REGISTRY" ]; then
+    echo "    Updating image registry to $REGISTRY"
+    sed -i -E "s|image:$IMAGE_REGEX|image:\1${REGISTRY}/\3/\4:\5\6|" "$devfile"
+  fi
+  if [ -n "$ORGANIZATION" ]; then
+    echo "    Updating image organization to $ORGANIZATION"
+    sed -i -E "s|image:$IMAGE_REGEX|image:\1\2/${ORGANIZATION}/\4:\5\6|" "$devfile"
+  fi
+  if [ -n "$TAG" ]; then
+    echo "    Updating image tag to $TAG"
+    sed -i -E "s|image:$IMAGE_REGEX|image:\1\2/\3/\4:${TAG}\6|" "$devfile"
+  fi
+done
+
+if [ -n "$PUBLIC_URL" ]; then
+  echo "Updating devfiles to point at internal project zip files"
+  PUBLIC_URL=${PUBLIC_URL%/}
+  sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${PUBLIC_URL}|" "${devfiles[@]}" "${metas[@]}" "$INDEX_JSON"
+else
+  if grep -q '{{ DEVFILE_REGISTRY_URL }}' "${devfiles[@]}"; then
+    echo "WARNING: environment variable 'CHE_DEVFILE_REGISTRY_URL' not configured" \
+         "for an offline build of this registry. This may cause issues with importing" \
+         "projects in a workspace."
+    # Experimental workaround -- detect service IP for che-devfile-registry
+    # Depends on service used being named 'che-devfile-registry' and only works
+    # within the cluster (i.e. browser-side retrieval won't work)
+    URL="http://${CHE_DEVFILE_REGISTRY_SERVICE_HOST}:${CHE_DEVFILE_REGISTRY_SERVICE_PORT}"
+    sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${URL}|" "${devfiles[@]}" "${metas[@]}" "$INDEX_JSON"
+  fi
+fi
+
+exec "${@}"