Merge pull request #32 from PizzaFactory/prp-update-to-the-upstream

Update to the upstream

.github/CODEOWNERS

@@ -1,2 +1,2 @@
 # Global Owners
-* @tsmaeder @JPinkney @svor @sunix @benoitf
+* @tsmaeder @JPinkney @svor @benoitf @ericwill @vitaliy-guliy @vinokurig

VERSION

@@ -1 +1 @@
-7.9.0-SNAPSHOT
+7.12.0-SNAPSHOT

arbitrary-users-patch/Dockerfile

@@ -3,7 +3,7 @@ FROM ${FROM_IMAGE}
 USER 0
 # Set permissions on /etc/passwd and /home to allow arbitrary users to write
 COPY --chown=0:0 entrypoint.sh /
-RUN mkdir -p /home/user && chgrp -R 0 /home && chmod -R g=u /etc/passwd /home && chmod +x /entrypoint.sh
+RUN mkdir -p /home/user && chgrp -R 0 /home && chmod -R g=u /etc/passwd /etc/group /home && chmod +x /entrypoint.sh
 
 USER 10001
 ENV HOME=/home/user

arbitrary-users-patch/base_images

@@ -1,13 +1,13 @@
 che-cpp-rhel7           registry.access.redhat.com/devtools/llvm-toolset-rhel7
 che-dotnet-2.2          mcr.microsoft.com/dotnet/core/sdk:2.2-stretch
 che-golang-1.12         golang:1.12-stretch
-che-java11-gradle       gradle:5.2.1-jdk11
+che-java11-gradle       gradle:6.2.1-jdk11
 che-java11-maven        maven:3.6.0-jdk-11
 che-java8-maven         maven:3.6.1-jdk-8
 che-nodejs10-community  node:10.16
 che-nodejs10-ubi        registry.access.redhat.com/ubi8/nodejs-10
 che-nodejs8-centos      registry.centos.org/che-stacks/centos-nodejs
-che-php-7               eclipse/php:7.1-che7
+che-php-7               quay.io/eclipse/che-php-base:7.4
 che-python-3.6          centos/python-36-centos7:1
 che-python-3.7          python:3.7.4-slim
 che-quarkus             quay.io/quarkus/centos-quarkus-maven:19.2.1

arbitrary-users-patch/build_images.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

arbitrary-users-patch/entrypoint.sh

@@ -10,10 +10,11 @@ if [ -w "${HOME}" ] && [ ! -f "${HOME}"/.bashrc ]; then
   echo "PS1='\s-\v \w \$ '" > "${HOME}"/.bashrc
 fi
 
-# Add current (arbitrary) user to /etc/passwd
+# Add current (arbitrary) user to /etc/passwd and /etc/group
 if ! whoami &> /dev/null; then
   if [ -w /etc/passwd ]; then
     echo "${USER_NAME:-user}:x:$(id -u):0:${USER_NAME:-user} user:${HOME}:/bin/bash" >> /etc/passwd
+    echo "${USER_NAME:-user}:x:$(id -u):" >> /etc/group
   fi
 fi
 

build.sh

@@ -14,6 +14,7 @@ REGISTRY="quay.io"
 ORGANIZATION="eclipse"
 TAG="nightly"
 TARGET="registry"
+USE_DIGESTS=false
 DOCKERFILE="./build/dockerfiles/Dockerfile"
 
 USAGE="
@@ -27,6 +28,8 @@ Options:
         Docker registry to be used for image; default 'quay.io'
     --organization, -o [ORGANIZATION]
         Docker image organization to be used for image; default: 'eclipse'
+    --use-digests
+        Build registry to use images pinned by digest instead of tag
     --offline
         Build offline version of registry, with all sample projects
         cached in the registry; disabled by default.
@@ -54,6 +57,10 @@ function parse_arguments() {
             ORGANIZATION="$2"
             shift; shift;
             ;;
+            --use-digests)
+            USE_DIGESTS=true
+            shift
+            ;;
             --offline)
             TARGET="offline-registry"
             shift
@@ -76,10 +83,19 @@ VERSION=$(head -n 1 VERSION)
 case $VERSION in
   *SNAPSHOT)
     echo "Snapshot version (${VERSION}) specified in $(find . -name VERSION): building nightly plugin registry."
-    docker build -t "${IMAGE}" -f ${DOCKERFILE} --target ${TARGET} .
+    docker build \
+        -t "${IMAGE}" \
+        -f ${DOCKERFILE} \
+        --build-arg "USE_DIGESTS=${USE_DIGESTS}" \
+        --target ${TARGET} .
     ;;
   *)
     echo "Release version specified in $(find . -name VERSION): Building plugin registry for release ${VERSION}."
-    docker build -t "${IMAGE}" -f ${DOCKERFILE} --target ${TARGET} --build-arg "PATCHED_IMAGES_TAG=${VERSION}" .
+    docker build \
+        -t "${IMAGE}" \
+        -f ${DOCKERFILE} \
+        --build-arg "USE_DIGESTS=${USE_DIGESTS}" \
+        --build-arg "PATCHED_IMAGES_TAG=${VERSION}" \
+        --target ${TARGET} .
     ;;
 esac

build/dockerfiles/Dockerfile

@@ -7,13 +7,14 @@
 # SPDX-License-Identifier: EPL-2.0
 #
 FROM alpine:3.10 AS builder
-RUN apk add --no-cache py-pip jq bash wget git && pip install yq
+RUN apk add --no-cache py-pip jq bash wget git skopeo && pip install yq
 
 # Registry, organization, and tag to use for base images in dockerfiles. Devfiles
 # will be rewritten during build to use these values for base images.
 ARG PATCHED_IMAGES_REG="quay.io"
 ARG PATCHED_IMAGES_ORG="eclipse"
 ARG PATCHED_IMAGES_TAG="nightly"
+ARG USE_DIGESTS=false
 
 COPY ./build/scripts ./arbitrary-users-patch/base_images /build/
 COPY ./devfiles /build/devfiles
@@ -23,6 +24,7 @@ RUN TAG=${PATCHED_IMAGES_TAG} \
     REGISTRY=${PATCHED_IMAGES_REG} \
     ./update_devfile_patched_image_tags.sh
 RUN ./check_mandatory_fields.sh devfiles
+RUN if [[ ${USE_DIGESTS} == "true" ]]; then ./write_image_digests.sh devfiles; fi
 RUN ./index.sh > /build/devfiles/index.json
 RUN ./list_referenced_images.sh devfiles > /build/devfiles/external_images.txt
 RUN chmod -R g+rwX /build/devfiles

build/dockerfiles/content_sets_centos8_appstream.repo

@@ -0,0 +1,5 @@
+[AppStream]
+name=CentOS-8 - AppStream
+baseurl=http://mirror.centos.org/centos-8/8/AppStream/x86_64/os/
+gpgcheck=0
+enabled=1

build/dockerfiles/entrypoint.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/
@@ -40,9 +40,10 @@ INDEX_JSON="${DEVFILES_DIR}/index.json"
 #   \2 - Registry portion of image, e.g. (quay.io)/eclipse/che-theia:tag
 #   \3 - Organization portion of image, e.g. quay.io/(eclipse)/che-theia:tag
 #   \4 - Image name portion of image, e.g. quay.io/eclipse/(che-theia):tag
-#   \5 - Tag of image, e.g. quay.io/eclipse/che-theia:(tag)
-#   \6 - Optional quotation following image reference
-IMAGE_REGEX='([[:space:]]*"?)([._:a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*):([._a-zA-Z0-9-]*)("?)'
+#   \5 - Optional image digest identifier (empty for tags), e.g. quay.io/eclipse/che-theia(@sha256):digest
+#   \6 - Tag of image or digest, e.g. quay.io/eclipse/che-theia:(tag)
+#   \7 - Optional quotation following image reference
+IMAGE_REGEX='([[:space:]]*"?)([._:a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*)(@sha256)?:([._a-zA-Z0-9-]*)("?)'
 
 # We can't use the `-d` option for readarray because
 # registry.centos.org/centos/httpd-24-centos7 ships with Bash 4.2
@@ -55,15 +56,15 @@ for devfile in "${devfiles[@]}"; do
   # Defaults don't work because registry and tags may be different.
   if [ -n "$REGISTRY" ]; then
     echo "    Updating image registry to $REGISTRY"
-    sed -i -E "s|image:$IMAGE_REGEX|image:\1${REGISTRY}/\3/\4:\5\6|" "$devfile"
+    sed -i -E "s|image:$IMAGE_REGEX|image:\1${REGISTRY}/\3/\4\5:\6\7|" "$devfile"
   fi
   if [ -n "$ORGANIZATION" ]; then
     echo "    Updating image organization to $ORGANIZATION"
-    sed -i -E "s|image:$IMAGE_REGEX|image:\1\2/${ORGANIZATION}/\4:\5\6|" "$devfile"
+    sed -i -E "s|image:$IMAGE_REGEX|image:\1\2/${ORGANIZATION}/\4\5:\6\7|" "$devfile"
   fi
   if [ -n "$TAG" ]; then
     echo "    Updating image tag to $TAG"
-    sed -i -E "s|image:$IMAGE_REGEX|image:\1\2/\3/\4:${TAG}\6|" "$devfile"
+    sed -i -E "s|image:$IMAGE_REGEX|image:\1\2/\3/\4:${TAG}\7|" "$devfile"
   fi
 done
 
@@ -79,7 +80,9 @@ else
     # Experimental workaround -- detect service IP for che-devfile-registry
     # Depends on service used being named 'che-devfile-registry' and only works
     # within the cluster (i.e. browser-side retrieval won't work)
-    URL="http://${CHE_DEVFILE_REGISTRY_SERVICE_HOST}:${CHE_DEVFILE_REGISTRY_SERVICE_PORT}"
+    SERVICE_HOST=$(env | grep DEVFILE_REGISTRY_SERVICE_HOST= | cut -d '=' -f 2)
+    SERVICE_PORT=$(env | grep DEVFILE_REGISTRY_SERVICE_PORT= | cut -d '=' -f 2)
+    URL="http://${SERVICE_HOST}:${SERVICE_PORT}"
     sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${URL}|" "${devfiles[@]}" "${metas[@]}" "$INDEX_JSON"
   fi
 fi

build/dockerfiles/rhel.Dockerfile

@@ -21,8 +21,8 @@ USER 0
 
 ARG BOOTSTRAP=false
 ENV BOOTSTRAP=${BOOTSTRAP}
-ARG LATEST_ONLY=false
-ENV LATEST_ONLY=${LATEST_ONLY}
+ARG USE_DIGESTS=false
+ENV USE_DIGESTS=${USE_DIGESTS}
 
 # to get all the python deps pre-fetched so we can build in Brew:
 # 1. extract files in the container to your local filesystem
@@ -39,7 +39,7 @@ ENV LATEST_ONLY=${LATEST_ONLY}
 
 # NOTE: uncomment for local build. Must also set full registry path in FROM to registry.redhat.io or registry.access.redhat.com
 # enable rhel 7 or 8 content sets (from Brew) to resolve jq as rpm
-COPY ./build/dockerfiles/content_sets_epel7.repo /etc/yum.repos.d/
+COPY ./build/dockerfiles/content_set*.repo /etc/yum.repos.d/
 
 COPY ./build/dockerfiles/rhel.install.sh /tmp
 RUN /tmp/rhel.install.sh && rm -f /tmp/rhel.install.sh
@@ -58,6 +58,7 @@ RUN TAG=${PATCHED_IMAGES_TAG} \
     REGISTRY=${PATCHED_IMAGES_REG} \
     ./update_devfile_patched_image_tags.sh
 RUN ./check_mandatory_fields.sh devfiles
+RUN if [[ ${USE_DIGESTS} == "true" ]]; then ./write_image_digests.sh devfiles; fi
 RUN ./index.sh > /build/devfiles/index.json
 RUN ./list_referenced_images.sh devfiles > /build/devfiles/external_images.txt
 RUN chmod -R g+rwX /build/devfiles
@@ -69,7 +70,7 @@ RUN chmod -R g+rwX /build/devfiles
 # Build registry, copying meta.yamls and index.json from builder
 # UPSTREAM: use RHEL7/RHSCL/httpd image so we're not required to authenticate with registry.redhat.io
 # https://access.redhat.com/containers/?tab=tags#/registry.access.redhat.com/rhscl/httpd-24-rhel7
-FROM registry.access.redhat.com/rhscl/httpd-24-rhel7:2.4-108.1575996463 AS registry
+FROM registry.access.redhat.com/rhscl/httpd-24-rhel7:2.4-109 AS registry
 
 # DOWNSTREAM: use RHEL8/httpd
 # https://access.redhat.com/containers/?tab=tags#/registry.access.redhat.com/rhel8/httpd-24

build/dockerfiles/rhel.cache_projects.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright (c) 2018-2020 Red Hat, Inc.
+# This program and the accompanying materials are made
+# available under the terms of the Eclipse Public License 2.0
+# which is available at https://www.eclipse.org/legal/epl-2.0/
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+# Contributors:
+#   Red Hat, Inc. - initial API and implementation
+#
+
+set -x
+
+if [[ ! -f /tmp/resources.tgz ]] || [[ ${BOOTSTRAP} == "true" ]]; then
+  ./cache_projects.sh devfiles resources
+else
+  # unpack into specified folder
+  tar -xvf /tmp/resources.tgz -C "$1/"
+fi

build/dockerfiles/rhel.entrypoint.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

build/dockerfiles/rhel.install.sh

@@ -1,17 +1,20 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/
 #
 # SPDX-License-Identifier: EPL-2.0
 #
 
-microdnf install -y findutils bash wget yum gzip git tar jq python3-six python3-pip && microdnf -y clean all && \
+microdnf install -y findutils bash wget yum gzip git tar jq python3-six python3-pip skopeo && microdnf -y clean all && \
 # install yq (depends on jq and pyyaml - if jq and pyyaml not already installed, this will try to compile it)
 if [[ -f /tmp/root-local.tgz ]] || [[ ${BOOTSTRAP} == "true" ]]; then \
-    mkdir -p /root/.local; tar xf /tmp/root-local.tgz -C /root/.local/; rm -fr /tmp/root-local.tgz;  \
+    mkdir -p /root/.local; \
+	if [[ -f /tmp/root-local.tgz ]]; then \
+		tar xf /tmp/root-local.tgz -C /root/.local/; rm -fr /tmp/root-local.tgz; \
+	fi; \
     /usr/bin/pip3.6 install --user yq jsonschema; \
     # could be installed in /opt/app-root/src/.local/bin or /root/.local/bin
     for d in /opt/app-root/src/.local /root/.local; do \

build/scripts/cache_images.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

build/scripts/cache_projects.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/
@@ -24,14 +24,25 @@ TEMP_REPO="${TEMP_DIR}/cloned"
 # Clone a git repository and create an archive zip at a specified location
 # Args:
 #   $1 - URL of git repo
-#   $2 - (optional) branch to archive
+#   $2 - branch to archive
 #   $3 - destination path for the archived project zip file
+#   $4 - sparse checkout directory
 function cache_project() {
   local repo="$1"
   local branch="$2"
   local destination="$3"
-  git clone "$repo" -b "$branch" --depth 1 "$TEMP_REPO" &>/dev/null
-  git archive "$branch" --remote="$TEMP_REPO" --format zip --output "$destination"
+  local sparse_checkout_dir="$4"
+
+  rm -fr "$TEMP_REPO"
+  git clone "$repo" -b "$branch" --depth 1 "$TEMP_REPO" -q
+  pushd "$TEMP_REPO" &>/dev/null
+    if [ -n "$sparse_checkout_dir" ]; then
+      echo "    Using sparse checkout dir '$sparse_checkout_dir'"
+      git archive -9 "$branch" "$sparse_checkout_dir" -o "$destination"
+    else 
+      git archive -9 "$branch" -o "$destination"
+    fi
+  popd &>/dev/null
   rm -rf "$TEMP_REPO"
 }
 
@@ -93,7 +104,6 @@ for devfile in "${devfiles[@]}"; do
   devfile_name=${devfile_name%-}
   for project in $(yq -c '.projects[]?' "$devfile"); do
     project_name=$(echo "$project" | jq -r '.name')
-    echo "    Caching project $project_name"
 
     type=$(echo "$project" | jq -r '.source.type')
     if [ "$type" != "git" ]; then
@@ -103,13 +113,20 @@ for devfile in "${devfiles[@]}"; do
 
     location=$(echo "$project" | jq -r '.source.location')
     branch=$(echo "$project" | jq -r '.source.branch')
-    if [ -n "$branch" ]; then
+    if [[ ! "$branch" ]] || [[ "$branch" == "null" ]]; then
       branch="master"
     fi
+    sparse_checkout_dir=$(echo "$project" | jq -r '.source.sparseCheckoutDir')
+    if [[ ! "$sparse_checkout_dir" ]] || [[ "$sparse_checkout_dir" == "null" ]]; then
+      unset sparse_checkout_dir
+    fi
+
+    # echo "    Caching project $project_name from branch $branch"
     destination="${RESOURCES_DIR}/${devfile_name}-${project_name}-${branch}.zip"
     absolute_destination=$(realpath "$destination")
-    echo "    Caching project to $absolute_destination"
-    cache_project "$location" "$branch" "$absolute_destination"
+    # echo "    Caching project to $absolute_destination"
+    echo "    Caching project from $location/blob/${branch} to $destination"
+    cache_project "$location" "$branch" "$absolute_destination" "$sparse_checkout_dir"
 
     echo "    Updating devfile $devfile to point at cached project zip $destination"
     update_devfile "$devfile" "$project_name" "$destination"

build/scripts/index.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/