From 467bcbd0251b7ef75320594b8989cdaa89ecd3cc Mon Sep 17 00:00:00 2001 From: nickboldt Date: Thu, 16 Jul 2020 13:50:14 -0400 Subject: [PATCH] update olm scripts from master branch (needed for CRW 2.2) Change-Id: I24763ea1c36145fb45ca39fde80af1e179b700ef Signed-off-by: nickboldt --- olm/addDigests.sh | 127 +++++++++++++++++++++++++++++------------- olm/buildDigestMap.sh | 112 ++++++++++++++++++++++++++----------- olm/digestExcludeList | 3 + olm/images.sh | 44 +++++++++++++++ olm/olm.sh | 1 - 5 files changed, 212 insertions(+), 75 deletions(-) create mode 100644 olm/digestExcludeList create mode 100644 olm/images.sh diff --git a/olm/addDigests.sh b/olm/addDigests.sh index 2c7a336d5..d4f450b16 100755 --- a/olm/addDigests.sh +++ b/olm/addDigests.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2019 Red Hat, Inc. +# Copyright (c) 2019-2020 Red Hat, Inc. # This program and the accompanying materials are made # available under the terms of the Eclipse Public License 2.0 # which is available at https://www.eclipse.org/legal/epl-2.0/ @@ -10,6 +10,9 @@ # Contributors: # Red Hat, Inc. - initial API and implementation +set +x +set -e + SCRIPTS_DIR=$(cd "$(dirname "$0")"; pwd) BASE_DIR="$(pwd)" QUIET="" @@ -25,9 +28,8 @@ fi command -v yq >/dev/null 2>&1 || { echo "yq is not installed. Aborting."; exit 1; } usage () { - echo "Usage: $0 [-w WORKDIR] -s [SOURCE_PATH] -n [csv name] -v [VERSION] " - echo "Example: $0 -w $(pwd) -s eclipse-che-preview-openshift/deploy/olm-catalog/eclipse-che-preview-openshift -n eclipse-che-preview-openshift -v 7.9.0" - echo "Example: $0 -w $(pwd) -s controller-manifests -n codeready-workspaces -v 2.1.0" + echo "Usage: $0 [-w WORKDIR] [-s SOURCE_PATH] -r [CSV_FILE_PATH_REGEXP] -t [IMAGE_TAG] " + echo "Example: $0 -w $(pwd) -r \"eclipse-che-preview-.*/eclipse-che-preview-.*\.v7.15.0.*yaml\" -t 7.15.0" } if [[ $# -lt 1 ]]; then usage; exit; fi @@ -36,53 +38,98 @@ while [[ "$#" -gt 0 ]]; do case $1 in '-w') BASE_DIR="$2"; shift 1;; '-s') SRC_DIR="$2"; shift 1;; - '-n') CSV_NAME="$2"; shift 1;; - '-v') VERSION="$2"; shift 1;; + '-t') IMAGE_TAG="$2"; shift 1;; + '-r') CSV_FILE_PATH_REGEXP="$2"; shift 1;; '-q') QUIET="-q"; shift 0;; '--help'|'-h') usage; exit;; esac shift 1 done -if [[ ! $SRC_DIR ]] || [[ ! $CSV_NAME ]] || [[ ! $VERSION ]]; then usage; exit 1; fi +if [[ ! ${CSV_FILE_PATH_REGEXP} ]] || [[ ! $IMAGE_TAG ]]; then usage; exit 1; fi -rm -Rf ${BASE_DIR}/generated/${CSV_NAME}/ -mkdir -p ${BASE_DIR}/generated/${CSV_NAME}/ -cp -R ${BASE_DIR}/${SRC_DIR}/* ${BASE_DIR}/generated/${CSV_NAME}/ +CSV_FILES_DIR=${BASE_DIR} +if [ -n "${SRC_DIR}" ]; then + CSV_FILES_DIR="${BASE_DIR}/${SRC_DIR}" +fi +echo "Resolved CSV files dir: ${CSV_FILES_DIR}" -CSV_FILE="$(find ${BASE_DIR}/generated/${CSV_NAME}/*${VERSION}/ -name "${CSV_NAME}.*${VERSION}.clusterserviceversion.yaml" | tail -1)"; # echo "[INFO] CSV = ${CSV_FILE}" -${SCRIPTS_DIR}/buildDigestMap.sh -w ${BASE_DIR} -c ${CSV_FILE} -v ${VERSION} ${QUIET} +echo "find ${CSV_FILES_DIR} -regextype posix-egrep -regex \"${CSV_FILES_DIR}/?${CSV_FILE_PATH_REGEXP}\"" +CSV_FILES=( $(find ${CSV_FILES_DIR} -regextype posix-egrep -regex "${CSV_FILES_DIR}/?${CSV_FILE_PATH_REGEXP}") ) +RELATED_IMAGE_PREFIX="RELATED_IMAGE_" -# inject relatedImages block -names=" " -count=1 -RELATED_IMAGES='. * { spec : { relatedImages: [ ' -if [[ ! "${QUIET}" ]]; then cat ${BASE_DIR}/generated/digests-mapping.txt; fi -for mapping in $(cat ${BASE_DIR}/generated/digests-mapping.txt) +rm -Rf "${BASE_DIR}/generated/csv" +mkdir -p "${BASE_DIR}/generated/csv" +# Copy original csv files +for CSV_FILE in "${CSV_FILES[@]}" do - source=$(echo "${mapping}" | sed -e 's/\(.*\)=.*/\1/') - dest=$(echo "${mapping}" | sed -e 's/.*=\(.*\)/\1/') - sed -i -e "s;${source};${dest};" ${CSV_FILE} - name=$(echo "${dest}" | sed -e 's;.*/\([^\/][^\/]*\)@.*;\1;') - nameWithSpaces=" ${name} " - if [[ "${names}" != *${nameWithSpaces}* ]]; then - if [ "${names}" != " " ]; then - RELATED_IMAGES="${RELATED_IMAGES}," - fi - RELATED_IMAGES="${RELATED_IMAGES} { name: \"${name}\", image: \"${dest}\", tag: \"${source}\"}" - names="${names} ${name} " - fi + echo "CSV file: ${CSV_FILE}" + cp -pR "${CSV_FILE}" "${BASE_DIR}/generated/csv" + csvs_args="${csvs_args} -c ${CSV_FILE}" done -RELATED_IMAGES="${RELATED_IMAGES} ] } }" -mv ${CSV_FILE} ${CSV_FILE}.old -yq -Y "$RELATED_IMAGES" ${CSV_FILE}.old > ${CSV_FILE} -sed -i ${CSV_FILE} -r -e "s|tag: |# tag: |" -rm -f ${CSV_FILE}.old -# update original file with generated changes -CSV_FILE_ORIG=$(find ${BASE_DIR} -name "${CSV_FILE##*/}" | grep -v generated | tail -1) -mv "${CSV_FILE}" "${CSV_FILE_ORIG}" -echo "[INFO] CSV updated: ${CSV_FILE_ORIG}" +# shellcheck source=buildDigestMap.sh +eval "${SCRIPTS_DIR}/buildDigestMap.sh" -w "${BASE_DIR}" -t "${IMAGE_TAG}" "${csvs_args}" ${QUIET} + +if [[ ! "${QUIET}" ]]; then cat "${BASE_DIR}"/generated/digests-mapping.txt; fi +for CSV_FILE in "${CSV_FILES[@]}" +do + CSV_FILE_COPY=${BASE_DIR}/generated/csv/$(basename ${CSV_FILE}) + + echo "[INFO] Generate digest update for CSV file ${CSV_FILE}" + RELATED_IMAGES="" + RELATED_IMAGES_ENV="" + for mapping in $(cat "${BASE_DIR}/generated/digests-mapping.txt") + do + source=$(echo "${mapping}" | sed -e 's;\(.*\)=.*=.*;\1;') + # Image with digest. + dest=$(echo "${mapping}" | sed -e 's;.*=.*=\(.*\);\1;') + # Image label to set image target. For example: 'devfile-registry-image' + imageLabel=$(echo "${mapping}" | sed -e 's;.*=\(.*\)=.*;\1;') + name=$(echo "${dest}" | sed -e 's;.*/\([^\/][^\/]*\)@.*;\1;') + tagOrDigest="" + if [[ ${source} == *"@"* ]]; then + tagOrDigest="@${source#*@}" + elif [[ ${source} == *":"* ]]; then + tagOrDigest="${source#*:}" + fi + + if [[ ${imageLabel} == "plugin-registry-image" ]] || [[ ${imageLabel} == "devfile-registry-image" ]]; then + # Image tag could contains invalid for Env variable name characters, so let's encode it using base32. + # But alphabet of base32 uses one invalid for env variable name character '=' at the end of the line, so let's replace it by '_'. + # To recovery original tag should be done opposite actions: replace '_' to '=', and decode string using 'base32 -d'. + encodedTag=$(echo "${tagOrDigest}" | base32 -w 0 | tr "=" "_") + relatedImageEnvName=$(echo "${RELATED_IMAGE_PREFIX}${name}_${imageLabel}_${encodedTag}" | sed -r 's/[-.]/_/g') + ENV="{ name: \"${relatedImageEnvName}\", value: \"${dest}\"}" + if [[ -z ${RELATED_IMAGES_ENV} ]]; then + RELATED_IMAGES_ENV="${ENV}" + else + RELATED_IMAGES_ENV="${RELATED_IMAGES_ENV}, ${ENV}" + fi + fi + + RELATED_IMAGE="{ name: \"${name}-${tagOrDigest}\", image: \"${dest}\", tag: \"${source}\"}" + if [[ -z ${RELATED_IMAGES} ]]; then + RELATED_IMAGES="${RELATED_IMAGE}" + else + RELATED_IMAGES="${RELATED_IMAGES}, ${RELATED_IMAGE}" + fi + + sed -i -e "s;${source};${dest};" "${CSV_FILE_COPY}" + done + + mv "${CSV_FILE_COPY}" "${CSV_FILE_COPY}.old" + yq -ryY " + ( .spec.relatedImages ) += [${RELATED_IMAGES}] | + ( .spec.install.spec.deployments[0].spec.template.spec.containers[0].env ) += [${RELATED_IMAGES_ENV}] + " "${CSV_FILE_COPY}.old" > "${CSV_FILE_COPY}" + sed -i "${CSV_FILE_COPY}" -r -e "s|tag: |# tag: |" + rm -f "${CSV_FILE_COPY}.old" + + # update original file with generated changes + mv "${CSV_FILE_COPY}" "${CSV_FILE}" + echo "[INFO] CSV updated: ${CSV_FILE}" +done # cleanup -rm -fr ${BASE_DIR}/generated \ No newline at end of file +rm -fr "${BASE_DIR}/generated" diff --git a/olm/buildDigestMap.sh b/olm/buildDigestMap.sh index 66469010c..1ae754bac 100755 --- a/olm/buildDigestMap.sh +++ b/olm/buildDigestMap.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2019 Red Hat, Inc. +# Copyright (c) 2019-2020 Red Hat, Inc. # This program and the accompanying materials are made # available under the terms of the Eclipse Public License 2.0 # which is available at https://www.eclipse.org/legal/epl-2.0/ @@ -10,7 +10,9 @@ # Contributors: # Red Hat, Inc. - initial API and implementation -SCRIPTS_DIR=$(cd "$(dirname "$0")"; pwd) +set -e + +SCRIPTS_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) BASE_DIR="$1" QUIET="" @@ -23,10 +25,11 @@ if [[ ! -x $PODMAN ]]; then fi fi command -v yq >/dev/null 2>&1 || { echo "yq is not installed. Aborting."; exit 1; } +command -v skopeo > /dev/null 2>&1 || { echo "skopeo is not installed. Aborting."; exit 1; } usage () { - echo "Usage: $0 [-w WORKDIR] -c [/path/to/csv.yaml] " - echo "Example: $0 -w $(pwd) -c $(pwd)/generated/eclipse-che-preview-openshift/7.9.0/eclipse-che-preview-openshift.v7.9.0.clusterserviceversion.yaml" + echo "Usage: $0 [-w WORKDIR] -c [/path/to/csv.yaml] -t [IMAGE_TAG]" + echo "Example: $0 -w $(pwd) -c $(pwd)/olm/eclipse-che-preview-kubernetes/deploy/olm-catalog/eclipse-che-preview-kubernetes/7.9.0/eclipse-che-preview-kubernetes.v7.9.0.clusterserviceversion.yaml -t 7.9.0" } if [[ $# -lt 1 ]]; then usage; exit; fi @@ -34,56 +37,77 @@ if [[ $# -lt 1 ]]; then usage; exit; fi while [[ "$#" -gt 0 ]]; do case $1 in '-w') BASE_DIR="$2"; shift 1;; - '-c') CSV="$2"; shift 1;; - '-v') VERSION="$2"; shift 1;; + '-c') CSV="$2"; CSVS+=("${CSV}");shift 1;; + '-t') IMAGE_TAG="$2"; shift 1;; '-q') QUIET="-q"; shift 0;; '--help'|'-h') usage; exit;; esac shift 1 done -if [[ ! $CSV ]] || [[ ! $VERSION ]]; then usage; exit 1; fi +if [[ ! $CSV ]] || [[ ! $IMAGE_TAG ]]; then usage; exit 1; fi -mkdir -p ${BASE_DIR}/generated +mkdir -p "${BASE_DIR}/generated" echo "[INFO] Get images from CSV ${CSV}" -IMAGE_LIST=$(yq -r '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("IMAGE_default_.*"; "g")) | .value' "${CSV}") -OPERATOR_IMAGE=$(yq -r '.spec.install.spec.deployments[].spec.template.spec.containers[].image' "${CSV}") +# shellcheck source=images.sh +. "${SCRIPTS_DIR}"/images.sh -REGISTRY_LIST=$(yq -r '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("IMAGE_default_.*_registry"; "g")) | .value' "${CSV}") -REGISTRY_IMAGES_ALL="" -for registry in ${REGISTRY_LIST}; do - registry="${registry/\@sha256:*/:${VERSION}}" # remove possible existing @sha256:... and use current version instead - # echo -n "[INFO] Pull container ${registry} ..." - ${PODMAN} pull ${registry} ${QUIET} +# todo create init method +setImagesFromDeploymentEnv - REGISTRY_IMAGES="$(${PODMAN} run --rm --entrypoint /bin/sh ${registry} -c "cat /var/www/html/*/external_images.txt")" - echo "[INFO] Found $(echo "${REGISTRY_IMAGES}" | wc -l) images in registry" - REGISTRY_IMAGES_ALL="${REGISTRY_IMAGES_ALL} ${REGISTRY_IMAGES}" -done +setOperatorImage +echo "${OPERATOR_IMAGE}" -rm -Rf ${BASE_DIR}/generated/digests-mapping.txt -touch ${BASE_DIR}/generated/digests-mapping.txt -for image in ${OPERATOR_IMAGE} ${IMAGE_LIST} ${REGISTRY_IMAGES_ALL}; do +setPluginRegistryList +echo "${PLUGIN_REGISTRY_LIST}" + +setDevfileRegistryList +echo "${DEVFILE_REGISTRY_LIST}" + +writeDigest() { + image=$1 + + # Check exclude image list + excludeFile="${SCRIPTS_DIR}/digestExcludeList" + if [ -f "${excludeFile}" ]; then + IFS=$'\n' read -d '' -r -a excludedImages < "${excludeFile}" || true + if [[ " ${excludedImages[*]} " =~ ${image} ]]; then + echo "[INFO] Image '${image}' was excluded" + return + fi + fi + + imageType=$2 + digest="" case ${image} in *@sha256:*) - withDigest="${image}";; + withDigest=${image};; *@) - continue;; + return;; *) - digest="$(skopeo inspect --tls-verify=false docker://${image} 2>/dev/null | jq -r '.Digest')" + # for other build methods or for falling back to other registries when not found, can apply transforms here + orig_image=${image} + if [[ -x ${SCRIPTS_DIR}/buildDigestMapAlternateURLs.sh ]]; then + # shellcheck source=buildDigestMapAlternateURLs.sh + . ${SCRIPTS_DIR}/buildDigestMapAlternateURLs.sh + fi if [[ ${digest} ]]; then if [[ ! "${QUIET}" ]]; then echo -n "[INFO] Got digest"; fi - echo " $digest # ${image}" + echo " $digest \# ${image}" else - # for other build methods or for falling back to other registries when not found, can apply transforms here - if [[ -x ${SCRIPTS_DIR}/buildDigestMapAlternateURLs.sh ]]; then - . ${SCRIPTS_DIR}/buildDigestMapAlternateURLs.sh - fi + image="${orig_image}" + digest="$(skopeo inspect --tls-verify=false docker://${image} 2>/dev/null | jq -r '.Digest')" + fi + if [[ -z ${digest} ]]; then + echo "==================== Failed to get digest for image: ${image}======================" + withoutTag="" + withDigest="" + else + withoutTag="$(echo "${image}" | sed -e 's/^\(.*\):[^:]*$/\1/')" + withDigest="${withoutTag}@${digest}"; fi - withoutTag="$(echo "${image}" | sed -e 's/^\(.*\):[^:]*$/\1/')" - withDigest="${withoutTag}@${digest}";; esac dots="${withDigest//[^\.]}" separators="${withDigest//[^\/]}" @@ -92,5 +116,25 @@ for image in ${OPERATOR_IMAGE} ${IMAGE_LIST} ${REGISTRY_IMAGES_ALL}; do withDigest="docker.io/${withDigest}" fi - echo "${image}=${withDigest}" >> ${BASE_DIR}/generated/digests-mapping.txt + if [[ -n ${withDigest} ]]; then + echo "${image}=${imageType}=${withDigest}" >> "${DIGEST_FILE}" + fi +} + +DIGEST_FILE=${BASE_DIR}/generated/digests-mapping.txt +rm -Rf "${DIGEST_FILE}" +touch "${DIGEST_FILE}" + +writeDigest "${OPERATOR_IMAGE}" "operator-image" + +for image in ${REQUIRED_IMAGES}; do + writeDigest "${image}" "required-image" +done + +for image in ${PLUGIN_REGISTRY_LIST}; do + writeDigest "${image}" "plugin-registry-image" +done + +for image in ${DEVFILE_REGISTRY_LIST}; do + writeDigest "${image}" "devfile-registry-image" done diff --git a/olm/digestExcludeList b/olm/digestExcludeList new file mode 100644 index 000000000..ab351f16e --- /dev/null +++ b/olm/digestExcludeList @@ -0,0 +1,3 @@ +quay.io/eclipse/che-theia:next +quay.io/eclipse/che-theia-dev:next +quay.io/eclipse/che-theia-endpoint-runtime-binary:next \ No newline at end of file diff --git a/olm/images.sh b/olm/images.sh new file mode 100644 index 000000000..3503b5717 --- /dev/null +++ b/olm/images.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright (c) 2019-2020 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +# Contributors: +# Red Hat, Inc. - initial API and implementation + +setImagesFromDeploymentEnv() { + REQUIRED_IMAGES=$(yq -r '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.value) | select(.name | test("RELATED_IMAGE_.*"; "g")) | .value' "${CSVS[@]}" | sort | uniq) +} + +setOperatorImage() { + OPERATOR_IMAGE=$(yq -r '.spec.install.spec.deployments[].spec.template.spec.containers[].image' "${CSV}") +} + +setPluginRegistryList() { + registry=$(yq -r '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("RELATED_IMAGE_.*plugin_registry"; "g")) | .value' "${CSV}") + setRegistryImages "${registry}" + + PLUGIN_REGISTRY_LIST=${registryImages} +} + +setDevfileRegistryList() { + registry=$(yq -r '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("RELATED_IMAGE_.*devfile_registry"; "g")) | .value' "${CSV}") + + setRegistryImages "${registry}" + DEVFILE_REGISTRY_LIST=${registryImages} +} + +setRegistryImages() { + registry="${1}" + registry="${registry/\@sha256:*/:${IMAGE_TAG}}" # remove possible existing @sha256:... and use current tag instead + + echo -n "[INFO] Pull container ${registry} ..." + ${PODMAN} pull ${registry} ${QUIET} + + registryImages="$(${PODMAN} run --rm --entrypoint /bin/sh "${registry}" -c "cat /var/www/html/*/external_images.txt")" + echo "[INFO] Found $(echo "${registryImages}" | wc -l) images in registry" +} diff --git a/olm/olm.sh b/olm/olm.sh index 12de4849e..33169ee40 100755 --- a/olm/olm.sh +++ b/olm/olm.sh @@ -202,7 +202,6 @@ applyCRCheCluster() { CRs=$(yq -r '.metadata.annotations["alm-examples"]' "${packageFolderPath}/${PACKAGE_VERSION}/${packageName}.v${PACKAGE_VERSION}.clusterserviceversion.yaml") CR=$(echo "$CRs" | yq -r ".[0]") - CR=$(echo "$CR" | yq -r ".spec.server.tlsSupport = false") if [ "${platform}" == "kubernetes" ] then CR=$(echo "$CR" | yq -r ".spec.k8s.ingressDomain = \"$(minikube ip).nip.io\"")