From fbe3c1fd19f801d9594fda65ef03bf098d041dc4 Mon Sep 17 00:00:00 2001 From: Nick Boldt Date: Thu, 26 Mar 2020 04:43:12 -0400 Subject: [PATCH] CRW-563 improve generation of digests from tags in registries and operator (#206) * implement overwriting csv with generated digests and related images; add tags as comments into sources too; support a quieter console mode; if podman not installed fall back to docker Change-Id: I47da5adc3db79e64bd595179223a5fee9635343f Signed-off-by: nickboldt * do not use dockerContainerExtract.sh -- not required; support a quieter console mode; if podman not installed fall back to docker; allow secondary level of skopeo inspection (for Brew images not yet in RHCC or any other additional parsing rules) Change-Id: I33fb68842b41f361d638ba20696eabebf9b8efde Signed-off-by: nickboldt * if podman not installed fall back to docker; slighly noisier console ouptu while pulling images (long operation) Change-Id: I7ea9db2baab32ed03cef71aa7aac6c80a95d6850 Signed-off-by: nickboldt * in case we already have digests in the csv for the registries, replace them with :VERSION tag (so we can rebuild Brew stuff for Quay) Change-Id: I973f0433fedbbda49235da20229ea982c91e53f4 Signed-off-by: nickboldt --- olm/addDigests.sh | 28 +++++++++++++++++--- olm/buildDigestMap.sh | 50 +++++++++++++++++++++++------------ olm/dockerContainerExtract.sh | 11 ++++++-- 3 files changed, 67 insertions(+), 22 deletions(-) diff --git a/olm/addDigests.sh b/olm/addDigests.sh index c6aa27026..9ecee7ed6 100755 --- a/olm/addDigests.sh +++ b/olm/addDigests.sh @@ -12,8 +12,18 @@ SCRIPTS_DIR=$(cd "$(dirname "$0")"; pwd) BASE_DIR="$(pwd)" -command -v podman >/dev/null 2>&1 || { echo "podman is not installed. Aborting."; exit 1; } +QUIET="" + +PODMAN=$(command -v podman) +if [[ ! -x $PODMAN ]]; then + echo "[WARNING] podman is not installed." + PODMAN=$(command -v docker) + if [[ ! -x $PODMAN ]]; then + echo "[ERROR] docker is not installed. Aborting."; exit 1 + fi +fi command -v yq >/dev/null 2>&1 || { echo "yq is not installed. Aborting."; exit 1; } + usage () { echo "Usage: $0 [-w WORKDIR] -s [SOURCE_PATH] -n [csv name] -v [VERSION] " echo "Example: $0 -w $(pwd) -s eclipse-che-preview-openshift/deploy/olm-catalog/eclipse-che-preview-openshift -n eclipse-che-preview-openshift -v 7.9.0" @@ -28,6 +38,7 @@ while [[ "$#" -gt 0 ]]; do '-s') SRC_DIR="$2"; shift 1;; '-n') CSV_NAME="$2"; shift 1;; '-v') VERSION="$2"; shift 1;; + '-q') QUIET="-q"; shift 0;; '--help'|'-h') usage; exit;; esac shift 1 @@ -40,8 +51,9 @@ mkdir -p ${BASE_DIR}/generated/${CSV_NAME}/ cp -R ${BASE_DIR}/${SRC_DIR}/* ${BASE_DIR}/generated/${CSV_NAME}/ CSV_FILE="$(find ${BASE_DIR}/generated/${CSV_NAME}/*${VERSION}/ -name "${CSV_NAME}.*${VERSION}.clusterserviceversion.yaml" | tail -1)"; # echo "[INFO] CSV = ${CSV_FILE}" -${SCRIPTS_DIR}/buildDigestMap.sh -w ${BASE_DIR} -c ${CSV_FILE} +${SCRIPTS_DIR}/buildDigestMap.sh -w ${BASE_DIR} -c ${CSV_FILE} -v ${VERSION} ${QUIET} +# inject relatedImages block names=" " count=1 RELATED_IMAGES='. * { spec : { relatedImages: [ ' @@ -59,9 +71,19 @@ do if [ "${names}" != " " ]; then RELATED_IMAGES="${RELATED_IMAGES}," fi - RELATED_IMAGES="${RELATED_IMAGES} { name: \"${name}\", image: \"${dest}\"}" + RELATED_IMAGES="${RELATED_IMAGES} { name: \"${name}\", image: \"${dest}\", tag: \"${source}\"}" names="${names} ${name} " done RELATED_IMAGES="${RELATED_IMAGES} ] } }" mv ${CSV_FILE} ${CSV_FILE}.old yq -Y "$RELATED_IMAGES" ${CSV_FILE}.old > ${CSV_FILE} +sed -i ${CSV_FILE} -r -e "s|tag: |# tag: |" +rm -f ${CSV_FILE}.old + +# update original file with generated changes +CSV_FILE_ORIG=$(find ${BASE_DIR} -name "${CSV_FILE##*/}" | grep -v generated | tail -1) +mv "${CSV_FILE}" "${CSV_FILE_ORIG}" +echo "[INFO] CSV updated: ${CSV_FILE_ORIG}" + +# cleanup +rm -fr ${BASE_DIR}/generated \ No newline at end of file diff --git a/olm/buildDigestMap.sh b/olm/buildDigestMap.sh index 4a43de1a1..58f3fcf50 100755 --- a/olm/buildDigestMap.sh +++ b/olm/buildDigestMap.sh @@ -12,6 +12,17 @@ SCRIPTS_DIR=$(cd "$(dirname "$0")"; pwd) BASE_DIR="$1" +QUIET="" + +PODMAN=$(command -v podman) +if [[ ! -x $PODMAN ]]; then + echo "[WARNING] podman is not installed." + PODMAN=$(command -v docker) + if [[ ! -x $PODMAN ]]; then + echo "[ERROR] docker is not installed. Aborting."; exit 1 + fi +fi +command -v yq >/dev/null 2>&1 || { echo "yq is not installed. Aborting."; exit 1; } usage () { echo "Usage: $0 [-w WORKDIR] -c [/path/to/csv.yaml] " @@ -24,12 +35,14 @@ while [[ "$#" -gt 0 ]]; do case $1 in '-w') BASE_DIR="$2"; shift 1;; '-c') CSV="$2"; shift 1;; - '--help'|'-h') usage; exit;; + '-v') VERSION="$2"; shift 1;; + '-q') QUIET="-q"; shift 0;; + '--help'|'-h') usage; exit;; esac shift 1 done -if [[ ! $CSV ]]; then usage; exit 1; fi +if [[ ! $CSV ]] || [[ ! $VERSION ]]; then usage; exit 1; fi mkdir -p ${BASE_DIR}/generated @@ -39,33 +52,36 @@ IMAGE_LIST=$(yq -r '.spec.install.spec.deployments[].spec.template.spec.containe OPERATOR_IMAGE=$(yq -r '.spec.install.spec.deployments[].spec.template.spec.containers[].image' "${CSV}") REGISTRY_LIST=$(yq -r '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("IMAGE_default_.*_registry"; "g")) | .value' "${CSV}") -REGISTRY_IMAGES="" +REGISTRY_IMAGES_ALL="" for registry in ${REGISTRY_LIST}; do - extracted=$(${SCRIPTS_DIR}/dockerContainerExtract.sh ${registry} var/www/html/*/external_images.txt | tail -n 1) + registry="${registry/\@sha256:*/:${VERSION}}" # remove possible existing @sha256:... and use current version instead + # echo -n "[INFO] Pull container ${registry} ..." + ${PODMAN} pull ${registry} ${QUIET} - # Container quay.io/eclipse/che-devfile-registry:7.9.0 unpacked to /tmp/quay.io-eclipse-che-devfile-registry-7.9.0-1584588272 - extracted=${extracted##* } # the last token in the above line is the path we want - - echo -n "[INFO] Extract images from registry ${registry} ... " - if [[ -d ${extracted} ]]; then - # cat ${extracted}/var/www/html/*/external_images.txt - REGISTRY_IMAGES="${REGISTRY_IMAGES} $(cat ${extracted}/var/www/html/*/external_images.txt)" - fi - echo "found $(cat ${extracted}/var/www/html/*/external_images.txt | wc -l)" - rm -fr ${extracted} 2>&1 >/dev/null + REGISTRY_IMAGES="$(${PODMAN} run --rm --entrypoint /bin/sh ${registry} -c "cat /var/www/html/*/external_images.txt")" + echo "[INFO] Found $(echo "${REGISTRY_IMAGES}" | wc -l) images in registry" + REGISTRY_IMAGES_ALL="${REGISTRY_IMAGES_ALL} ${REGISTRY_IMAGES}" done rm -Rf ${BASE_DIR}/generated/digests-mapping.txt touch ${BASE_DIR}/generated/digests-mapping.txt -for image in ${OPERATOR_IMAGE} ${IMAGE_LIST} ${REGISTRY_IMAGES}; do +for image in ${OPERATOR_IMAGE} ${IMAGE_LIST} ${REGISTRY_IMAGES_ALL}; do case ${image} in *@sha256:*) withDigest="${image}";; *@) continue;; *) - echo "[INFO] Get digest from ${image}" - digest="$(skopeo inspect docker://${image} | jq -r '.Digest')" + digest="$(skopeo inspect docker://${image} 2>/dev/null | jq -r '.Digest')" + if [[ ${digest} ]]; then + if [[ ! "${QUIET}" ]]; then echo -n "[INFO] Got digest"; fi + echo " $digest # ${image}" + else + # for other build methods or for falling back to other registries when not found, can apply transforms here + if [[ -x ${SCRIPTS_DIR}/buildDigestMapAlternateURLs.sh ]]; then + . ${SCRIPTS_DIR}/buildDigestMapAlternateURLs.sh + fi + fi withoutTag="$(echo "${image}" | sed -e 's/^\(.*\):[^:]*$/\1/')" withDigest="${withoutTag}@${digest}";; esac diff --git a/olm/dockerContainerExtract.sh b/olm/dockerContainerExtract.sh index e978d775b..974783011 100755 --- a/olm/dockerContainerExtract.sh +++ b/olm/dockerContainerExtract.sh @@ -9,7 +9,14 @@ if [[ ! $1 ]]; then exit fi -PODMAN=docker # or user podman +PODMAN=$(command -v podman) +if [[ ! -x $PODMAN ]]; then + echo "[WARNING] podman is not installed." + PODMAN=$(command -v docker) + if [[ ! -x $PODMAN ]]; then + echo "[ERROR] docker is not installed. Aborting."; exit 1 + fi +fi container="$1"; shift 1 tmpcontainer="$(echo $container | tr "/:" "--")-$(date +%s)" @@ -17,7 +24,7 @@ unpackdir="/tmp/${tmpcontainer}" # get remote image echo "[INFO] Pulling $container ..." -${PODMAN} pull $container 2>&1 >/dev/null +${PODMAN} pull $container 2>&1 # create local container ${PODMAN} rm -f "${tmpcontainer}" 2>&1 >/dev/null || true