CRW-563 improve generation of digests from tags in registries and operator (#206)

* implement overwriting csv with generated digests and related images; add tags as comments into sources too; support a quieter console mode; if podman not installed fall back to docker

Change-Id: I47da5adc3db79e64bd595179223a5fee9635343f
Signed-off-by: nickboldt <nboldt@redhat.com>

* do not use dockerContainerExtract.sh -- not required; support a quieter console mode; if podman not installed fall back to docker; allow secondary level of skopeo inspection (for Brew images not yet in RHCC or any other additional parsing rules)

Change-Id: I33fb68842b41f361d638ba20696eabebf9b8efde
Signed-off-by: nickboldt <nboldt@redhat.com>

* if podman not installed fall back to docker; slighly noisier console ouptu while pulling images (long operation)

Change-Id: I7ea9db2baab32ed03cef71aa7aac6c80a95d6850
Signed-off-by: nickboldt <nboldt@redhat.com>

* in case we already have digests in the csv for the registries, replace them with :VERSION tag (so we can rebuild Brew stuff for Quay)

Change-Id: I973f0433fedbbda49235da20229ea982c91e53f4
Signed-off-by: nickboldt <nboldt@redhat.com>

olm/addDigests.sh

@@ -12,8 +12,18 @@
 
 SCRIPTS_DIR=$(cd "$(dirname "$0")"; pwd)
 BASE_DIR="$(pwd)"
-command -v podman >/dev/null 2>&1 || { echo "podman is not installed. Aborting."; exit 1; }
+QUIET=""
+
+PODMAN=$(command -v podman)
+if [[ ! -x $PODMAN ]]; then
+  echo "[WARNING] podman is not installed."
+  PODMAN=$(command -v docker)
+  if [[ ! -x $PODMAN ]]; then
+    echo "[ERROR] docker is not installed. Aborting."; exit 1
+  fi
+fi
 command -v yq >/dev/null 2>&1 || { echo "yq is not installed. Aborting."; exit 1; }
+
 usage () {
 	echo "Usage:   $0 [-w WORKDIR] -s [SOURCE_PATH] -n [csv name] -v [VERSION] "
 	echo "Example: $0 -w $(pwd) -s eclipse-che-preview-openshift/deploy/olm-catalog/eclipse-che-preview-openshift -n eclipse-che-preview-openshift -v 7.9.0"
@@ -28,6 +38,7 @@ while [[ "$#" -gt 0 ]]; do
     '-s') SRC_DIR="$2"; shift 1;;
     '-n') CSV_NAME="$2"; shift 1;;
     '-v') VERSION="$2"; shift 1;;
+    '-q') QUIET="-q"; shift 0;;
 	'--help'|'-h') usage; exit;;
   esac
   shift 1
@@ -40,8 +51,9 @@ mkdir -p ${BASE_DIR}/generated/${CSV_NAME}/
 cp -R ${BASE_DIR}/${SRC_DIR}/* ${BASE_DIR}/generated/${CSV_NAME}/
 
 CSV_FILE="$(find ${BASE_DIR}/generated/${CSV_NAME}/*${VERSION}/ -name "${CSV_NAME}.*${VERSION}.clusterserviceversion.yaml" | tail -1)"; # echo "[INFO] CSV = ${CSV_FILE}"
-${SCRIPTS_DIR}/buildDigestMap.sh -w ${BASE_DIR} -c ${CSV_FILE}
+${SCRIPTS_DIR}/buildDigestMap.sh -w ${BASE_DIR} -c ${CSV_FILE} -v ${VERSION} ${QUIET}
 
+# inject relatedImages block
 names=" "
 count=1
 RELATED_IMAGES='. * { spec : { relatedImages: [ '
@@ -59,9 +71,19 @@ do
   if [ "${names}" != " " ]; then
     RELATED_IMAGES="${RELATED_IMAGES},"
   fi
-  RELATED_IMAGES="${RELATED_IMAGES} { name: \"${name}\", image: \"${dest}\"}"
+  RELATED_IMAGES="${RELATED_IMAGES} { name: \"${name}\", image: \"${dest}\", tag: \"${source}\"}"
   names="${names} ${name} "
 done
 RELATED_IMAGES="${RELATED_IMAGES} ] } }"
 mv ${CSV_FILE} ${CSV_FILE}.old
 yq -Y "$RELATED_IMAGES" ${CSV_FILE}.old > ${CSV_FILE}
+sed -i ${CSV_FILE} -r -e "s|tag: |# tag: |" 
+rm -f ${CSV_FILE}.old
+
+# update original file with generated changes
+CSV_FILE_ORIG=$(find ${BASE_DIR} -name "${CSV_FILE##*/}" | grep -v generated | tail -1)
+mv "${CSV_FILE}" "${CSV_FILE_ORIG}"
+echo "[INFO] CSV updated: ${CSV_FILE_ORIG}"
+
+# cleanup
+rm -fr ${BASE_DIR}/generated

olm/buildDigestMap.sh

@@ -12,6 +12,17 @@
 
 SCRIPTS_DIR=$(cd "$(dirname "$0")"; pwd)
 BASE_DIR="$1"
+QUIET=""
+
+PODMAN=$(command -v podman)
+if [[ ! -x $PODMAN ]]; then
+  echo "[WARNING] podman is not installed."
+  PODMAN=$(command -v docker)
+  if [[ ! -x $PODMAN ]]; then
+    echo "[ERROR] docker is not installed. Aborting."; exit 1
+  fi
+fi
+command -v yq >/dev/null 2>&1 || { echo "yq is not installed. Aborting."; exit 1; }
 
 usage () {
 	echo "Usage:   $0 [-w WORKDIR] -c [/path/to/csv.yaml] "
@@ -24,12 +35,14 @@ while [[ "$#" -gt 0 ]]; do
   case $1 in
     '-w') BASE_DIR="$2"; shift 1;;
     '-c') CSV="$2"; shift 1;;
-	'--help'|'-h') usage; exit;;
+    '-v') VERSION="$2"; shift 1;;
+    '-q') QUIET="-q"; shift 0;;
+    '--help'|'-h') usage; exit;;
   esac
   shift 1
 done
 
-if [[ ! $CSV ]]; then usage; exit 1; fi
+if [[ ! $CSV ]] || [[ ! $VERSION ]]; then usage; exit 1; fi
 
 mkdir -p ${BASE_DIR}/generated
 
@@ -39,33 +52,36 @@ IMAGE_LIST=$(yq -r '.spec.install.spec.deployments[].spec.template.spec.containe
 OPERATOR_IMAGE=$(yq -r '.spec.install.spec.deployments[].spec.template.spec.containers[].image' "${CSV}")
 
 REGISTRY_LIST=$(yq -r '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("IMAGE_default_.*_registry"; "g")) | .value' "${CSV}")
-REGISTRY_IMAGES=""
+REGISTRY_IMAGES_ALL=""
 for registry in ${REGISTRY_LIST}; do
-  extracted=$(${SCRIPTS_DIR}/dockerContainerExtract.sh ${registry} var/www/html/*/external_images.txt | tail -n 1)
+  registry="${registry/\@sha256:*/:${VERSION}}" # remove possible existing @sha256:... and use current version instead
+  # echo -n "[INFO] Pull container ${registry} ..."
+  ${PODMAN} pull ${registry} ${QUIET}
 
-  # Container quay.io/eclipse/che-devfile-registry:7.9.0 unpacked to /tmp/quay.io-eclipse-che-devfile-registry-7.9.0-1584588272
-  extracted=${extracted##* } # the last token in the above line is the path we want
-
-  echo -n "[INFO] Extract images from registry ${registry} ... "
-  if [[ -d ${extracted} ]]; then
-    # cat ${extracted}/var/www/html/*/external_images.txt
-    REGISTRY_IMAGES="${REGISTRY_IMAGES} $(cat ${extracted}/var/www/html/*/external_images.txt)"
-  fi
-  echo "found $(cat ${extracted}/var/www/html/*/external_images.txt | wc -l)"
-  rm -fr ${extracted} 2>&1 >/dev/null
+  REGISTRY_IMAGES="$(${PODMAN} run --rm  --entrypoint /bin/sh  ${registry} -c "cat /var/www/html/*/external_images.txt")"
+  echo "[INFO] Found $(echo "${REGISTRY_IMAGES}" | wc -l) images in registry"
+  REGISTRY_IMAGES_ALL="${REGISTRY_IMAGES_ALL} ${REGISTRY_IMAGES}"
 done
 
 rm -Rf ${BASE_DIR}/generated/digests-mapping.txt
 touch ${BASE_DIR}/generated/digests-mapping.txt
-for image in ${OPERATOR_IMAGE} ${IMAGE_LIST} ${REGISTRY_IMAGES}; do
+for image in ${OPERATOR_IMAGE} ${IMAGE_LIST} ${REGISTRY_IMAGES_ALL}; do
   case ${image} in
     *@sha256:*)
       withDigest="${image}";;
     *@)
       continue;;
     *)
-      echo "[INFO] Get digest from ${image}"
-      digest="$(skopeo inspect docker://${image} | jq -r '.Digest')"
+      digest="$(skopeo inspect docker://${image} 2>/dev/null | jq -r '.Digest')"
+      if [[ ${digest} ]]; then
+        if [[ ! "${QUIET}" ]]; then echo -n "[INFO] Got digest"; fi
+        echo "    $digest # ${image}"
+      else
+        # for other build methods or for falling back to other registries when not found, can apply transforms here
+        if [[ -x ${SCRIPTS_DIR}/buildDigestMapAlternateURLs.sh ]]; then
+          . ${SCRIPTS_DIR}/buildDigestMapAlternateURLs.sh
+        fi
+      fi
       withoutTag="$(echo "${image}" | sed -e 's/^\(.*\):[^:]*$/\1/')"
       withDigest="${withoutTag}@${digest}";;
   esac

olm/dockerContainerExtract.sh

@@ -9,7 +9,14 @@ if [[ ! $1 ]]; then
   exit
 fi
 
-PODMAN=docker # or user podman
+PODMAN=$(command -v podman)
+if [[ ! -x $PODMAN ]]; then
+  echo "[WARNING] podman is not installed."
+  PODMAN=$(command -v docker)
+  if [[ ! -x $PODMAN ]]; then
+    echo "[ERROR] docker is not installed. Aborting."; exit 1
+  fi
+fi
 
 container="$1"; shift 1
 tmpcontainer="$(echo $container | tr "/:" "--")-$(date +%s)"
@@ -17,7 +24,7 @@ unpackdir="/tmp/${tmpcontainer}"
 
 # get remote image
 echo "[INFO] Pulling $container ..."
-${PODMAN} pull $container 2>&1 >/dev/null
+${PODMAN} pull $container 2>&1
 
 # create local container
 ${PODMAN} rm -f "${tmpcontainer}" 2>&1 >/dev/null || true