Add info messages to git-flow tests (#676)

* Add info messages to tests
* Add 'waitFinishDeploymentCheServer' to avoid 'error: lost connection to pod'

.ci/openshift-ci/common.sh

@@ -36,39 +36,47 @@ export CUSTOM_CONFIG_MAP_NAME=${CUSTOM_CONFIG_MAP_NAME:-"custom-ca-certificates"
 export GIT_SSL_CONFIG_MAP_NAME=${GIT_SSL_CONFIG_MAP_NAME:-"che-self-signed-cert"}
 
 provisionOpenShiftOAuthUser() {
-  echo -e "[INFO] Provisioning Openshift OAuth user"
+  echo "------- [INFO] Start provisioning Openshift OAuth user -------"
   htpasswd -c -B -b users.htpasswd ${OCP_ADMIN_USER_NAME} ${OCP_LOGIN_PASSWORD}
   htpasswd -b users.htpasswd ${OCP_NON_ADMIN_USER_NAME} ${OCP_LOGIN_PASSWORD}
   oc create secret generic htpass-secret --from-file=htpasswd="users.htpasswd" -n openshift-config
   oc apply -f ".ci/openshift-ci/htpasswdProvider.yaml"
   oc adm policy add-cluster-role-to-user cluster-admin ${OCP_ADMIN_USER_NAME}
 
-  echo -e "[INFO] Waiting for htpasswd auth to be working up to 5 minutes"
+  echo "------- [INFO] Waiting for htpasswd auth to be working up to 5 minutes -------"
   CURRENT_TIME=$(date +%s)
   ENDTIME=$((CURRENT_TIME + 300))
   while [ "$(date +%s)" -lt $ENDTIME ]; do
       if oc login -u=${OCP_ADMIN_USER_NAME} -p=${OCP_LOGIN_PASSWORD} --insecure-skip-tls-verify=false; then
-          break
+          echo "======= [INFO] OpenShift OAuth htpasswd is configured. =======
+======= [INFO] Login to OCP cluster with admin user credentials is success.======="
+          return 0
       fi
-      sleep 10
+      sleep 5
   done
+
+  echo "####### [ERROR] Error occurred while waiting OpenShift OAuth htpasswd setup. Try to rerun test. #######"
+  exit 1
 }
 
 configureGitSelfSignedCertificate() {
-  echo "[INFO] Configure self-signed certificate for Git provider"
+  echo "------- [INFO] Configure self-signed certificate for Git provider -------"
   oc adm new-project ${CHE_NAMESPACE}
   oc project ${CHE_NAMESPACE}
 
-  echo -e "[INFO] Create ConfigMap with the required TLS certificate"
+  echo "------- [INFO] Create ConfigMap with the required TLS certificate -------"
   oc create configmap ${CUSTOM_CONFIG_MAP_NAME} --from-file=.ci/openshift-ci/ca.crt
   oc label configmap ${CUSTOM_CONFIG_MAP_NAME} app.kubernetes.io/part-of=che.eclipse.org app.kubernetes.io/component=ca-bundle
 
-  echo "[INFO] Create ConfigMap to support Git repositories with self-signed certificates"
+  echo "------- [INFO] Create ConfigMap to support Git repositories with self-signed certificates -------"
-  oc create configmap ${GIT_SSL_CONFIG_MAP_NAME} --from-file=.ci/openshift-ci/ca.crt --from-literal=githost=$GIT_PROVIDER_URL
+  oc create configmap ${GIT_SSL_CONFIG_MAP_NAME} --from-file=.ci/openshift-ci/ca.crt --from-literal=githost=${GIT_PROVIDER_URL}
   oc label configmap ${GIT_SSL_CONFIG_MAP_NAME} app.kubernetes.io/part-of=che.eclipse.org
+
+  echo "======= [INFO] ConfigMaps are configured ======="
 }
 
 createCustomResourcesFile() {
+  echo "------- [INFO] Create custom resourses file -------"
   cat > custom-resources.yaml <<-END
 apiVersion: org.eclipse.che/v2
 spec:
@@ -76,26 +84,53 @@ spec:
     maxNumberOfRunningWorkspacesPerUser: 10000
 END
 
-  echo "Generated custom resources file"
+  echo "======= [INFO] Generated custom resources file ======="
   cat custom-resources.yaml
 }
 
 patchCustomResourcesFile() {
+  echo "------- [INFO] Edit the custom resources file to add 'gitTrustedCertsConfigMapName' -------"
   yq -y '.spec.devEnvironments.trustedCerts += {"gitTrustedCertsConfigMapName": "'${GIT_SSL_CONFIG_MAP_NAME}'"}' custom-resources.yaml -i
 
+  echo "======= [INFO] Patched custom resources file ======="
   cat custom-resources.yaml
 }
 
 deployChe() {
+  echo "------- [INFO] Start installing Eclipse Che -------"
   chectl server:deploy --cheimage=$CHE_SERVER_IMAGE \
                        --che-operator-cr-patch-yaml=custom-resources.yaml \
                        --platform=openshift \
                        --telemetry=off \
                        --batch
+
+  waitFinishDeploymentCheServer
+  echo "======= [INFO] Eclipse Che is successfully installed ======="
+}
+
+waitFinishDeploymentCheServer() {
+          CURRENT_TIME=$(date +%s)
+          ENDTIME=$((CURRENT_TIME + 60))
+
+          while [ "$(date +%s)" -lt $ENDTIME ]; do
+              podCheServerName=$(oc get pod -n ${CHE_NAMESPACE} -l component=che | grep "che" | awk '{ print $1 }')
+              echo "Pod Che_Server: $podCheServerName"
+              count=$(echo "$podCheServerName" | wc -l)
+              if [ $count -eq 1 ]; then
+                  echo "------- [INFO] Only one Che Server pod is left. -------"
+                  return 0
+              fi
+              echo "------- [INFO] Waiting until only one Che Server pod remains. -------"
+              sleep 5
+          done
+
+          echo "####### [ERROR] Error occurred while waiting for only one Che Server pod. #######"
+          exit 1
 }
 
 # this command starts port forwarding between the local machine and the che-host service in the OpenShift cluster.
 forwardPortToService() {
+  echo "------- [INFO] Start forwarding between the local machine and the che-host service -------"
   oc port-forward service/che-host ${CHE_FORWARDED_PORT}:8080 -n ${CHE_NAMESPACE} &
   sleep 3s
 }
@@ -120,28 +155,28 @@ requestFactoryResolverGitRepoUrl() {
 
 # check that factory resolver returns correct value without any PAT/OAuth setup
 testFactoryResolverNoPatOAuth() {
-  echo "[INFO] Check factory resolver for public repository with NO PAT/OAuth setup"
+  echo "------- [INFO] Check factory resolver for public repository with NO PAT/OAuth setup -------"
   testFactoryResolverResponse $1 200
 
-  echo "[INFO] Check factory resolver for private repository with NO PAT/OAuth setup"
+  echo "------- [INFO] Check factory resolver for private repository with NO PAT/OAuth setup -------"
   testFactoryResolverResponse $2 500
 }
 
 # check that raw devfile url factory resolver returns correct value without any PAT/OAuth setup
 testFactoryResolverNoPatOAuthRaw() {
-  echo "[INFO] Check factory resolver for public repository with NO PAT/OAuth setup"
+  echo "------- [INFO] Check factory resolver for public repository with NO PAT/OAuth setup -------"
   testFactoryResolverResponse $1 200
 
-  echo "[INFO] Check factory resolver for private repository with NO PAT/OAuth setup"
+  echo "------- [INFO] Check factory resolver for private repository with NO PAT/OAuth setup -------"
   testFactoryResolverResponse $2 400
 }
 
 # check that factory resolver returns correct value with PAT/OAuth setup
 testFactoryResolverWithPatOAuth() {
-  echo "[INFO] Check factory resolver for public repository with PAT/OAuth setup"
+  echo "------- [INFO] Check factory resolver for public repository with PAT/OAuth setup -------"
   testFactoryResolverResponse $1 200
 
-  echo "[INFO] Check factory resolver for private repository with PAT/OAuth setup"
+  echo "------- [INFO] Check factory resolver for private repository with PAT/OAuth setup -------"
   testFactoryResolverResponse $2 200
 }
 
@@ -149,11 +184,11 @@ testFactoryResolverResponse() {
   URL=$1
   RESPONSE_CODE=$2
 
-  echo "[INFO] Check factory resolver"
   if [ "$(requestFactoryResolverGitRepoUrl ${URL} | grep "HTTP/1.1 ${RESPONSE_CODE}")" ]; then
-    echo "[INFO] Factory resolver returned 'HTTP/1.1 ${RESPONSE_CODE}' status code. Expected client side response."
+    echo "======= [INFO] Factory resolver returned 'HTTP/1.1 ${RESPONSE_CODE}' status code. Expected client side response. ======="
   else
-    echo "[ERROR] Factory resolver returned wrong status code. Expected: HTTP/1.1 ${RESPONSE_CODE}."
+    echo "####### [ERROR] Factory resolver returned wrong status code. Expected: HTTP/1.1 ${RESPONSE_CODE}. #######
+####### Cause possible: PR code regress or service is changed. Need to investigate it. #######"
     exit 1
   fi
 }
@@ -171,12 +206,19 @@ requestProvisionNamespace() {
 initUserNamespace() {
   OCP_USER_NAME=$1
 
-  echo "[INFO] Initialize user namespace"
+  echo "------- [INFO] Initialize user namespace -------"
   oc login -u=${OCP_USER_NAME} -p=${OCP_LOGIN_PASSWORD} --insecure-skip-tls-verify=false
-  if [ "$(requestProvisionNamespace | grep "HTTP/1.1 200")" ]; then
+
-    echo "[INFO] Request provision user namespace returned 'HTTP/1.1 200' status code."
+  request_result=$(requestProvisionNamespace) || true
+
+  if [ -z "$request_result" ]; then
+    echo "####### [ERROR] Cause possible: lost connection to pod, this is an infrastructure problem. Try to rerun the test. #######"
+    exit 1
+  elif [ "$(echo "$request_result" | grep "HTTP/1.1 200")" ]; then
+    echo "======= [INFO] Request provision user namespace returned 'HTTP/1.1 200' status code. User namespace is created. ======="
   else
-    echo "[ERROR] Request provision user namespace returned wrong status code. Expected: HTTP/1.1 200"
+    echo "####### [ERROR] Request provision user namespace returned wrong status code. Expected: HTTP/1.1 200. #######
+####### User namespace creation failed. Cause possible: PR code regression or service is changed. Need to investigate. #######"
     exit 1
   fi
 }
@@ -186,7 +228,7 @@ setupPersonalAccessToken() {
   GIT_PROVIDER_URL=$2
   GIT_PROVIDER_PAT=$3
 
-  echo "[INFO] Setup Personal Access Token Secret"
+  echo "------- [INFO] Setup Personal Access Token Secret -------"
   oc project ${USER_CHE_NAMESPACE}
   CHE_USER_ID=$(oc get secret user-profile -o jsonpath='{.data.id}' | base64 -d)
   ENCODED_PAT=$(echo -n ${GIT_PROVIDER_PAT} | base64)
@@ -205,13 +247,14 @@ setupPersonalAccessToken() {
   cat pat-secret.yaml
 
   oc apply -f pat-secret.yaml -n ${USER_CHE_NAMESPACE}
+  echo "======= [INFO] Personal Access Token is created. ======="
 }
 
 setupSSHKeyPairs() {
   GIT_PRIVATE_KEY=$1
   GIT_PUBLIC_KEY=$2
 
-  echo "[INFO] Setup SSH Key Pairs Secret"
+  echo "------- [INFO] Setup SSH Key Pairs Secret "-------
   oc project ${USER_CHE_NAMESPACE}
   ENCODED_GIT_PRIVATE_KEY=$(echo "${GIT_PRIVATE_KEY}" | base64 -w 0)
   ENCODED_GIT_PUBLIC_KEY=$(echo "${GIT_PUBLIC_KEY}" | base64 -w 0)
@@ -224,6 +267,7 @@ setupSSHKeyPairs() {
   cat ssh-secret.yaml
 
   oc apply -f ssh-secret.yaml -n ${USER_CHE_NAMESPACE}
+  echo "======= [INFO] SSH Secret is created. ======="
 }
 
 # Only for GitLab server administrator users
@@ -232,7 +276,7 @@ createOAuthApplicationGitLabServer() {
 
   CHE_URL=https://$(oc get route -n ${CHE_NAMESPACE} che -o jsonpath='{.spec.host}')
 
-  echo "[INFO] Create OAuth Application"
+  echo "------- [INFO] Create OAuth Application -------"
   response=$(curl -k -X POST \
     ${GIT_PROVIDER_URL}/api/v4/applications \
     -H "PRIVATE-TOKEN: ${ADMIN_ACCESS_TOKEN}" \
@@ -240,15 +284,11 @@ createOAuthApplicationGitLabServer() {
     -d "redirect_uri=${CHE_URL}/api/oauth/callback" \
     -d "scopes=api write_repository openid")
 
-  echo "[INFO] Response of the created OAuth Application"
+  echo "------- [INFO] Response of the created OAuth Application -------"
 
   OAUTH_ID=$(echo "$response" | jq -r '.id')
   APPLICATION_ID=$(echo "$response" | jq -r '.application_id')
   APPLICATION_SECRET=$(echo "$response" | jq -r '.secret')
-
-  echo "[INFO] OAuth ID: ${OAUTH_ID}"
-  echo "[INFO] Application ID: ${APPLICATION_ID}"
-  echo "[INFO] Application Secret: ${APPLICATION_SECRET}"
 }
 
 # Only for GitLab server administrator users
@@ -256,10 +296,12 @@ deleteOAuthApplicationGitLabServer() {
   OAUTH_ID=$1
   ADMIN_ACCESS_TOKEN=$2
 
-  echo "[INFO] Delete OAuth Application"
+  echo "------- [INFO] Delete OAuth Application -------"
   curl -i -k -X DELETE \
     ${GIT_PROVIDER_URL}/api/v4/applications/${OAUTH_ID} \
     -H "PRIVATE-TOKEN: ${ADMIN_ACCESS_TOKEN}"
+
+  echo "======= [INFO] OAuth Application is deleted ======="
 }
 
 # Only for GitLab server
@@ -267,18 +309,18 @@ revokeAuthorizedOAuthApplication() {
   APPLICATION_ID=$1
   APPLICATION_SECRET=$2
 
-  echo "[INFO] Revoke authorized OAuth application"
+  echo "------- [INFO] Revoke authorized OAuth application -------"
   oc project ${USER_CHE_NAMESPACE}
   OAUTH_TOKEN_NAME=$(oc get secret | grep 'personal-access-token'| awk 'NR==1 { print $1 }')
-  echo "[INFO] OAuth token name: "$OAUTH_TOKEN_NAME
   OAUTH_TOKEN=$(oc get secret $OAUTH_TOKEN_NAME -o jsonpath='{.data.token}' | base64 -d)
-  echo "[INFO] Oauth token: "$OAUTH_TOKEN
 
   curl -i -k -X POST \
     ${GIT_PROVIDER_URL}/oauth/revoke \
     -d "client_id=${APPLICATION_ID}" \
     -d "client_secret=${APPLICATION_SECRET}" \
     -d "token=${OAUTH_TOKEN}"
+
+  echo "======= [INFO] Authorized OAuth application is revoked ======="
 }
 
 # Only for GitLab server
@@ -286,7 +328,7 @@ setupOAuthSecret() {
   APPLICATION_ID=$1
   APPLICATION_SECRET=$2
 
-  echo "[INFO] Setup OAuth Secret"
+  echo "------- [INFO] Setup OAuth Secret -------"
   oc login -u=${OCP_ADMIN_USER_NAME} -p=${OCP_LOGIN_PASSWORD} --insecure-skip-tls-verify=false
   oc project ${CHE_NAMESPACE}
   SERVER_POD=$(oc get pod -l component=che | grep "che" | awk 'NR==1 { print $1 }')
@@ -303,7 +345,7 @@ setupOAuthSecret() {
   cat oauth-secret.yaml
   oc apply -f oauth-secret.yaml -n ${CHE_NAMESPACE}
 
-  echo "[INFO] Wait updating deployment after create OAuth secret"
+  echo "------- [INFO] Wait updating deployment after create OAuth secret -------"
   oc wait --for=delete pod/${SERVER_POD} --timeout=120s
 }
 
@@ -316,6 +358,7 @@ runTestWorkspaceWithGitRepoUrl() {
   oc project ${OCP_USER_NAMESPACE}
   cat .ci/openshift-ci/devworkspace-test.yaml > devworkspace-test.yaml
 
+  echo "------- [INFO] Preparing 'devworkspace-test.yaml' and run test workspace -------"
   # patch the devworkspace-test.yaml file
   sed -i "s#ws-name#${WS_NAME}#g" devworkspace-test.yaml
   sed -i "s#project-name#${PROJECT_NAME}#g" devworkspace-test.yaml
@@ -325,7 +368,7 @@ runTestWorkspaceWithGitRepoUrl() {
 
   oc apply -f devworkspace-test.yaml -n ${OCP_USER_NAMESPACE}
   oc wait -n ${OCP_USER_NAMESPACE} --for=condition=Ready dw ${WS_NAME} --timeout=360s
-  echo "[INFO] Test workspace is run"
+  echo "======= [INFO] Test workspace is run ======="
 }
 
 testProjectIsCloned() {
@@ -334,9 +377,9 @@ testProjectIsCloned() {
 
   WORKSPACE_POD_NAME=$(oc get pods -n ${OCP_USER_NAMESPACE} | grep workspace | awk '{print $1}')
   if oc exec -it -n ${OCP_USER_NAMESPACE} ${WORKSPACE_POD_NAME} -- test -f /projects/${PROJECT_NAME}/${YAML_FILE_NAME}; then
-    echo "[INFO] Project file /projects/${PROJECT_NAME}/${YAML_FILE_NAME} exists."
+    echo "======= [INFO] Project file /projects/${PROJECT_NAME}/${YAML_FILE_NAME} exists. ======="
   else
-    echo "[INFO] Project file /projects/${PROJECT_NAME}/${YAML_FILE_NAME} is absent."
+    echo "======= [INFO] Project file /projects/${PROJECT_NAME}/${YAML_FILE_NAME} is absent. ======="
     return 1
   fi
 }
@@ -346,7 +389,7 @@ testGitCredentialsData() {
   GIT_PROVIDER_PAT=$2
   GIT_PROVIDER_URL=$3
 
-  echo "[INFO] Check the 'git credentials' is in a workspace"
+  echo "------- [INFO] Check the 'git credentials' is in a workspace -------"
   hostName="${GIT_PROVIDER_URL#https://}"
 
   if [ "${GIT_PROVIDER_TYPE}" == "azure-devops" ]; then
@@ -358,9 +401,10 @@ testGitCredentialsData() {
   gitCredentials="https://${userName}:${GIT_PROVIDER_PAT}@${hostName}"
   WORKSPACE_POD_NAME=$(oc get pods -n ${OCP_USER_NAMESPACE} | grep workspace | awk '{print $1}')
   if oc exec -it -n ${OCP_USER_NAMESPACE} ${WORKSPACE_POD_NAME} -- cat /.git-credentials/credentials | grep -q ${gitCredentials}; then
-    echo "[INFO] Git credentials file '/.git-credentials/credentials' exists and has the expected content."
+    echo "======= [INFO] Git credentials file '/.git-credentials/credentials' exists and has the expected content. ======="
   else
-    echo "[ERROR] Git credentials file '/.git-credentials/credentials' does not exist or has incorrect content."
+    echo "####### [ERROR] Git credentials file '/.git-credentials/credentials' does not exist or has incorrect content. ######
+###### Cause possible: PR code regress or service is changed. Need to investigate it. #######"
     exit 1
   fi
 }
@@ -368,7 +412,7 @@ testGitCredentialsData() {
 deleteTestWorkspace() {
   WS_NAME=$1
   OCP_USER_NAMESPACE=$2
-
+  echo "------- [INFO] Delete test workspace -------"
   oc delete dw ${WS_NAME} -n ${OCP_USER_NAMESPACE}
 }
 
@@ -386,7 +430,7 @@ startOAuthFactoryTest() {
   sed -i "s#GIT_PROVIDER_USER_NAME#${GIT_PROVIDER_LOGIN}#g" oauth-factory-test.yaml
   sed -i "s#GIT_PROVIDER_USER_PASSWORD#${GIT_PROVIDER_PASSWORD}#g" oauth-factory-test.yaml
 
-  echo "[INFO] Applying the following patched OAuth Factory Test Pod:"
+  echo "------- [INFO] Applying the following patched OAuth Factory Test Pod: -------"
   cat oauth-factory-test.yaml
   echo "[INFO] --------------------------------------------------"
   oc apply -f oauth-factory-test.yaml
@@ -397,7 +441,7 @@ startOAuthFactoryTest() {
     PHASE=$(oc get pod -n ${CHE_NAMESPACE} ${TEST_POD_NAME} \
         --template='{{ .status.phase }}')
     if [[ ${PHASE} == "Running" ]]; then
-      echo "[INFO] Factory test started successfully."
+      echo "======= [INFO] Factory test started successfully. ======="
       return
     fi
 
@@ -405,7 +449,8 @@ startOAuthFactoryTest() {
     n=$(( n+1 ))
   done
 
-  echo "[ERROR] Failed to start Factory test."
+  echo "####### [ERROR] Failed to start Factory test. #######
+###### Cause possible: an infrastructure problem, pod could not start, try to rerun the test. #######"
   exit 1
 }
 
@@ -418,7 +463,7 @@ startSmokeTest() {
   sed -i "s#OCP_USER_NAME#${OCP_NON_ADMIN_USER_NAME}#g" che-smoke-test.yaml
   sed -i "s#OCP_USER_PASSWORD#${OCP_LOGIN_PASSWORD}#g" che-smoke-test.yaml
 
-  echo "[INFO] Applying the following patched Smoke Test Pod:"
+  echo "------- [INFO] Applying the following patched Smoke Test Pod: -------"
   cat che-smoke-test.yaml
   echo "[INFO] --------------------------------------------------"
   oc apply -f che-smoke-test.yaml
@@ -429,7 +474,7 @@ startSmokeTest() {
     PHASE=$(oc get pod -n ${CHE_NAMESPACE} ${TEST_POD_NAME} \
         --template='{{ .status.phase }}')
     if [[ ${PHASE} == "Running" ]]; then
-      echo "[INFO] Smoke test started successfully."
+      echo "======= [INFO] Smoke test started successfully. ======="
       return
     fi
 
@@ -437,22 +482,26 @@ startSmokeTest() {
     n=$(( n+1 ))
   done
 
-  echo "[ERROR] Failed to start Smoke test."
+  echo "####### [ERROR] Failed to start Smoke test. #######
+####### Cause possible: an infrastructure problem, pod could not start, try to rerun the test. #######"
   exit 1
 }
 
 # Catch the finish of the job and write logs in artifacts.
 catchFinish() {
-  echo "[INFO] Terminate the process after finish the test script."
   local RESULT=$?
-  killProcessByPort
   if [ "$RESULT" != "0" ]; then
     set +e
     collectEclipseCheLogs
     set -e
   fi
 
-  [[ "${RESULT}" != "0" ]] && echo "[ERROR] Job failed." || echo "[INFO] Job completed successfully."
+  echo "------- [INFO] Terminate the process after finish the test script. -------"
+  set +e
+  killProcessByPort
+  set -e
+
+  [[ "${RESULT}" != "0" ]] && echo "####### [ERROR] Job failed. #######" || echo "####### [INFO] Job completed successfully. #######"
   exit $RESULT
 }
 
@@ -466,16 +515,16 @@ collectEclipseCheLogs() {
 }
 
 collectLogs() {
-  echo "[INFO] Waiting until test pod finished"
+  echo "------- [INFO] Waiting until test pod finished. -------"
   oc logs -n ${CHE_NAMESPACE} ${TEST_POD_NAME} -c test -f
   sleep 3
 
   # Download artifacts
   set +e
-  echo "[INFO] Collect all Eclipse Che logs and cluster CR."
+  echo -------" [INFO] Collect all Eclipse Che logs and cluster CR. -------"
   collectEclipseCheLogs
 
-  echo "[INFO] Downloading test report."
+  echo "------- [INFO] Downloading test report. -------"
   mkdir -p ${ARTIFACTS_DIR}/e2e
   oc rsync -n ${CHE_NAMESPACE} ${TEST_POD_NAME}:/tmp/e2e/report/ ${ARTIFACTS_DIR}/e2e -c download-reports
   oc exec -n ${CHE_NAMESPACE} ${TEST_POD_NAME} -c download-reports -- touch /tmp/done
@@ -490,10 +539,11 @@ collectLogs() {
 
   EXIT_CODE=$(oc logs -n ${CHE_NAMESPACE} ${TEST_POD_NAME} -c test | grep EXIT_CODE)
   if [[ ${EXIT_CODE} != "+ EXIT_CODE=0" ]]; then
-    echo "[ERROR] GUI test failed. Job failed."
+    echo "####### [ERROR] GUI test failed. Job failed. #######
+###### Cause possible: PR code regress or service is changed. Need to investigate it. #######"
     exit 1
   fi
-  echo "[INFO] Job completed successfully."
+  echo "======= [INFO] Job completed successfully. ======="
 }
 
 testCloneGitRepoNoProjectExists() {
@@ -503,10 +553,11 @@ testCloneGitRepoNoProjectExists() {
     OCP_USER_NAMESPACE=$4
 
     runTestWorkspaceWithGitRepoUrl ${WS_NAME} ${PROJECT_NAME} ${GIT_REPO_URL} ${OCP_USER_NAMESPACE}
-    echo "[INFO] Check the private repository is NOT cloned with NO PAT/OAuth setup"
+    echo "------- [INFO] Check the private repository is NOT cloned with NO PAT/OAuth setup. -------"
     testProjectIsCloned ${PROJECT_NAME} ${OCP_USER_NAMESPACE} && \
-    { echo "[ERROR] Project file /projects/${PROJECT_NAME}/${YAML_FILE_NAME} should NOT be present" && exit 1; }
+    { echo "####### [ERROR] Project file /projects/${PROJECT_NAME}/${YAML_FILE_NAME} should NOT be present. #######
-    echo "[INFO] Project file /projects/${PROJECT_NAME}/${YAML_FILE_NAME} is NOT present. This is EXPECTED"
+####### Cause possible: PR code regress or service is changed. Need to investigate it. #######" && exit 1; }
+    echo "======= [INFO] Project file /projects/${PROJECT_NAME}/${YAML_FILE_NAME} is NOT present. This is EXPECTED. ======="
 }
 
 # Test that the repository is cloned when PAT, OAuth or SSH is configured
@@ -517,9 +568,10 @@ testCloneGitRepoProjectShouldExists() {
   OCP_USER_NAMESPACE=$4
 
   runTestWorkspaceWithGitRepoUrl ${WS_NAME} ${PROJECT_NAME} ${GIT_REPO_URL} ${OCP_USER_NAMESPACE}
-  echo "[INFO] Check the repository is cloned"
+  echo "------- [INFO] Check the repository is cloned. -------"
   testProjectIsCloned ${PROJECT_NAME} ${OCP_USER_NAMESPACE} || \
-  { echo "[ERROR] Project file /projects/${PROJECT_NAME}/${YAML_FILE_NAME} should be present." && exit 1; }
+  { echo "####### [ERROR] Project file /projects/${PROJECT_NAME}/${YAML_FILE_NAME} should be present. #######
+###### Cause possible: PR code regress or service is changed. Need to investigate it. #######" && exit 1; }
 }
 
 setupTestEnvironment() {

.ci/openshift-ci/test-azure-no-pat-oauth-flow-raw-devfile-url.sh

@@ -16,6 +16,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PUBLIC_REPO_RAW_PATH_URL=${PUBLIC_REPO_RAW_PATH_URL:-"https://dev.azure.com/chepullreq1/che-pr-public/_apis/git/repositories/public-repo/items?path=/devfile.yaml"}
 export PRIVATE_REPO_RAW_PATH_URL=${PRIVATE_REPO_URL:-"https://dev.azure.com/chepullreq1/che-pr-private/_apis/git/repositories/private-repo/items?path=/devfile.yaml"}
 

.ci/openshift-ci/test-azure-no-pat-oauth-flow-ssh-url.sh

@@ -17,6 +17,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PUBLIC_REPO_SSH_URL=${PUBLIC_REPO_SSH_URL:-"git@ssh.dev.azure.com:v3/chepullreq1/che-pr-public/public-repo"}
 export PRIVATE_REPO_SSH_URL=${PRIVATE_REPO_SSH_URL:-"git@ssh.dev.azure.com:v3/chepullreq1/che-pr-private/private-repo"}
 

.ci/openshift-ci/test-azure-no-pat-oauth-flow.sh

@@ -16,6 +16,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PUBLIC_REPO_URL=${PUBLIC_REPO_URL:-"https://chepullreq1@dev.azure.com/chepullreq1/che-pr-public/_git/public-repo"}
 export PRIVATE_REPO_URL=${PRIVATE_REPO_URL:-"https://dev.azure.com/chepullreq1/che-pr-private/_git/private-repo"}
 

.ci/openshift-ci/test-azure-with-pat-setup-flow.sh

@@ -16,6 +16,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PUBLIC_REPO_URL=${PUBLIC_REPO_URL:-"https://chepullreq1@dev.azure.com/chepullreq1/che-pr-public/_git/public-repo"}
 export PRIVATE_REPO_URL=${PRIVATE_REPO_URL:-"https://dev.azure.com/chepullreq1/che-pr-private/_git/private-repo"}
 export GIT_PROVIDER_TYPE=${GIT_PROVIDER_TYPE:-"azure-devops"}
@@ -34,24 +36,24 @@ setupPersonalAccessToken  ${GIT_PROVIDER_TYPE} ${GIT_PROVIDER_URL} ${AZURE_PAT}
 requestProvisionNamespace
 testFactoryResolverWithPatOAuth ${PUBLIC_REPO_URL} ${PRIVATE_REPO_URL}
 
-echo "[INFO] Check clone public repository with PAT setup"
+echo "------- [INFO] Check clone public repository with PAT setup -------"
 testCloneGitRepoProjectShouldExists ${PUBLIC_REPO_WORKSPACE_NAME} ${PUBLIC_PROJECT_NAME} ${PUBLIC_REPO_URL} ${USER_CHE_NAMESPACE}
 testGitCredentialsData ${USER_CHE_NAMESPACE} ${AZURE_PAT} ${GIT_PROVIDER_URL}
 deleteTestWorkspace ${PUBLIC_REPO_WORKSPACE_NAME} ${USER_CHE_NAMESPACE}
 
-echo "[INFO] Check clone private repository with PAT setup"
+echo "------- [INFO] Check clone private repository with PAT setup -------"
 testCloneGitRepoProjectShouldExists ${PRIVATE_REPO_WORKSPACE_NAME} ${PRIVATE_PROJECT_NAME} ${PRIVATE_REPO_URL} ${USER_CHE_NAMESPACE}
 testGitCredentialsData ${USER_CHE_NAMESPACE} ${AZURE_PAT} ${GIT_PROVIDER_URL}
 deleteTestWorkspace ${PRIVATE_REPO_WORKSPACE_NAME} ${USER_CHE_NAMESPACE}
 
-echo "[INFO] Check clone private repository by raw devfile URL with PAT setup"
+echo "------- [INFO] Check clone private repository by raw devfile URL with PAT setup -------"
 testFactoryResolverResponse ${PRIVATE_REPO_RAW_PATH_URL} 200
 testCloneGitRepoProjectShouldExists ${PRIVATE_REPO_WORKSPACE_NAME} ${PRIVATE_PROJECT_NAME} ${PRIVATE_REPO_RAW_PATH_URL} ${USER_CHE_NAMESPACE}
 deleteTestWorkspace ${PRIVATE_REPO_WORKSPACE_NAME} ${USER_CHE_NAMESPACE}
 
 setupSSHKeyPairs "${AZURE_PRIVATE_KEY}" "${AZURE_PUBLIC_KEY}"
 
-echo "[INFO] Check clone private repository by SSH URL with PAT setup"
+echo "------- [INFO] Check clone private repository by SSH URL with PAT setup -------"
 testFactoryResolverResponse ${PRIVATE_REPO_SSH_URL} 200
 testCloneGitRepoProjectShouldExists ${PRIVATE_REPO_WORKSPACE_NAME} ${PRIVATE_PROJECT_NAME} ${PRIVATE_REPO_SSH_URL} ${USER_CHE_NAMESPACE}
 deleteTestWorkspace ${PRIVATE_REPO_WORKSPACE_NAME} ${USER_CHE_NAMESPACE}

.ci/openshift-ci/test-bitbucket-no-pat-oauth-flow-raw-devfile-url.sh

@@ -16,6 +16,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PUBLIC_REPO_RAW_PATH_URL=${PUBLIC_REPO_RAW_PATH_URL:-"https://bitbucket.org/chepullreq/public-repo/raw/746000bd63a54eaf8ea8aba9dfe6620e5c6c61d7/devfile.yaml"}
 export PRIVATE_REPO_RAW_PATH_URL=${PRIVATE_REPO_URL:-"https://bitbucket.org/chepullreq/private-repo/raw/80b183d27c6c533462128b0c092208aad73b2906/devfile.yaml"}
 

.ci/openshift-ci/test-bitbucket-no-pat-oauth-flow-ssh-url.sh

@@ -17,6 +17,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PUBLIC_REPO_SSH_URL=${PUBLIC_REPO_SSH_URL:-"git@bitbucket.org:chepullreq/public-repo.git"}
 export PRIVATE_REPO_SSH_URL=${PRIVATE_REPO_SSH_URL:-"git@bitbucket.org:chepullreq/private-repo.git"}
 

.ci/openshift-ci/test-bitbucket-no-pat-oauth-flow.sh

@@ -16,6 +16,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PUBLIC_REPO_URL=${PUBLIC_REPO_URL:-"https://chepullreq1@bitbucket.org/chepullreq/public-repo.git"}
 export PRIVATE_REPO_URL=${PRIVATE_REPO_URL:-"https://chepullreq1@bitbucket.org/chepullreq/private-repo.git"}
 

.ci/openshift-ci/test-che-smoke-test.sh

@@ -16,6 +16,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export TEST_POD_NAME=${TEST_POD_NAME:-"che-smoke-test"}
 
 # import common test functions

.ci/openshift-ci/test-gitea-no-pat-oauth-flow.sh

@@ -16,6 +16,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PUBLIC_REPO_SSH_URL=${PUBLIC_REPO_SSH_URL:-"git@gitea.com:chepullreq1/public-repo.git"}
 export PRIVATE_REPO_SSH_URL=${PRIVATE_REPO_SSH_URL:-"git@gitea.com:chepullreq1/private-repo.git"}
 export PUBLIC_REPO_RAW_PATH_URL=${PUBLIC_REPO_RAW_PATH_URL:-"https://gitea.com/chepullreq1/public-repo/raw/branch/main/devfile.yaml"}

.ci/openshift-ci/test-gitea-with-pat-setup-flow.sh

@@ -16,6 +16,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PRIVATE_REPO_RAW_PATH_URL=${PRIVATE_REPO_RAW_PATH_URL:-"https://${GITEA_PAT}@gitea.com/chepullreq1/private-repo/raw/branch/main/devfile.yaml"}
 
 # import common test functions

.ci/openshift-ci/test-github-no-pat-oauth-flow-raw-devfile-url.sh

@@ -16,6 +16,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PUBLIC_REPO_RAW_PATH_URL=${PUBLIC_REPO_RAW_PATH_URL:-"https://raw.githubusercontent.com/chepullreq1/public-repo/main/devfile.yaml"}
 export PRIVATE_REPO_RAW_PATH_URL=${PRIVATE_REPO_URL:-"https://raw.githubusercontent.com/chepullreq1/private-repo/main/devfile.yaml"}
 

.ci/openshift-ci/test-github-no-pat-oauth-flow-ssh-url.sh

@@ -17,6 +17,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PUBLIC_REPO_SSH_URL=${PUBLIC_REPO_SSH_URL:-"git@github.com:chepullreq1/public-repo.git"}
 export PRIVATE_REPO_SSH_URL=${PRIVATE_REPO_SSH_URL:-"git@github.com:chepullreq1/private-repo.git"}
 

.ci/openshift-ci/test-github-no-pat-oauth-flow.sh

@@ -16,6 +16,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PUBLIC_REPO_URL=${PUBLIC_REPO_URL:-"https://github.com/chepullreq1/public-repo.git"}
 export PRIVATE_REPO_URL=${PRIVATE_REPO_URL:-"https://github.com/chepullreq1/private-repo.git"}
 

.ci/openshift-ci/test-github-with-pat-setup-flow.sh

@@ -16,6 +16,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PUBLIC_REPO_URL=${PUBLIC_REPO_URL:-"https://github.com/chepullreq1/public-repo.git"}
 export PRIVATE_REPO_URL=${PRIVATE_REPO_URL:-"https://github.com/chepullreq1/private-repo.git"}
 export GIT_PROVIDER_TYPE=${GIT_PROVIDER_TYPE:-"github"}
@@ -34,24 +36,24 @@ setupPersonalAccessToken  ${GIT_PROVIDER_TYPE} ${GIT_PROVIDER_URL} ${GITHUB_PAT}
 requestProvisionNamespace
 testFactoryResolverWithPatOAuth ${PUBLIC_REPO_URL} ${PRIVATE_REPO_URL}
 
-echo "[INFO] Check clone public repository with PAT setup"
+echo "------- [INFO] Check clone public repository with PAT setup -------"
 testCloneGitRepoProjectShouldExists ${PUBLIC_REPO_WORKSPACE_NAME} ${PUBLIC_PROJECT_NAME} ${PUBLIC_REPO_URL} ${USER_CHE_NAMESPACE}
 testGitCredentialsData ${USER_CHE_NAMESPACE} ${GITHUB_PAT} ${GIT_PROVIDER_URL}
 deleteTestWorkspace ${PUBLIC_REPO_WORKSPACE_NAME} ${USER_CHE_NAMESPACE}
 
-echo "[INFO] Check clone private repository with PAT setup"
+echo "------- [INFO] Check clone private repository with PAT setup -------"
 testCloneGitRepoProjectShouldExists ${PRIVATE_REPO_WORKSPACE_NAME} ${PRIVATE_PROJECT_NAME} ${PRIVATE_REPO_URL} ${USER_CHE_NAMESPACE}
 testGitCredentialsData ${USER_CHE_NAMESPACE} ${GITHUB_PAT} ${GIT_PROVIDER_URL}
 deleteTestWorkspace ${PRIVATE_REPO_WORKSPACE_NAME} ${USER_CHE_NAMESPACE}
 
-echo "[INFO] Check clone private repository by raw devfile URL with PAT setup"
+echo "------- [INFO] Check clone private repository by raw devfile URL with PAT setup -------"
 testFactoryResolverResponse ${PRIVATE_REPO_RAW_PATH_URL} 200
 testCloneGitRepoProjectShouldExists ${PRIVATE_REPO_WORKSPACE_NAME} ${PRIVATE_PROJECT_NAME} ${PRIVATE_REPO_RAW_PATH_URL} ${USER_CHE_NAMESPACE}
 deleteTestWorkspace ${PRIVATE_REPO_WORKSPACE_NAME} ${USER_CHE_NAMESPACE}
 
 setupSSHKeyPairs "${GITHUB_PRIVATE_KEY}" "${GITHUB_PUBLIC_KEY}"
 
-echo "[INFO] Check clone private repository by SSH URL with PAT setup"
+echo "------- [INFO] Check clone private repository by SSH URL with PAT setup -------"
 testFactoryResolverResponse ${PRIVATE_REPO_SSH_URL} 200
 testCloneGitRepoProjectShouldExists ${PRIVATE_REPO_WORKSPACE_NAME} ${PRIVATE_PROJECT_NAME} ${PRIVATE_REPO_SSH_URL} ${USER_CHE_NAMESPACE}
 deleteTestWorkspace ${PRIVATE_REPO_WORKSPACE_NAME} ${USER_CHE_NAMESPACE}

.ci/openshift-ci/test-gitlab-no-pat-oauth-flow-raw-devfile-url.sh

@@ -16,6 +16,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PUBLIC_REPO_RAW_PATH_URL=${PUBLIC_REPO_RAW_PATH_URL:-"https://gitlab.com/chepullreq1/public-repo/-/raw/main/devfile.yaml"}
 export PRIVATE_REPO_RAW_PATH_URL=${PRIVATE_REPO_URL:-"https://gitlab.com/chepullreq1/private-repo/-/raw/main/devfile.yaml"}
 

.ci/openshift-ci/test-gitlab-no-pat-oauth-flow-ssh-url.sh

@@ -17,6 +17,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PUBLIC_REPO_SSH_URL=${PUBLIC_REPO_SSH_URL:-"git@gitlab.com:chepullreq1/public-repo.git"}
 export PRIVATE_REPO_SSH_URL=${PRIVATE_REPO_SSH_URL:-"git@gitlab.com:chepullreq1/private-repo.git"}
 

.ci/openshift-ci/test-gitlab-no-pat-oauth-flow.sh

@@ -16,6 +16,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PUBLIC_REPO_URL=${PUBLIC_REPO_URL:-"https://gitlab.com/chepullreq1/public-repo.git"}
 export PRIVATE_REPO_URL=${PRIVATE_REPO_URL:-"https://gitlab.com/chepullreq1/private-repo.git"}
 

.ci/openshift-ci/test-gitlab-with-oauth-setup-flow.sh

@@ -16,6 +16,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export TEST_POD_NAME=${TEST_POD_NAME:-"oauth-factory-test"}
 export GIT_PROVIDER_TYPE=${GIT_PROVIDER_TYPE:-"gitlab"}
 export GIT_PROVIDER_URL=${GIT_PROVIDER_URL:-"https://gitlab-gitlab-system.apps.git.crw-qe.com"}

.ci/openshift-ci/test-gitlab-with-pat-setup-flow.sh

@@ -16,6 +16,8 @@ set -ex
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+echo "======= [INFO] OpenShift CI infrastructure is ready. Running test. ======="
+
 export PUBLIC_REPO_URL=${PUBLIC_REPO_URL:-"https://gitlab.com/chepullreq1/public-repo.git"}
 export PRIVATE_REPO_URL=${PRIVATE_REPO_URL:-"https://gitlab.com/chepullreq1/private-repo.git"}
 export GIT_PROVIDER_TYPE=${GIT_PROVIDER_TYPE:-"gitlab"}
@@ -34,24 +36,24 @@ setupPersonalAccessToken  ${GIT_PROVIDER_TYPE} ${GIT_PROVIDER_URL} ${GITLAB_PAT}
 requestProvisionNamespace
 testFactoryResolverWithPatOAuth ${PUBLIC_REPO_URL} ${PRIVATE_REPO_URL}
 
-echo "[INFO] Check clone public repository with PAT setup"
+echo "------- [INFO] Check clone public repository with PAT setup -------"
 testCloneGitRepoProjectShouldExists ${PUBLIC_REPO_WORKSPACE_NAME} ${PUBLIC_PROJECT_NAME} ${PUBLIC_REPO_URL} ${USER_CHE_NAMESPACE}
 testGitCredentialsData ${USER_CHE_NAMESPACE} ${GITLAB_PAT} ${GIT_PROVIDER_URL}
 deleteTestWorkspace ${PUBLIC_REPO_WORKSPACE_NAME} ${USER_CHE_NAMESPACE}
 
-echo "[INFO] Check clone private repository with PAT setup"
+echo "------- [INFO] Check clone private repository with PAT setup -------"
 testCloneGitRepoProjectShouldExists ${PRIVATE_REPO_WORKSPACE_NAME} ${PRIVATE_PROJECT_NAME} ${PRIVATE_REPO_URL} ${USER_CHE_NAMESPACE}
 testGitCredentialsData ${USER_CHE_NAMESPACE} ${GITLAB_PAT} ${GIT_PROVIDER_URL}
 deleteTestWorkspace ${PRIVATE_REPO_WORKSPACE_NAME} ${USER_CHE_NAMESPACE}
 
-echo "[INFO] Check clone private repository by raw devfile URL with PAT setup"
+echo "------- [INFO] Check clone private repository by raw devfile URL with PAT setup -------"
 testFactoryResolverResponse ${PRIVATE_REPO_RAW_PATH_URL} 200
 testCloneGitRepoProjectShouldExists ${PRIVATE_REPO_WORKSPACE_NAME} ${PRIVATE_PROJECT_NAME} ${PRIVATE_REPO_RAW_PATH_URL} ${USER_CHE_NAMESPACE}
 deleteTestWorkspace ${PRIVATE_REPO_WORKSPACE_NAME} ${USER_CHE_NAMESPACE}
 
 setupSSHKeyPairs "${GITLAB_PRIVATE_KEY}" "${GITLAB_PUBLIC_KEY}"
 
-echo "[INFO] Check clone private repository by SSH URL with PAT setup"
+echo "------- [INFO] Check clone private repository by SSH URL with PAT setup -------"
 testFactoryResolverResponse ${PRIVATE_REPO_SSH_URL} 200
 testCloneGitRepoProjectShouldExists ${PRIVATE_REPO_WORKSPACE_NAME} ${PRIVATE_PROJECT_NAME} ${PRIVATE_REPO_SSH_URL} ${USER_CHE_NAMESPACE}
 deleteTestWorkspace ${PRIVATE_REPO_WORKSPACE_NAME} ${USER_CHE_NAMESPACE}