From aa52e2892bf08dc167c31a929fb5c5c1c816e86d Mon Sep 17 00:00:00 2001 From: Anatolii Bazko Date: Fri, 8 Jan 2021 18:30:58 +0200 Subject: [PATCH] Add liveness and readiness probes (#610) * Add liveness and readiness probes Signed-off-by: Anatolii Bazko --- .../{test-latest-olm.sh => test-olm.sh} | 0 ...sh => test-operator-singlehost-gateway.sh} | 1 - ....sh => test-operator-singlehost-native.sh} | 1 - .github/bin/minikube/test-operator-update.sh | 46 ++++++++++++++ ...pdate-olm.sh => test-stable-olm-update.sh} | 0 .github/bin/minishift/test-operator-update.sh | 46 ++++++++++++++ ...st-latest-operator.sh => test-operator.sh} | 0 ...ator.sh => test-stable-operator-update.sh} | 0 ...kube-latest-olm.yaml => minikube-olm.yaml} | 2 +- ...yaml => minikube-operator-singlehost.yaml} | 4 +- .../workflows/minikube-operator-update.yaml | 37 +++++++++++ ...s.yaml => minikube-stable-olm-update.yaml} | 2 +- .../workflows/minishift-operator-update.yaml | 61 +++++++++++++++++++ ...ft-latest.yaml => minishift-operator.yaml} | 2 +- ... => minishift-stable-operator-update.yaml} | 2 +- cmd/manager/main.go | 17 ++++-- .../che-operator.clusterserviceversion.yaml | 28 +++++++-- .../che-operator.clusterserviceversion.yaml | 28 +++++++-- deploy/operator.yaml | 21 +++++++ 19 files changed, 278 insertions(+), 20 deletions(-) rename .github/bin/minikube/{test-latest-olm.sh => test-olm.sh} (100%) rename .github/bin/minikube/{test-latest-operator-singlehost-gateway.sh => test-operator-singlehost-gateway.sh} (97%) rename .github/bin/minikube/{test-latest-operator-singlehost-native.sh => test-operator-singlehost-native.sh} (97%) create mode 100755 .github/bin/minikube/test-operator-update.sh rename .github/bin/minikube/{test-update-olm.sh => test-stable-olm-update.sh} (100%) create mode 100755 .github/bin/minishift/test-operator-update.sh rename .github/bin/minishift/{test-latest-operator.sh => test-operator.sh} (100%) rename .github/bin/minishift/{test-update-operator.sh => test-stable-operator-update.sh} (100%) rename .github/workflows/{minikube-latest-olm.yaml => minikube-olm.yaml} (94%) rename .github/workflows/{minikube-latest-operator-singlehost.yaml => minikube-operator-singlehost.yaml} (91%) create mode 100644 .github/workflows/minikube-operator-update.yaml rename .github/workflows/{minikube-stable-updates.yaml => minikube-stable-olm-update.yaml} (93%) create mode 100644 .github/workflows/minishift-operator-update.yaml rename .github/workflows/{minishift-latest.yaml => minishift-operator.yaml} (97%) rename .github/workflows/{minishift-stable-updates.yaml => minishift-stable-operator-update.yaml} (94%) diff --git a/.github/bin/minikube/test-latest-olm.sh b/.github/bin/minikube/test-olm.sh similarity index 100% rename from .github/bin/minikube/test-latest-olm.sh rename to .github/bin/minikube/test-olm.sh diff --git a/.github/bin/minikube/test-latest-operator-singlehost-gateway.sh b/.github/bin/minikube/test-operator-singlehost-gateway.sh similarity index 97% rename from .github/bin/minikube/test-latest-operator-singlehost-gateway.sh rename to .github/bin/minikube/test-operator-singlehost-gateway.sh index 83ea82ce4..cfde421ef 100755 --- a/.github/bin/minikube/test-latest-operator-singlehost-gateway.sh +++ b/.github/bin/minikube/test-operator-singlehost-gateway.sh @@ -22,7 +22,6 @@ source "${OPERATOR_REPO}"/.github/bin/common.sh trap "catchFinish" EXIT SIGINT prepareTemplates() { - disableOpenShiftOAuth ${TEMPLATES} disableUpdateAdminPassword ${TEMPLATES} setCustomOperatorImage ${TEMPLATES} ${OPERATOR_IMAGE} setServerExposureStrategy ${TEMPLATES} "single-host" diff --git a/.github/bin/minikube/test-latest-operator-singlehost-native.sh b/.github/bin/minikube/test-operator-singlehost-native.sh similarity index 97% rename from .github/bin/minikube/test-latest-operator-singlehost-native.sh rename to .github/bin/minikube/test-operator-singlehost-native.sh index d90239617..3252badfe 100755 --- a/.github/bin/minikube/test-latest-operator-singlehost-native.sh +++ b/.github/bin/minikube/test-operator-singlehost-native.sh @@ -21,7 +21,6 @@ source "${OPERATOR_REPO}"/.github/bin/common.sh trap "catchFinish" EXIT SIGINT prepareTemplates() { - disableOpenShiftOAuth ${TEMPLATES} disableUpdateAdminPassword ${TEMPLATES} setCustomOperatorImage ${TEMPLATES} ${OPERATOR_IMAGE} setServerExposureStrategy ${TEMPLATES} "single-host" diff --git a/.github/bin/minikube/test-operator-update.sh b/.github/bin/minikube/test-operator-update.sh new file mode 100755 index 000000000..632169b84 --- /dev/null +++ b/.github/bin/minikube/test-operator-update.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# +# Copyright (c) 2020 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +# Contributors: +# Red Hat, Inc. - initial API and implementation + +set -e +set -x + +# Get absolute path for root repo directory from github actions context: https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions +export OPERATOR_REPO="${GITHUB_WORKSPACE}" +source "${OPERATOR_REPO}"/.github/bin/common.sh + +# Stop execution on any error +trap "catchFinish" EXIT SIGINT + +prepareTemplates() { + disableUpdateAdminPassword ${LAST_OPERATOR_TEMPLATE} + setIngressDomain ${LAST_OPERATOR_TEMPLATE} "$(minikube ip).nip.io" + setCustomOperatorImage ${TEMPLATES} ${OPERATOR_IMAGE} +} + +runTest() { + deployEclipseChe "operator" "minikube" "quay.io/eclipse/che-operator:${LAST_PACKAGE_VERSION}" ${LAST_OPERATOR_TEMPLATE} + createWorkspace + + updateEclipseChe ${OPERATOR_IMAGE} ${TEMPLATES} + waitEclipseCheDeployed "nightly" + + startExistedWorkspace + waitWorkspaceStart +} + +init +initLatestTemplates +initStableTemplates "kubernetes" "stable" +prepareTemplates +buildCheOperatorImage +copyCheOperatorImageToMinikube +runTest diff --git a/.github/bin/minikube/test-update-olm.sh b/.github/bin/minikube/test-stable-olm-update.sh similarity index 100% rename from .github/bin/minikube/test-update-olm.sh rename to .github/bin/minikube/test-stable-olm-update.sh diff --git a/.github/bin/minishift/test-operator-update.sh b/.github/bin/minishift/test-operator-update.sh new file mode 100755 index 000000000..0e20f000f --- /dev/null +++ b/.github/bin/minishift/test-operator-update.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# +# Copyright (c) 2020 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +# Contributors: +# Red Hat, Inc. - initial API and implementation + +set -e +set -x + +# Get absolute path for root repo directory from github actions context: https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions +export OPERATOR_REPO="${GITHUB_WORKSPACE}" +source "${OPERATOR_REPO}"/.github/bin/common.sh + +# Stop execution on any error +trap "catchFinish" EXIT SIGINT + +prepareTemplates() { + disableOpenShiftOAuth ${LAST_OPERATOR_TEMPLATE} + disableUpdateAdminPassword ${LAST_OPERATOR_TEMPLATE} + setCustomOperatorImage ${TEMPLATES} ${OPERATOR_IMAGE} +} + +runTest() { + deployEclipseChe "operator" "minishift" "quay.io/eclipse/che-operator:${LAST_PACKAGE_VERSION}" ${LAST_OPERATOR_TEMPLATE} + createWorkspace + + updateEclipseChe ${OPERATOR_IMAGE} ${TEMPLATES} + waitEclipseCheDeployed "nightly" + + startExistedWorkspace + waitWorkspaceStart +} + +init +installYq +initLatestTemplates +initStableTemplates "openshift" "stable" +prepareTemplates +copyCheOperatorImageToMinishift +runTest diff --git a/.github/bin/minishift/test-latest-operator.sh b/.github/bin/minishift/test-operator.sh similarity index 100% rename from .github/bin/minishift/test-latest-operator.sh rename to .github/bin/minishift/test-operator.sh diff --git a/.github/bin/minishift/test-update-operator.sh b/.github/bin/minishift/test-stable-operator-update.sh similarity index 100% rename from .github/bin/minishift/test-update-operator.sh rename to .github/bin/minishift/test-stable-operator-update.sh diff --git a/.github/workflows/minikube-latest-olm.yaml b/.github/workflows/minikube-olm.yaml similarity index 94% rename from .github/workflows/minikube-latest-olm.yaml rename to .github/workflows/minikube-olm.yaml index 771e472d2..ee9f13f3b 100644 --- a/.github/workflows/minikube-latest-olm.yaml +++ b/.github/workflows/minikube-olm.yaml @@ -25,7 +25,7 @@ jobs: - name: Install jq run: sudo pip install yq - name: Run tests - run: /bin/bash .github/bin/minikube/test-latest-olm.sh + run: /bin/bash .github/bin/minikube/test-olm.sh # Run this step even the previous step fail - uses: actions/upload-artifact@v2 if: ${{ always() }} diff --git a/.github/workflows/minikube-latest-operator-singlehost.yaml b/.github/workflows/minikube-operator-singlehost.yaml similarity index 91% rename from .github/workflows/minikube-latest-operator-singlehost.yaml rename to .github/workflows/minikube-operator-singlehost.yaml index 894375ae4..eb7be4740 100644 --- a/.github/workflows/minikube-latest-operator-singlehost.yaml +++ b/.github/workflows/minikube-operator-singlehost.yaml @@ -24,7 +24,7 @@ jobs: - name: Install jq run: sudo pip install yq - name: Run tests - run: /bin/bash .github/bin/minikube/test-latest-operator-singlehost-gateway.sh + run: /bin/bash .github/bin/minikube/test-operator-singlehost-gateway.sh # Run this step even the previous step fail to upload artifacts to GH - uses: actions/upload-artifact@v2 if: ${{ always() }} @@ -43,7 +43,7 @@ jobs: - name: Install jq run: sudo pip install yq - name: Run tests - run: /bin/bash .github/bin/minikube/test-latest-operator-singlehost-native.sh + run: /bin/bash .github/bin/minikube/test-operator-singlehost-native.sh # Run this step even the previous step fail to upload artifacts to GH - uses: actions/upload-artifact@v2 if: ${{ always() }} diff --git a/.github/workflows/minikube-operator-update.yaml b/.github/workflows/minikube-operator-update.yaml new file mode 100644 index 000000000..e22297610 --- /dev/null +++ b/.github/workflows/minikube-operator-update.yaml @@ -0,0 +1,37 @@ +# +# Copyright (c) 2012-2020 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +# Contributors: +# Red Hat, Inc. - initial API and implementation +# + +name: Minikube +on: + pull_request: + branches: + - master +jobs: + minikube: + name: Test Eclipse Che update from stable to nightly + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Provision Minikube cluster + run: /bin/bash .github/bin/minikube/provision-cluster.sh + - name: Install chectl + run: bash <(curl -sL https://www.eclipse.org/che/chectl/) --channel=stable + - name: Install jq + run: sudo pip install yq + - name: Tests + run: /bin/bash .github/bin/minikube/test-operator-update.sh + # Run this step even the previous step fail + - uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: minikube-updates-artifacts + path: /tmp/artifacts-che diff --git a/.github/workflows/minikube-stable-updates.yaml b/.github/workflows/minikube-stable-olm-update.yaml similarity index 93% rename from .github/workflows/minikube-stable-updates.yaml rename to .github/workflows/minikube-stable-olm-update.yaml index e7e9e79b9..d32d1381f 100644 --- a/.github/workflows/minikube-stable-updates.yaml +++ b/.github/workflows/minikube-stable-olm-update.yaml @@ -28,7 +28,7 @@ jobs: - name: Install jq run: sudo pip install yq - name: Run tests - run: /bin/bash .github/bin/minikube/test-update-olm.sh + run: /bin/bash .github/bin/minikube/test-stable-olm-update.sh # Run this step even the previous step fail - uses: actions/upload-artifact@v2 if: ${{ always() }} diff --git a/.github/workflows/minishift-operator-update.yaml b/.github/workflows/minishift-operator-update.yaml new file mode 100644 index 000000000..077ebbd3f --- /dev/null +++ b/.github/workflows/minishift-operator-update.yaml @@ -0,0 +1,61 @@ +# +# Copyright (c) 2012-2020 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +# Contributors: +# Red Hat, Inc. - initial API and implementation +# + +name: Minishift +on: + pull_request: + branches: + - master +jobs: + minishift: + name: Test Eclipse Che update from stable to nightly + runs-on: macos-latest + steps: + - uses: actions/checkout@v1 + - name: Build Che operator image + run: | + export OPERATOR_IMAGE=quay.io/eclipse/che-operator:test + # coreutils provides a readlink that supports `-f` + + brew install coreutils docker docker-machine + + mkdir -p ~/.docker/machine/cache/ + sudo curl -Lo ~/.docker/machine/cache/boot2docker.iso https://github.com/boot2docker/boot2docker/releases/download/v19.03.12/boot2docker.iso + + docker-machine --github-api-token="${{ secrets.GITHUB_TOKEN }}" create --driver virtualbox default + eval "$(docker-machine env default)" + export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH" + + docker build -t "${OPERATOR_IMAGE}" -f Dockerfile . && docker save "${OPERATOR_IMAGE}" > operator.tar + + docker-machine stop + - name: Provision Minishift cluster + run: | + brew cask install minishift + export MINISHIFT_GITHUB_API_TOKEN=${{ secrets.GITHUB_TOKEN }} + minishift start --memory=5500 --vm-driver=virtualbox + - name: Install chectl + run: bash <(curl -sL https://www.eclipse.org/che/chectl/) --channel=stable + - name: Install jq + run: sudo pip install yq + - name: Replace Minishift default certificates + run: /bin/bash .github/bin/minishift/certs.sh + - name: Tests + run: | + eval $(minishift oc-env) + /bin/bash .github/bin/minishift/test-operator-update.sh + # Run this step even the previous step fail + - uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: minishift-updates-artifacts + path: /tmp/artifacts-che diff --git a/.github/workflows/minishift-latest.yaml b/.github/workflows/minishift-operator.yaml similarity index 97% rename from .github/workflows/minishift-latest.yaml rename to .github/workflows/minishift-operator.yaml index a48bc473d..d03b672f4 100644 --- a/.github/workflows/minishift-latest.yaml +++ b/.github/workflows/minishift-operator.yaml @@ -49,7 +49,7 @@ jobs: - name: Run tests run: | eval $(minishift oc-env) - /bin/bash .github/bin/minishift/test-latest-operator.sh + /bin/bash .github/bin/minishift/test-operator.sh # Run this step even the previous step fail - uses: actions/upload-artifact@v2 if: ${{ always() }} diff --git a/.github/workflows/minishift-stable-updates.yaml b/.github/workflows/minishift-stable-operator-update.yaml similarity index 94% rename from .github/workflows/minishift-stable-updates.yaml rename to .github/workflows/minishift-stable-operator-update.yaml index b8cb56426..afc6fad18 100644 --- a/.github/workflows/minishift-stable-updates.yaml +++ b/.github/workflows/minishift-stable-operator-update.yaml @@ -35,7 +35,7 @@ jobs: - name: Update minishift deprecated certificates and run e2e run: | eval $(minishift oc-env) - /bin/bash .github/bin/minishift/test-update-operator.sh + /bin/bash .github/bin/minishift/test-stable-operator-update.sh # Run this step even the previous step fail - uses: actions/upload-artifact@v2 if: ${{ always() }} diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 3241b709c..496bd0bcc 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -35,6 +35,7 @@ import ( sdkVersion "github.com/operator-framework/operator-sdk/version" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" ) @@ -103,9 +104,6 @@ func main() { os.Exit(1) } - // Become the leader before proceeding - leader.Become(context.TODO(), "che-operator-lock") - r := ready.NewFileReady() err = r.Set() if err != nil { @@ -114,10 +112,15 @@ func main() { } defer r.Unset() + // Become the leader before proceeding + leader.Become(context.TODO(), "che-operator-lock") + // Create a new Cmd to provide shared dependencies and start components options := manager.Options{ - Namespace: namespace, + Namespace: namespace, + HealthProbeBindAddress: ":6789", } + mgr, err := manager.New(cfg, options) if err != nil { log.Error(err, "") @@ -158,6 +161,12 @@ func main() { os.Exit(1) } + // Setup health checks + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + log.Error(err, "Unable to set up health check") + os.Exit(1) + } + logrus.Info("Starting the Cmd") // Start the Cmd diff --git a/deploy/olm-catalog/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml b/deploy/olm-catalog/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml index 1e23a509d..49aad21d2 100644 --- a/deploy/olm-catalog/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml +++ b/deploy/olm-catalog/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml @@ -84,13 +84,13 @@ metadata: categories: Developer Tools certified: "false" containerImage: quay.io/eclipse/che-operator:nightly - createdAt: "2021-01-08T12:41:34Z" + createdAt: "2021-01-08T14:09:35Z" description: A Kube-native development solution that delivers portable and collaborative developer workspaces. operatorframework.io/suggested-namespace: eclipse-che repository: https://github.com/eclipse/che-operator support: Eclipse Foundation - name: eclipse-che-preview-kubernetes.v7.24.0-64.nightly + name: eclipse-che-preview-kubernetes.v7.25.0-66.nightly namespace: placeholder spec: apiservicedefinitions: {} @@ -291,7 +291,8 @@ spec: selector: matchLabels: app: che-operator - strategy: {} + strategy: + type: RollingUpdate template: metadata: labels: @@ -357,10 +358,29 @@ spec: value: ca-certs image: quay.io/eclipse/che-operator:nightly imagePullPolicy: Always + livenessProbe: + failureThreshold: 10 + httpGet: + path: /healthz + port: 6789 + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 name: che-operator ports: - containerPort: 60000 name: metrics + readinessProbe: + exec: + command: + - stat + - /tmp/operator-sdk-ready + failureThreshold: 10 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 resources: {} restartPolicy: Always serviceAccountName: che-operator @@ -494,4 +514,4 @@ spec: maturity: stable provider: name: Eclipse Foundation - version: 7.24.0-64.nightly + version: 7.25.0-66.nightly diff --git a/deploy/olm-catalog/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml b/deploy/olm-catalog/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml index 7887111ad..d4e0074f9 100644 --- a/deploy/olm-catalog/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml +++ b/deploy/olm-catalog/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml @@ -75,13 +75,13 @@ metadata: categories: Developer Tools, OpenShift Optional certified: "false" containerImage: quay.io/eclipse/che-operator:nightly - createdAt: "2021-01-08T12:41:46Z" + createdAt: "2021-01-08T14:09:42Z" description: A Kube-native development solution that delivers portable and collaborative developer workspaces in OpenShift. operatorframework.io/suggested-namespace: eclipse-che repository: https://github.com/eclipse/che-operator support: Eclipse Foundation - name: eclipse-che-preview-openshift.v7.24.0-64.nightly + name: eclipse-che-preview-openshift.v7.25.0-66.nightly namespace: placeholder spec: apiservicedefinitions: {} @@ -306,7 +306,8 @@ spec: selector: matchLabels: app: che-operator - strategy: {} + strategy: + type: RollingUpdate template: metadata: labels: @@ -370,10 +371,29 @@ spec: value: ca-certs image: quay.io/eclipse/che-operator:nightly imagePullPolicy: Always + livenessProbe: + failureThreshold: 10 + httpGet: + path: /healthz + port: 6789 + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 name: che-operator ports: - containerPort: 60000 name: metrics + readinessProbe: + exec: + command: + - stat + - /tmp/operator-sdk-ready + failureThreshold: 10 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 resources: {} restartPolicy: Always serviceAccountName: che-operator @@ -513,4 +533,4 @@ spec: maturity: stable provider: name: Eclipse Foundation - version: 7.24.0-64.nightly + version: 7.25.0-66.nightly diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 130d0f4ff..68ecde49b 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -17,6 +17,8 @@ spec: selector: matchLabels: app: che-operator + strategy: + type: RollingUpdate template: metadata: labels: @@ -86,6 +88,25 @@ spec: value: che-postgres-secret - name: CHE_SERVER_TRUST_STORE_CONFIGMAP_NAME value: ca-certs + livenessProbe: + httpGet: + path: /healthz + port: 6789 + initialDelaySeconds: 15 + periodSeconds: 10 + failureThreshold: 10 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - stat + - /tmp/operator-sdk-ready + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 10 + successThreshold: 1 + timeoutSeconds: 5 restartPolicy: Always serviceAccountName: che-operator terminationGracePeriodSeconds: 5