Add liveness and readiness probes (#598)

Signed-off-by: Anatolii Bazko <abazko@redhat.com>

cmd/manager/main.go

@@ -35,6 +35,7 @@ import (
 	sdkVersion "github.com/operator-framework/operator-sdk/version"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 )
@@ -115,7 +116,11 @@ func main() {
 	defer r.Unset()
 
 	// Create a new Cmd to provide shared dependencies and start components
-	mgr, err := manager.New(cfg, manager.Options{Namespace: namespace})
+	options := manager.Options{
+		Namespace:              namespace,
+		HealthProbeBindAddress: ":6789",
+	}
+	mgr, err := manager.New(cfg, options)
 	if err != nil {
 		log.Error(err, "")
 		os.Exit(1)
@@ -155,6 +160,16 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Setup health checks
+	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
+		log.Error(err, "Unable to set up health check")
+		os.Exit(1)
+	}
+	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+		log.Error(err, "Unable to set up ready check")
+		os.Exit(1)
+	}
+
 	logrus.Info("Starting the Cmd")
 
 	// Start the Cmd

deploy/olm-catalog/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml

@@ -84,13 +84,13 @@ metadata:
     categories: Developer Tools
     certified: "false"
     containerImage: quay.io/eclipse/che-operator:nightly
-    createdAt: "2020-12-17T10:09:13Z"
+    createdAt: "2020-12-29T12:54:43Z"
     description: A Kube-native development solution that delivers portable and collaborative
       developer workspaces.
     operatorframework.io/suggested-namespace: eclipse-che
     repository: https://github.com/eclipse/che-operator
     support: Eclipse Foundation
-  name: eclipse-che-preview-kubernetes.v7.24.0-52.nightly
+  name: eclipse-che-preview-kubernetes.v7.24.0-56.nightly
   namespace: placeholder
 spec:
   apiservicedefinitions: {}
@@ -357,10 +357,28 @@ spec:
                         value: ca-certs
                     image: quay.io/eclipse/che-operator:nightly
                     imagePullPolicy: Always
+                    livenessProbe:
+                      failureThreshold: 10
+                      httpGet:
+                        path: /readyz
+                        port: 6789
+                      initialDelaySeconds: 15
+                      periodSeconds: 10
+                      successThreshold: 1
+                      timeoutSeconds: 5
                     name: che-operator
                     ports:
                       - containerPort: 60000
                         name: metrics
+                    readinessProbe:
+                      failureThreshold: 10
+                      httpGet:
+                        path: /healthz
+                        port: 6789
+                      initialDelaySeconds: 5
+                      periodSeconds: 10
+                      successThreshold: 1
+                      timeoutSeconds: 5
                     resources: {}
                 restartPolicy: Always
                 serviceAccountName: che-operator
@@ -494,4 +512,4 @@ spec:
   maturity: stable
   provider:
     name: Eclipse Foundation
-  version: 7.24.0-52.nightly
+  version: 7.24.0-56.nightly

deploy/olm-catalog/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml

@@ -75,13 +75,13 @@ metadata:
     categories: Developer Tools, OpenShift Optional
     certified: "false"
     containerImage: quay.io/eclipse/che-operator:nightly
-    createdAt: "2020-12-17T10:09:20Z"
+    createdAt: "2020-12-29T12:54:51Z"
     description: A Kube-native development solution that delivers portable and collaborative
       developer workspaces in OpenShift.
     operatorframework.io/suggested-namespace: eclipse-che
     repository: https://github.com/eclipse/che-operator
     support: Eclipse Foundation
-  name: eclipse-che-preview-openshift.v7.24.0-52.nightly
+  name: eclipse-che-preview-openshift.v7.24.0-54.nightly
   namespace: placeholder
 spec:
   apiservicedefinitions: {}
@@ -370,10 +370,28 @@ spec:
                         value: ca-certs
                     image: quay.io/eclipse/che-operator:nightly
                     imagePullPolicy: Always
+                    livenessProbe:
+                      failureThreshold: 10
+                      httpGet:
+                        path: /readyz
+                        port: 6789
+                      initialDelaySeconds: 15
+                      periodSeconds: 10
+                      successThreshold: 1
+                      timeoutSeconds: 5
                     name: che-operator
                     ports:
                       - containerPort: 60000
                         name: metrics
+                    readinessProbe:
+                      failureThreshold: 10
+                      httpGet:
+                        path: /healthz
+                        port: 6789
+                      initialDelaySeconds: 5
+                      periodSeconds: 10
+                      successThreshold: 1
+                      timeoutSeconds: 5
                     resources: {}
                 restartPolicy: Always
                 serviceAccountName: che-operator
@@ -513,4 +531,4 @@ spec:
   maturity: stable
   provider:
     name: Eclipse Foundation
-  version: 7.24.0-52.nightly
+  version: 7.24.0-54.nightly

deploy/operator-local.yaml

@@ -87,3 +87,21 @@ spec:
               value: che-postgres-secret
             - name: CHE_SERVER_TRUST_STORE_CONFIGMAP_NAME
               value: ca-certs
+          livenessProbe:
+            httpGet:
+              path: /readyz
+              port: 6789
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            failureThreshold: 10
+            successThreshold: 1
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: 6789
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            failureThreshold: 10
+            successThreshold: 1
+            timeoutSeconds: 5

deploy/operator.yaml

@@ -86,6 +86,24 @@ spec:
               value: che-postgres-secret
             - name: CHE_SERVER_TRUST_STORE_CONFIGMAP_NAME
               value: ca-certs
+          livenessProbe:
+            httpGet:
+              path: /readyz
+              port: 6789
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            failureThreshold: 10
+            successThreshold: 1
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: 6789
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            failureThreshold: 10
+            successThreshold: 1
+            timeoutSeconds: 5
       restartPolicy: Always
       serviceAccountName: che-operator
       terminationGracePeriodSeconds: 5

pkg/deploy/identity-provider/deployment_keycloak.go

@@ -616,6 +616,18 @@ func getSpecKeycloakDeployment(
 								PeriodSeconds:       10,
 								SuccessThreshold:    1,
 							},
+							LivenessProbe: &corev1.Probe{
+								Handler: corev1.Handler{
+									TCPSocket: &corev1.TCPSocketAction{
+										Port: intstr.FromInt(8080),
+									},
+								},
+								InitialDelaySeconds: 30,
+								FailureThreshold:    10,
+								TimeoutSeconds:      5,
+								PeriodSeconds:       10,
+								SuccessThreshold:    1,
+							},
 							Env: keycloakEnv,
 							VolumeMounts: []corev1.VolumeMount{
 								customPublicCertsVolumeMount,

pkg/deploy/postgres/deployment_postgres.go

@@ -18,6 +18,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
@@ -142,6 +143,18 @@ func getSpecPostgresDeployment(deployContext *deploy.DeployContext, clusterDeplo
 								PeriodSeconds:       10,
 								TimeoutSeconds:      5,
 							},
+							LivenessProbe: &corev1.Probe{
+								Handler: corev1.Handler{
+									TCPSocket: &corev1.TCPSocketAction{
+										Port: intstr.FromInt(5432),
+									},
+								},
+								InitialDelaySeconds: 30,
+								FailureThreshold:    10,
+								SuccessThreshold:    1,
+								PeriodSeconds:       10,
+								TimeoutSeconds:      5,
+							},
 							Env: []corev1.EnvVar{
 								{
 									Name:  "POSTGRESQL_DATABASE",

pkg/deploy/registry/registry.go

@@ -99,8 +99,8 @@ func GetSpecRegistryDeployment(
 								InitialDelaySeconds: 3,
 								FailureThreshold:    10,
 								TimeoutSeconds:      3,
-								PeriodSeconds:       10,
 								SuccessThreshold:    1,
+								PeriodSeconds:       10,
 							},
 							LivenessProbe: &v1.Probe{
 								Handler: v1.Handler{