From 39b7a1e7b83ca72c7d1a8cf4b7bb0ced80bc71a6 Mon Sep 17 00:00:00 2001 From: Anatolii Bazko Date: Wed, 30 Dec 2020 12:51:46 +0200 Subject: [PATCH] Add liveness and readiness probes (#598) Signed-off-by: Anatolii Bazko --- cmd/manager/main.go | 17 ++++++++++++- .../che-operator.clusterserviceversion.yaml | 24 ++++++++++++++++--- .../che-operator.clusterserviceversion.yaml | 24 ++++++++++++++++--- deploy/operator-local.yaml | 18 ++++++++++++++ deploy/operator.yaml | 18 ++++++++++++++ .../identity-provider/deployment_keycloak.go | 12 ++++++++++ pkg/deploy/postgres/deployment_postgres.go | 13 ++++++++++ pkg/deploy/registry/registry.go | 2 +- 8 files changed, 120 insertions(+), 8 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 62f0eeea9..df02a4457 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -35,6 +35,7 @@ import ( sdkVersion "github.com/operator-framework/operator-sdk/version" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" ) @@ -115,7 +116,11 @@ func main() { defer r.Unset() // Create a new Cmd to provide shared dependencies and start components - mgr, err := manager.New(cfg, manager.Options{Namespace: namespace}) + options := manager.Options{ + Namespace: namespace, + HealthProbeBindAddress: ":6789", + } + mgr, err := manager.New(cfg, options) if err != nil { log.Error(err, "") os.Exit(1) @@ -155,6 +160,16 @@ func main() { os.Exit(1) } + // Setup health checks + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + log.Error(err, "Unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + log.Error(err, "Unable to set up ready check") + os.Exit(1) + } + logrus.Info("Starting the Cmd") // Start the Cmd diff --git a/deploy/olm-catalog/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml b/deploy/olm-catalog/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml index 3654c936d..d738dd8f4 100644 --- a/deploy/olm-catalog/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml +++ b/deploy/olm-catalog/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml @@ -84,13 +84,13 @@ metadata: categories: Developer Tools certified: "false" containerImage: quay.io/eclipse/che-operator:nightly - createdAt: "2020-12-17T10:09:13Z" + createdAt: "2020-12-29T12:54:43Z" description: A Kube-native development solution that delivers portable and collaborative developer workspaces. operatorframework.io/suggested-namespace: eclipse-che repository: https://github.com/eclipse/che-operator support: Eclipse Foundation - name: eclipse-che-preview-kubernetes.v7.24.0-52.nightly + name: eclipse-che-preview-kubernetes.v7.24.0-56.nightly namespace: placeholder spec: apiservicedefinitions: {} @@ -357,10 +357,28 @@ spec: value: ca-certs image: quay.io/eclipse/che-operator:nightly imagePullPolicy: Always + livenessProbe: + failureThreshold: 10 + httpGet: + path: /readyz + port: 6789 + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 name: che-operator ports: - containerPort: 60000 name: metrics + readinessProbe: + failureThreshold: 10 + httpGet: + path: /healthz + port: 6789 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 resources: {} restartPolicy: Always serviceAccountName: che-operator @@ -494,4 +512,4 @@ spec: maturity: stable provider: name: Eclipse Foundation - version: 7.24.0-52.nightly + version: 7.24.0-56.nightly diff --git a/deploy/olm-catalog/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml b/deploy/olm-catalog/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml index e2e8f6c7b..cbeef4bdb 100644 --- a/deploy/olm-catalog/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml +++ b/deploy/olm-catalog/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml @@ -75,13 +75,13 @@ metadata: categories: Developer Tools, OpenShift Optional certified: "false" containerImage: quay.io/eclipse/che-operator:nightly - createdAt: "2020-12-17T10:09:20Z" + createdAt: "2020-12-29T12:54:51Z" description: A Kube-native development solution that delivers portable and collaborative developer workspaces in OpenShift. operatorframework.io/suggested-namespace: eclipse-che repository: https://github.com/eclipse/che-operator support: Eclipse Foundation - name: eclipse-che-preview-openshift.v7.24.0-52.nightly + name: eclipse-che-preview-openshift.v7.24.0-54.nightly namespace: placeholder spec: apiservicedefinitions: {} @@ -370,10 +370,28 @@ spec: value: ca-certs image: quay.io/eclipse/che-operator:nightly imagePullPolicy: Always + livenessProbe: + failureThreshold: 10 + httpGet: + path: /readyz + port: 6789 + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 name: che-operator ports: - containerPort: 60000 name: metrics + readinessProbe: + failureThreshold: 10 + httpGet: + path: /healthz + port: 6789 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 resources: {} restartPolicy: Always serviceAccountName: che-operator @@ -513,4 +531,4 @@ spec: maturity: stable provider: name: Eclipse Foundation - version: 7.24.0-52.nightly + version: 7.24.0-54.nightly diff --git a/deploy/operator-local.yaml b/deploy/operator-local.yaml index 6c92fcee1..a8162d218 100644 --- a/deploy/operator-local.yaml +++ b/deploy/operator-local.yaml @@ -87,3 +87,21 @@ spec: value: che-postgres-secret - name: CHE_SERVER_TRUST_STORE_CONFIGMAP_NAME value: ca-certs + livenessProbe: + httpGet: + path: /readyz + port: 6789 + initialDelaySeconds: 15 + periodSeconds: 10 + failureThreshold: 10 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /healthz + port: 6789 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 10 + successThreshold: 1 + timeoutSeconds: 5 diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 130d0f4ff..4c5047252 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -86,6 +86,24 @@ spec: value: che-postgres-secret - name: CHE_SERVER_TRUST_STORE_CONFIGMAP_NAME value: ca-certs + livenessProbe: + httpGet: + path: /readyz + port: 6789 + initialDelaySeconds: 15 + periodSeconds: 10 + failureThreshold: 10 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /healthz + port: 6789 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 10 + successThreshold: 1 + timeoutSeconds: 5 restartPolicy: Always serviceAccountName: che-operator terminationGracePeriodSeconds: 5 diff --git a/pkg/deploy/identity-provider/deployment_keycloak.go b/pkg/deploy/identity-provider/deployment_keycloak.go index a5a3bd4b9..98cddc5b2 100644 --- a/pkg/deploy/identity-provider/deployment_keycloak.go +++ b/pkg/deploy/identity-provider/deployment_keycloak.go @@ -616,6 +616,18 @@ func getSpecKeycloakDeployment( PeriodSeconds: 10, SuccessThreshold: 1, }, + LivenessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(8080), + }, + }, + InitialDelaySeconds: 30, + FailureThreshold: 10, + TimeoutSeconds: 5, + PeriodSeconds: 10, + SuccessThreshold: 1, + }, Env: keycloakEnv, VolumeMounts: []corev1.VolumeMount{ customPublicCertsVolumeMount, diff --git a/pkg/deploy/postgres/deployment_postgres.go b/pkg/deploy/postgres/deployment_postgres.go index 3ba8c8a44..198d8a866 100644 --- a/pkg/deploy/postgres/deployment_postgres.go +++ b/pkg/deploy/postgres/deployment_postgres.go @@ -18,6 +18,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -142,6 +143,18 @@ func getSpecPostgresDeployment(deployContext *deploy.DeployContext, clusterDeplo PeriodSeconds: 10, TimeoutSeconds: 5, }, + LivenessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(5432), + }, + }, + InitialDelaySeconds: 30, + FailureThreshold: 10, + SuccessThreshold: 1, + PeriodSeconds: 10, + TimeoutSeconds: 5, + }, Env: []corev1.EnvVar{ { Name: "POSTGRESQL_DATABASE", diff --git a/pkg/deploy/registry/registry.go b/pkg/deploy/registry/registry.go index fa7f7abe1..38497c178 100644 --- a/pkg/deploy/registry/registry.go +++ b/pkg/deploy/registry/registry.go @@ -99,8 +99,8 @@ func GetSpecRegistryDeployment( InitialDelaySeconds: 3, FailureThreshold: 10, TimeoutSeconds: 3, - PeriodSeconds: 10, SuccessThreshold: 1, + PeriodSeconds: 10, }, LivenessProbe: &v1.Probe{ Handler: v1.Handler{