From 80583a19c02cd6335b109db80825cf0d0763a98e Mon Sep 17 00:00:00 2001 From: Mykola Morhun Date: Wed, 13 Oct 2021 16:15:11 +0300 Subject: [PATCH] Add init containers to start Che correctly after node restart (#1139) Signed-off-by: Mykola Morhun --- Makefile | 2 +- .../che-operator.clusterserviceversion.yaml | 2 + .../metadata/annotations.yaml | 2 +- .../che-operator.clusterserviceversion.yaml | 2 + .../che-operator.clusterserviceversion.yaml | 2 + config/manager/manager.yaml | 2 + controllers/che/checluster_controller.go | 2 +- pkg/deploy/defaults.go | 4 ++ .../identity-provider/deployment_keycloak.go | 11 ++++ pkg/deploy/identity-provider/exec.go | 11 ---- .../identity-provider/keycloak_readiness.go | 56 ++++++++++++++++++ pkg/deploy/postgres/postgres.go | 14 ++++- pkg/deploy/postgres/postgres_readiness.go | 58 +++++++++++++++++++ pkg/deploy/server/server_deployment.go | 20 +++++++ 14 files changed, 172 insertions(+), 16 deletions(-) create mode 100644 pkg/deploy/identity-provider/keycloak_readiness.go create mode 100644 pkg/deploy/postgres/postgres_readiness.go diff --git a/Makefile b/Makefile index 1d329baf0..83b3eb2b8 100644 --- a/Makefile +++ b/Makefile @@ -676,7 +676,7 @@ bundle: generate manifests kustomize ## Generate bundle manifests and metadata, yq -riY ".metadata.annotations[\"alm-examples\"] = \"$${CSV_CR_SAMPLES}\"" $${NEW_CSV} # Set specific OpenShift version - echo "\n com.redhat.openshift.versions: \"v4.8\"" >> $${BUNDLE_PATH}/metadata/annotations.yaml + printf "\n com.redhat.openshift.versions: \"v4.8\"" >> $${BUNDLE_PATH}/metadata/annotations.yaml fi # Base cluster service version file has got correctly sorted CRDs. diff --git a/bundle/next-all-namespaces/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml b/bundle/next-all-namespaces/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml index b1b835d98..59c4985d4 100644 --- a/bundle/next-all-namespaces/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml +++ b/bundle/next-all-namespaces/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml @@ -1133,6 +1133,8 @@ spec: value: "1" - name: ALLOW_DEVWORKSPACE_ENGINE value: "true" + - name: ADD_COMPONENT_READINESS_INIT_CONTAINERS + value: "false" image: quay.io/eclipse/che-operator:next imagePullPolicy: Always livenessProbe: diff --git a/bundle/next-all-namespaces/eclipse-che-preview-openshift/metadata/annotations.yaml b/bundle/next-all-namespaces/eclipse-che-preview-openshift/metadata/annotations.yaml index f8882e1be..5f1e1be49 100644 --- a/bundle/next-all-namespaces/eclipse-che-preview-openshift/metadata/annotations.yaml +++ b/bundle/next-all-namespaces/eclipse-che-preview-openshift/metadata/annotations.yaml @@ -14,4 +14,4 @@ annotations: operators.operatorframework.io.test.mediatype.v1: scorecard+v1 operators.operatorframework.io.test.config.v1: tests/scorecard/ - com.redhat.openshift.versions: "v4.8" + com.redhat.openshift.versions: "v4.8" \ No newline at end of file diff --git a/bundle/next/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml b/bundle/next/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml index e374fb464..5852a7c61 100644 --- a/bundle/next/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml +++ b/bundle/next/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml @@ -1122,6 +1122,8 @@ spec: value: "1" - name: ALLOW_DEVWORKSPACE_ENGINE value: "true" + - name: ADD_COMPONENT_READINESS_INIT_CONTAINERS + value: "false" image: quay.io/eclipse/che-operator:next imagePullPolicy: Always livenessProbe: diff --git a/bundle/next/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml b/bundle/next/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml index c0a94ad78..23e906398 100644 --- a/bundle/next/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml +++ b/bundle/next/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml @@ -1133,6 +1133,8 @@ spec: value: "1" - name: ALLOW_DEVWORKSPACE_ENGINE value: "true" + - name: ADD_COMPONENT_READINESS_INIT_CONTAINERS + value: "false" image: quay.io/eclipse/che-operator:next imagePullPolicy: Always livenessProbe: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 4c2cb968f..e9204fae8 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -117,6 +117,8 @@ spec: value: "1" - name: ALLOW_DEVWORKSPACE_ENGINE value: "true" + - name: ADD_COMPONENT_READINESS_INIT_CONTAINERS + value: "false" livenessProbe: httpGet: path: /healthz diff --git a/controllers/che/checluster_controller.go b/controllers/che/checluster_controller.go index 9cc8c396d..3cf94a96a 100644 --- a/controllers/che/checluster_controller.go +++ b/controllers/che/checluster_controller.go @@ -270,7 +270,7 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } return ctrl.Result{}, err } - if backupCR.Status.State == orgv1.STATE_IN_PROGRESS { + if backupCR.Status.State == orgv1.STATE_IN_PROGRESS || backupCR.Status.State == "" { // Backup is still in progress return ctrl.Result{RequeueAfter: time.Second * 5}, nil } diff --git a/pkg/deploy/defaults.go b/pkg/deploy/defaults.go index 19574df50..a6d1c769d 100644 --- a/pkg/deploy/defaults.go +++ b/pkg/deploy/defaults.go @@ -241,6 +241,10 @@ func MigratingToCRW2_0(cr *orgv1.CheCluster) bool { return false } +func IsComponentReadinessInitContainersConfigured(cr *orgv1.CheCluster) bool { + return os.Getenv("ADD_COMPONENT_READINESS_INIT_CONTAINERS") == "true" +} + func DefaultServerTrustStoreConfigMapName() string { return getDefaultFromEnv("CHE_SERVER_TRUST_STORE_CONFIGMAP_NAME") } diff --git a/pkg/deploy/identity-provider/deployment_keycloak.go b/pkg/deploy/identity-provider/deployment_keycloak.go index 9d77b9886..5c8a72161 100644 --- a/pkg/deploy/identity-provider/deployment_keycloak.go +++ b/pkg/deploy/identity-provider/deployment_keycloak.go @@ -19,6 +19,7 @@ import ( orgv1 "github.com/eclipse-che/che-operator/api/v1" "github.com/eclipse-che/che-operator/pkg/deploy" + "github.com/eclipse-che/che-operator/pkg/deploy/postgres" "github.com/eclipse-che/che-operator/pkg/util" "github.com/google/go-cmp/cmp" "github.com/sirupsen/logrus" @@ -716,6 +717,16 @@ func GetSpecKeycloakDeployment( }, } + if deploy.IsComponentReadinessInitContainersConfigured(deployContext.CheCluster) { + if !deployContext.CheCluster.Spec.Database.ExternalDb { + waitForPostgresInitContainer, err := postgres.GetWaitForPostgresInitContainer(deployContext) + if err != nil { + return nil, err + } + deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, *waitForPostgresInitContainer) + } + } + return deployment, nil } diff --git a/pkg/deploy/identity-provider/exec.go b/pkg/deploy/identity-provider/exec.go index e4e14dbc7..cbe01bc83 100644 --- a/pkg/deploy/identity-provider/exec.go +++ b/pkg/deploy/identity-provider/exec.go @@ -22,17 +22,6 @@ import ( "github.com/sirupsen/logrus" ) -func GetPostgresProvisionCommand(identityProviderPostgresPassword string) (command string) { - command = "OUT=$(psql postgres -tAc \"SELECT 1 FROM pg_roles WHERE rolname='keycloak'\"); " + - "if [ $OUT -eq 1 ]; then echo \"DB exists\"; exit 0; fi " + - "&& psql -c \"CREATE USER keycloak WITH PASSWORD '" + identityProviderPostgresPassword + "'\" " + - "&& psql -c \"CREATE DATABASE keycloak\" " + - "&& psql -c \"GRANT ALL PRIVILEGES ON DATABASE keycloak TO keycloak\" " + - "&& psql -c \"ALTER USER ${POSTGRESQL_USER} WITH SUPERUSER\"" - - return command -} - func GetKeycloakProvisionCommand(cr *v1.CheCluster) (command string, err error) { cheFlavor := deploy.DefaultCheFlavor(cr) requiredActions := (map[bool]string{true: "\"UPDATE_PASSWORD\"", false: ""})[cr.Spec.Auth.UpdateAdminPassword] diff --git a/pkg/deploy/identity-provider/keycloak_readiness.go b/pkg/deploy/identity-provider/keycloak_readiness.go new file mode 100644 index 000000000..bc5eb0e77 --- /dev/null +++ b/pkg/deploy/identity-provider/keycloak_readiness.go @@ -0,0 +1,56 @@ +// +// Copyright (c) 2021 Red Hat, Inc. +// This program and the accompanying materials are made +// available under the terms of the Eclipse Public License 2.0 +// which is available at https://www.eclipse.org/legal/epl-2.0/ +// +// SPDX-License-Identifier: EPL-2.0 +// +// Contributors: +// Red Hat, Inc. - initial API and implementation +// +package identity_provider + +import ( + "fmt" + + "github.com/eclipse-che/che-operator/pkg/deploy" + "github.com/eclipse-che/che-operator/pkg/util" + corev1 "k8s.io/api/core/v1" +) + +func GetWaitForKeycloakInitContainer(deployContext *deploy.DeployContext) (*corev1.Container, error) { + keycloakReadinessCheckerImage := deploy.DefaultKeycloakImage(deployContext.CheCluster) + imagePullPolicy := corev1.PullPolicy(deploy.DefaultPullPolicyFromDockerImage(keycloakReadinessCheckerImage)) + + return &corev1.Container{ + Name: "wait-for-identity-provider", + Image: keycloakReadinessCheckerImage, + ImagePullPolicy: imagePullPolicy, + Command: []string{ + "/bin/sh", + "-c", + getCheckKeycloakReadinessScript(deployContext), + }, + }, nil +} + +func getCheckKeycloakReadinessScript(deployContext *deploy.DeployContext) string { + cheFlavor := deploy.DefaultCheFlavor(deployContext.CheCluster) + realmName := util.GetValue(deployContext.CheCluster.Spec.Auth.IdentityProviderRealm, cheFlavor) + url := fmt.Sprintf("%s/realms/%s/.well-known/openid-configuration", deployContext.CheCluster.Status.KeycloakURL, realmName) + // URL example: https://keycloak-eclipse-che.192.168.99.254.nip.io/auth/realms/che/.well-known/openid-configuration + + script := ` + while : ; do + response_code=$(curl --connect-timeout 5 -kI %s 2>/dev/null | awk 'NR==1 {print $2}') + if [ "$response_code" == "200" ]; then + break + fi + echo 'waiting for Identity provider' + sleep 2 + done + ` + + return fmt.Sprintf(script, url) +} diff --git a/pkg/deploy/postgres/postgres.go b/pkg/deploy/postgres/postgres.go index 8872ed3ca..44f54d116 100644 --- a/pkg/deploy/postgres/postgres.go +++ b/pkg/deploy/postgres/postgres.go @@ -17,7 +17,6 @@ import ( orgv1 "github.com/eclipse-che/che-operator/api/v1" "github.com/eclipse-che/che-operator/pkg/deploy" - identity_provider "github.com/eclipse-che/che-operator/pkg/deploy/identity-provider" "github.com/eclipse-che/che-operator/pkg/util" "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" @@ -122,7 +121,7 @@ func (p *Postgres) ProvisionDB() (bool, error) { p.deployContext.CheCluster, deploy.PostgresName, func(cr *orgv1.CheCluster) (string, error) { - return identity_provider.GetPostgresProvisionCommand(identityProviderPostgresPassword), nil + return getPostgresProvisionCommand(identityProviderPostgresPassword), nil }, "create Keycloak DB, user, privileges") if err != nil { @@ -160,3 +159,14 @@ func (p *Postgres) setDbVersion() (bool, error) { return true, nil } + +func getPostgresProvisionCommand(identityProviderPostgresPassword string) (command string) { + command = "OUT=$(psql postgres -tAc \"SELECT 1 FROM pg_roles WHERE rolname='keycloak'\"); " + + "if [ $OUT -eq 1 ]; then echo \"DB exists\"; exit 0; fi " + + "&& psql -c \"CREATE USER keycloak WITH PASSWORD '" + identityProviderPostgresPassword + "'\" " + + "&& psql -c \"CREATE DATABASE keycloak\" " + + "&& psql -c \"GRANT ALL PRIVILEGES ON DATABASE keycloak TO keycloak\" " + + "&& psql -c \"ALTER USER ${POSTGRESQL_USER} WITH SUPERUSER\"" + + return command +} diff --git a/pkg/deploy/postgres/postgres_readiness.go b/pkg/deploy/postgres/postgres_readiness.go new file mode 100644 index 000000000..6e590dc84 --- /dev/null +++ b/pkg/deploy/postgres/postgres_readiness.go @@ -0,0 +1,58 @@ +// +// Copyright (c) 2021 Red Hat, Inc. +// This program and the accompanying materials are made +// available under the terms of the Eclipse Public License 2.0 +// which is available at https://www.eclipse.org/legal/epl-2.0/ +// +// SPDX-License-Identifier: EPL-2.0 +// +// Contributors: +// Red Hat, Inc. - initial API and implementation +// +package postgres + +import ( + "fmt" + + "github.com/eclipse-che/che-operator/pkg/deploy" + "github.com/eclipse-che/che-operator/pkg/util" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" +) + +func GetWaitForPostgresInitContainer(deployContext *deploy.DeployContext) (*corev1.Container, error) { + postgresDeployment := &appsv1.Deployment{} + exists, err := deploy.GetNamespacedObject(deployContext, deploy.PostgresName, postgresDeployment) + if err != nil { + return nil, err + } + if !exists { + postgresDeployment = nil + } + postgresReadinessCheckerImage, err := getPostgresImage(postgresDeployment, deployContext.CheCluster) + if err != nil { + return nil, err + } + imagePullPolicy := corev1.PullPolicy(deploy.DefaultPullPolicyFromDockerImage(postgresReadinessCheckerImage)) + + return &corev1.Container{ + Name: "wait-for-postgres", + Image: postgresReadinessCheckerImage, + ImagePullPolicy: imagePullPolicy, + Command: []string{ + "/bin/sh", + "-c", + getCheckPostgresReadinessScript(deployContext), + }, + }, nil +} + +func getCheckPostgresReadinessScript(deployContext *deploy.DeployContext) string { + chePostgresHostName := util.GetValue(deployContext.CheCluster.Spec.Database.ChePostgresHostName, deploy.DefaultChePostgresHostName) + chePostgresPort := util.GetValue(deployContext.CheCluster.Spec.Database.ChePostgresPort, deploy.DefaultChePostgresPort) + + return fmt.Sprintf( + "until pg_isready -h %s -p %s; do echo 'waiting for Postgres'; sleep 2; done;", + chePostgresHostName, + chePostgresPort) +} diff --git a/pkg/deploy/server/server_deployment.go b/pkg/deploy/server/server_deployment.go index 9e93c7bc6..4b08d5b18 100644 --- a/pkg/deploy/server/server_deployment.go +++ b/pkg/deploy/server/server_deployment.go @@ -17,6 +17,8 @@ import ( "strings" "github.com/eclipse-che/che-operator/pkg/deploy" + identity_provider "github.com/eclipse-che/che-operator/pkg/deploy/identity-provider" + "github.com/eclipse-che/che-operator/pkg/deploy/postgres" orgv1 "github.com/eclipse-che/che-operator/api/v1" "github.com/eclipse-che/che-operator/pkg/util" @@ -358,6 +360,24 @@ func (s Server) getDeploymentSpec() (*appsv1.Deployment, error) { } } + if deploy.IsComponentReadinessInitContainersConfigured(s.deployContext.CheCluster) { + if !s.deployContext.CheCluster.Spec.Database.ExternalDb { + waitForPostgresInitContainer, err := postgres.GetWaitForPostgresInitContainer(s.deployContext) + if err != nil { + return nil, err + } + deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, *waitForPostgresInitContainer) + } + + if !s.deployContext.CheCluster.Spec.Auth.ExternalIdentityProvider { + waitForKeycloakInitContainer, err := identity_provider.GetWaitForKeycloakInitContainer(s.deployContext) + if err != nil { + return nil, err + } + deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, *waitForKeycloakInitContainer) + } + } + return deployment, nil }