From 0b63bdc8352396d2ae48287c9aaf83a684a97063 Mon Sep 17 00:00:00 2001 From: Anatolii Bazko Date: Wed, 22 Sep 2021 12:00:08 +0300 Subject: [PATCH] feat: Migrate to the new PostgreSQL 13.3 (#1062) * feat: Migrate to the new PostgreSQL 13.3 Signed-off-by: Anatolii Bazko --- api/v1/checluster_types.go | 4 + .../che-operator.clusterserviceversion.yaml | 2 + .../manifests/org_v1_che_crd.yaml | 3 + .../che-operator.clusterserviceversion.yaml | 2 + .../manifests/org_v1_che_crd.yaml | 5 + config/crd/bases/org_v1_che_crd-v1beta1.yaml | 5 + config/crd/bases/org_v1_che_crd.yaml | 5 + config/manager/manager.yaml | 2 + pkg/deploy/defaults.go | 15 ++ pkg/deploy/defaults_test.go | 5 + pkg/deploy/postgres/postgres.go | 33 ++++ pkg/deploy/postgres/postgres_deployment.go | 40 ++++- pkg/deploy/postgres/postgres_test.go | 141 ++++++++++++++++++ 13 files changed, 261 insertions(+), 1 deletion(-) diff --git a/api/v1/checluster_types.go b/api/v1/checluster_types.go index 38a2e2757..dd84db4ba 100644 --- a/api/v1/checluster_types.go +++ b/api/v1/checluster_types.go @@ -364,6 +364,10 @@ type CheClusterSpecDB struct { // Overrides the container image used in the PostgreSQL database deployment. This includes the image tag. Omit it or leave it empty to use the default container image provided by the Operator. // +optional PostgresImage string `json:"postgresImage,omitempty"` + // Indicates a PostgreSQL version image to use. Allowed values are: `9.6` and `13.3`. + // Migrate your PostgreSQL database to switch from one version to another. + // +optional + PostgresVersion string `json:"postgresVersion,omitempty"` // Overrides the image pull policy used in the PostgreSQL database deployment. Default value is `Always` for `nightly`, `next` or `latest` images, and `IfNotPresent` in other cases. // +optional PostgresImagePullPolicy corev1.PullPolicy `json:"postgresImagePullPolicy,omitempty"` diff --git a/bundle/next/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml b/bundle/next/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml index 128d85e78..a934a51f9 100644 --- a/bundle/next/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml +++ b/bundle/next/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml @@ -874,6 +874,8 @@ spec: value: registry.access.redhat.com/ubi8-minimal:8.4-210 - name: RELATED_IMAGE_postgres value: quay.io/eclipse/che--centos--postgresql-96-centos7:9.6-b681d78125361519180a6ac05242c296f8906c11eab7e207b5ca9a89b6344392 + - name: RELATED_IMAGE_postgres_13_3 + value: quay.io/eclipse/che--centos--postgresql-13-centos7:1-71b24684d64da46f960682cc4216222a7e4ed8b1a31dd5a865b3e71afdea20d2 - name: RELATED_IMAGE_keycloak value: quay.io/eclipse/che-keycloak:next - name: RELATED_IMAGE_che_workspace_plugin_broker_metadata diff --git a/bundle/next/eclipse-che-preview-kubernetes/manifests/org_v1_che_crd.yaml b/bundle/next/eclipse-che-preview-kubernetes/manifests/org_v1_che_crd.yaml index 3babf2552..2473f05b3 100644 --- a/bundle/next/eclipse-che-preview-kubernetes/manifests/org_v1_che_crd.yaml +++ b/bundle/next/eclipse-che-preview-kubernetes/manifests/org_v1_che_crd.yaml @@ -212,6 +212,9 @@ spec: postgresImagePullPolicy: description: Overrides the image pull policy used in the PostgreSQL database deployment. Default value is `Always` for `nightly`, `next` or `latest` images, and `IfNotPresent` in other cases. type: string + postgresVersion: + description: 'Indicates a PostgreSQL version image to use. Allowed values are: `9.6` and `13.3`. Migrate your PostgreSQL database to switch from one version to another.' + type: string type: object devWorkspace: description: DevWorkspace operator configuration diff --git a/bundle/next/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml b/bundle/next/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml index 1f1b81451..e51ec2a93 100644 --- a/bundle/next/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml +++ b/bundle/next/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml @@ -877,6 +877,8 @@ spec: value: registry.access.redhat.com/ubi8-minimal:8.4-210 - name: RELATED_IMAGE_postgres value: quay.io/eclipse/che--centos--postgresql-96-centos7:9.6-b681d78125361519180a6ac05242c296f8906c11eab7e207b5ca9a89b6344392 + - name: RELATED_IMAGE_postgres_13_3 + value: quay.io/eclipse/che--centos--postgresql-13-centos7:1-71b24684d64da46f960682cc4216222a7e4ed8b1a31dd5a865b3e71afdea20d2 - name: RELATED_IMAGE_keycloak value: quay.io/eclipse/che-keycloak:next - name: RELATED_IMAGE_che_workspace_plugin_broker_metadata diff --git a/bundle/next/eclipse-che-preview-openshift/manifests/org_v1_che_crd.yaml b/bundle/next/eclipse-che-preview-openshift/manifests/org_v1_che_crd.yaml index 5038a55e4..7fc0a9a43 100644 --- a/bundle/next/eclipse-che-preview-openshift/manifests/org_v1_che_crd.yaml +++ b/bundle/next/eclipse-che-preview-openshift/manifests/org_v1_che_crd.yaml @@ -352,6 +352,11 @@ spec: database deployment. Default value is `Always` for `nightly`, `next` or `latest` images, and `IfNotPresent` in other cases. type: string + postgresVersion: + description: 'Indicates a PostgreSQL version image to use. Allowed + values are: `9.6` and `13.3`. Migrate your PostgreSQL database + to switch from one version to another.' + type: string type: object devWorkspace: description: DevWorkspace operator configuration diff --git a/config/crd/bases/org_v1_che_crd-v1beta1.yaml b/config/crd/bases/org_v1_che_crd-v1beta1.yaml index ef1b3fd2b..3c7100673 100644 --- a/config/crd/bases/org_v1_che_crd-v1beta1.yaml +++ b/config/crd/bases/org_v1_che_crd-v1beta1.yaml @@ -342,6 +342,11 @@ spec: database deployment. Default value is `Always` for `nightly`, `next` or `latest` images, and `IfNotPresent` in other cases. type: string + postgresVersion: + description: 'Indicates a PostgreSQL version image to use. Allowed + values are: `9.6` and `13.3`. Migrate your PostgreSQL database + to switch from one version to another.' + type: string type: object devWorkspace: description: DevWorkspace operator configuration diff --git a/config/crd/bases/org_v1_che_crd.yaml b/config/crd/bases/org_v1_che_crd.yaml index 72ece3f6b..a283e12be 100644 --- a/config/crd/bases/org_v1_che_crd.yaml +++ b/config/crd/bases/org_v1_che_crd.yaml @@ -352,6 +352,11 @@ spec: database deployment. Default value is `Always` for `nightly`, `next` or `latest` images, and `IfNotPresent` in other cases. type: string + postgresVersion: + description: 'Indicates a PostgreSQL version image to use. Allowed + values are: `9.6` and `13.3`. Migrate your PostgreSQL database + to switch from one version to another.' + type: string type: object devWorkspace: description: DevWorkspace operator configuration diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 5227b2cb6..0ca92db2f 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -71,6 +71,8 @@ spec: value: registry.access.redhat.com/ubi8-minimal:8.4-210 - name: RELATED_IMAGE_postgres value: quay.io/eclipse/che--centos--postgresql-96-centos7:9.6-b681d78125361519180a6ac05242c296f8906c11eab7e207b5ca9a89b6344392 + - name: RELATED_IMAGE_postgres_13_3 + value: quay.io/eclipse/che--centos--postgresql-13-centos7:1-71b24684d64da46f960682cc4216222a7e4ed8b1a31dd5a865b3e71afdea20d2 - name: RELATED_IMAGE_keycloak value: quay.io/eclipse/che-keycloak:next - name: RELATED_IMAGE_che_workspace_plugin_broker_metadata diff --git a/pkg/deploy/defaults.go b/pkg/deploy/defaults.go index aeb658ea1..952782183 100644 --- a/pkg/deploy/defaults.go +++ b/pkg/deploy/defaults.go @@ -36,6 +36,7 @@ var ( defaultCheTLSSecretsCreationJobImage string defaultPvcJobsImage string defaultPostgresImage string + defaultPostgres13Image string defaultKeycloakImage string defaultSingleHostGatewayImage string defaultSingleHostGatewayConfigSidecarImage string @@ -176,6 +177,7 @@ func InitDefaultsFromFile(defaultsPath string) { defaultDevfileRegistryImage = util.GetDeploymentEnv(operatorDeployment, util.GetArchitectureDependentEnv("RELATED_IMAGE_devfile_registry")) defaultPvcJobsImage = util.GetDeploymentEnv(operatorDeployment, util.GetArchitectureDependentEnv("RELATED_IMAGE_pvc_jobs")) defaultPostgresImage = util.GetDeploymentEnv(operatorDeployment, util.GetArchitectureDependentEnv("RELATED_IMAGE_postgres")) + defaultPostgres13Image = util.GetDeploymentEnv(operatorDeployment, util.GetArchitectureDependentEnv("RELATED_IMAGE_postgres_13_3")) defaultKeycloakImage = util.GetDeploymentEnv(operatorDeployment, util.GetArchitectureDependentEnv("RELATED_IMAGE_keycloak")) defaultSingleHostGatewayImage = util.GetDeploymentEnv(operatorDeployment, util.GetArchitectureDependentEnv("RELATED_IMAGE_single_host_gateway")) defaultSingleHostGatewayConfigSidecarImage = util.GetDeploymentEnv(operatorDeployment, util.GetArchitectureDependentEnv("RELATED_IMAGE_single_host_gateway_config_sidecar")) @@ -291,6 +293,14 @@ func DefaultPostgresImage(cr *orgv1.CheCluster) string { return patchDefaultImageName(cr, defaultPostgresImage) } +func DefaultPostgres13Image(cr *orgv1.CheCluster) string { + // it might be empty value until it propertly downstreamed + if defaultPostgres13Image == "" { + return defaultPostgres13Image + } + return patchDefaultImageName(cr, defaultPostgres13Image) +} + func DefaultDashboardImage(cr *orgv1.CheCluster) string { return patchDefaultImageName(cr, defaultDashboardImage) } @@ -437,6 +447,11 @@ func InitDefaultsFromEnv() { defaultDevfileRegistryImage = getDefaultFromEnv(util.GetArchitectureDependentEnv("RELATED_IMAGE_devfile_registry")) defaultPvcJobsImage = getDefaultFromEnv(util.GetArchitectureDependentEnv("RELATED_IMAGE_pvc_jobs")) defaultPostgresImage = getDefaultFromEnv(util.GetArchitectureDependentEnv("RELATED_IMAGE_postgres")) + + // allow not to set env variable into a container + // while downstream is not migrated to PostgreSQL 13.3 yet + defaultPostgres13Image = os.Getenv(util.GetArchitectureDependentEnv("RELATED_IMAGE_postgres_13_3")) + defaultKeycloakImage = getDefaultFromEnv(util.GetArchitectureDependentEnv("RELATED_IMAGE_keycloak")) defaultSingleHostGatewayImage = getDefaultFromEnv(util.GetArchitectureDependentEnv("RELATED_IMAGE_single_host_gateway")) defaultSingleHostGatewayConfigSidecarImage = getDefaultFromEnv(util.GetArchitectureDependentEnv("RELATED_IMAGE_single_host_gateway_config_sidecar")) diff --git a/pkg/deploy/defaults_test.go b/pkg/deploy/defaults_test.go index 5e986c72e..a089584d6 100644 --- a/pkg/deploy/defaults_test.go +++ b/pkg/deploy/defaults_test.go @@ -31,6 +31,7 @@ func TestDefaultFromEnv(t *testing.T) { devfileRegistryImageTest := os.Getenv(util.GetArchitectureDependentEnv("RELATED_IMAGE_devfile_registry")) pvcJobsImageTest := os.Getenv(util.GetArchitectureDependentEnv("RELATED_IMAGE_pvc_jobs")) postgresImageTest := os.Getenv(util.GetArchitectureDependentEnv("RELATED_IMAGE_postgres")) + postgres13ImageTest := os.Getenv(util.GetArchitectureDependentEnv("RELATED_IMAGE_postgres_13_3")) keycloakImageTest := os.Getenv(util.GetArchitectureDependentEnv("RELATED_IMAGE_keycloak")) brokerMetadataTest := os.Getenv(util.GetArchitectureDependentEnv("RELATED_IMAGE_che_workspace_plugin_broker_metadata")) brokerArtifactsTest := os.Getenv(util.GetArchitectureDependentEnv("RELATED_IMAGE_che_workspace_plugin_broker_artifacts")) @@ -70,6 +71,10 @@ func TestDefaultFromEnv(t *testing.T) { t.Errorf("Expected %s but was %s", postgresImageTest, DefaultPostgresImage(cheCluster)) } + if DefaultPostgres13Image(cheCluster) != postgres13ImageTest { + t.Errorf("Expected %s but was %s", postgres13ImageTest, DefaultPostgres13Image(cheCluster)) + } + if DefaultKeycloakImage(cheCluster) != keycloakImageTest { t.Errorf("Expected %s but was %s", keycloakImageTest, DefaultKeycloakImage(cheCluster)) } diff --git a/pkg/deploy/postgres/postgres.go b/pkg/deploy/postgres/postgres.go index 4a0f84adb..8872ed3ca 100644 --- a/pkg/deploy/postgres/postgres.go +++ b/pkg/deploy/postgres/postgres.go @@ -13,6 +13,7 @@ package postgres import ( "fmt" + "strings" orgv1 "github.com/eclipse-che/che-operator/api/v1" "github.com/eclipse-che/che-operator/pkg/deploy" @@ -58,6 +59,15 @@ func (p *Postgres) SyncAll() (bool, error) { } } + if p.deployContext.CheCluster.Spec.Database.PostgresVersion == "" { + if !util.IsTestMode() { // ignore in tests + done, err := p.setDbVersion() + if !done { + return false, err + } + } + } + return true, nil } @@ -127,3 +137,26 @@ func (p *Postgres) ProvisionDB() (bool, error) { return true, nil } + +func (p *Postgres) setDbVersion() (bool, error) { + postgresVersion, err := util.K8sclient.ExecIntoPod( + p.deployContext.CheCluster, + deploy.PostgresName, + func(cr *orgv1.CheCluster) (string, error) { + // don't take into account bugfix version + return "postgres -V | awk '{print $NF}' | cut -d '.' -f1-2", nil + }, + "get PostgreSQL version") + if err != nil { + return false, err + } + + postgresVersion = strings.TrimSpace(postgresVersion) + p.deployContext.CheCluster.Spec.Database.PostgresVersion = postgresVersion + err = deploy.UpdateCheCRSpec(p.deployContext, "database.postgresVersion", postgresVersion) + if err != nil { + return false, err + } + + return true, nil +} diff --git a/pkg/deploy/postgres/postgres_deployment.go b/pkg/deploy/postgres/postgres_deployment.go index 352625e2c..40dc9534a 100644 --- a/pkg/deploy/postgres/postgres_deployment.go +++ b/pkg/deploy/postgres/postgres_deployment.go @@ -12,6 +12,9 @@ package postgres import ( + "fmt" + + orgv1 "github.com/eclipse-che/che-operator/api/v1" "github.com/eclipse-che/che-operator/pkg/deploy" "github.com/eclipse-che/che-operator/pkg/util" appsv1 "k8s.io/api/apps/v1" @@ -20,6 +23,11 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) +const ( + PostgresVersion9_6 = "9.6" + PostgresVersion13_3 = "13.3" +) + var ( postgresAdminPassword = util.GeneratePasswd(12) ) @@ -28,7 +36,10 @@ func (p *Postgres) GetDeploymentSpec(clusterDeployment *appsv1.Deployment) (*app terminationGracePeriodSeconds := int64(30) labels, labelSelector := deploy.GetLabelsAndSelector(p.deployContext.CheCluster, deploy.PostgresName) chePostgresDb := util.GetValue(p.deployContext.CheCluster.Spec.Database.ChePostgresDb, "dbche") - postgresImage := util.GetValue(p.deployContext.CheCluster.Spec.Database.PostgresImage, deploy.DefaultPostgresImage(p.deployContext.CheCluster)) + postgresImage, err := getPostgresImage(clusterDeployment, p.deployContext.CheCluster) + if err != nil { + return nil, err + } pullPolicy := corev1.PullPolicy(util.GetValue(string(p.deployContext.CheCluster.Spec.Database.PostgresImagePullPolicy), deploy.DefaultPullPolicyFromDockerImage(postgresImage))) if clusterDeployment != nil { @@ -204,3 +215,30 @@ func (p *Postgres) GetDeploymentSpec(clusterDeployment *appsv1.Deployment) (*app return deployment, nil } + +func getPostgresImage(clusterDeployment *appsv1.Deployment, cheCluster *orgv1.CheCluster) (string, error) { + if cheCluster.Spec.Database.PostgresImage != "" { + // use image explicitly set in a CR + return cheCluster.Spec.Database.PostgresImage, nil + } else if cheCluster.Spec.Database.PostgresVersion == PostgresVersion9_6 { + return deploy.DefaultPostgresImage(cheCluster), nil + } else if cheCluster.Spec.Database.PostgresVersion == PostgresVersion13_3 { + return deploy.DefaultPostgres13Image(cheCluster), nil + } else if cheCluster.Spec.Database.PostgresVersion == "" { + if clusterDeployment == nil { + // Use PostgreSQL 13.3 for a new deployment if there is so. + // It allows to work in downstream until a new image is ready for production. + postgres13Image := deploy.DefaultPostgres13Image(cheCluster) + if postgres13Image != "" { + return postgres13Image, nil + } else { + return deploy.DefaultPostgresImage(cheCluster), nil + } + } else { + // Keep using current image + return clusterDeployment.Spec.Template.Spec.Containers[0].Image, nil + } + } + + return "", fmt.Errorf("PostgreSQL image for '%s' version not found", cheCluster.Spec.Database.PostgresVersion) +} diff --git a/pkg/deploy/postgres/postgres_test.go b/pkg/deploy/postgres/postgres_test.go index 421159bc2..c3f012315 100644 --- a/pkg/deploy/postgres/postgres_test.go +++ b/pkg/deploy/postgres/postgres_test.go @@ -13,9 +13,11 @@ package postgres import ( "context" + "fmt" "os" "github.com/eclipse-che/che-operator/pkg/util" + "github.com/stretchr/testify/assert" "github.com/eclipse-che/che-operator/pkg/deploy" appsv1 "k8s.io/api/apps/v1" @@ -165,3 +167,142 @@ func TestSyncAllToCluster(t *testing.T) { t.Fatalf("Failed to get deployment: %v", err) } } + +func TestGetPostgresImage(t *testing.T) { + type testCase struct { + name string + cheCluster *orgv1.CheCluster + postgresDeployment *appsv1.Deployment + + expectedPostgresImage string + expectedError bool + } + + testCases := []testCase{ + { + cheCluster: &orgv1.CheCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "eclipse-che", + }, + }, + expectedPostgresImage: deploy.DefaultPostgres13Image(&orgv1.CheCluster{}), + }, + { + cheCluster: &orgv1.CheCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "eclipse-che", + }, + Spec: orgv1.CheClusterSpec{ + Database: orgv1.CheClusterSpecDB{ + PostgresVersion: "13.3", + }, + }, + }, + expectedPostgresImage: deploy.DefaultPostgres13Image(&orgv1.CheCluster{}), + }, + { + cheCluster: &orgv1.CheCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "eclipse-che", + }, + Spec: orgv1.CheClusterSpec{ + Database: orgv1.CheClusterSpecDB{ + PostgresVersion: "9.6", + }, + }, + }, + expectedPostgresImage: deploy.DefaultPostgresImage(&orgv1.CheCluster{}), + }, + { + cheCluster: &orgv1.CheCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "eclipse-che", + }, + Spec: orgv1.CheClusterSpec{ + Database: orgv1.CheClusterSpecDB{ + PostgresImage: "custom_postgre_image", + PostgresVersion: "", + }, + }, + }, + expectedPostgresImage: "custom_postgre_image", + }, + { + cheCluster: &orgv1.CheCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "eclipse-che", + }, + Spec: orgv1.CheClusterSpec{ + Database: orgv1.CheClusterSpecDB{ + PostgresVersion: "unrecognized_version", + }, + }, + }, + expectedError: true, + }, + + { + cheCluster: &orgv1.CheCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "eclipse-che", + }, + Spec: orgv1.CheClusterSpec{ + Database: orgv1.CheClusterSpecDB{}, + }, + }, + postgresDeployment: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Image: "current_postgres_image", + }, + }, + }, + }, + }, + }, + expectedPostgresImage: "current_postgres_image", + }, + { + cheCluster: &orgv1.CheCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "eclipse-che", + }, + Spec: orgv1.CheClusterSpec{ + Database: orgv1.CheClusterSpecDB{ + PostgresVersion: "13.3", + }, + }, + }, + postgresDeployment: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Image: "current_postgres_image", + }, + }, + }, + }, + }, + }, + expectedPostgresImage: deploy.DefaultPostgres13Image(&orgv1.CheCluster{}), + }, + } + + for i, testCase := range testCases { + actualPostgreImage, err := getPostgresImage(testCase.postgresDeployment, testCase.cheCluster) + + t.Run(fmt.Sprintf("Test #%d", i), func(t *testing.T) { + if testCase.expectedError { + assert.NotNil(t, err, "Error expected") + } else { + assert.Nil(t, err, "Unexpected error occurred %v", err) + assert.Equal(t, testCase.expectedPostgresImage, actualPostgreImage, "A wrong PostgreSQL image") + } + }) + } +}