diff --git a/.gitignore b/.gitignore index 02020b6af..31b65cc7f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ # Temporary Build Files tmp/_output tmp/_test - +run-tests # Created by https://www.gitignore.io/api/go,vim,emacs,visualstudiocode diff --git a/Gopkg.lock b/Gopkg.lock index 8db43fd53..7ac13b3cb 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -903,6 +903,7 @@ "k8s.io/apimachinery/pkg/fields", "k8s.io/apimachinery/pkg/runtime", "k8s.io/apimachinery/pkg/runtime/schema", + "k8s.io/apimachinery/pkg/runtime/serializer", "k8s.io/apimachinery/pkg/types", "k8s.io/apimachinery/pkg/util/intstr", "k8s.io/apimachinery/pkg/watch", @@ -911,6 +912,8 @@ "k8s.io/client-go/kubernetes/fake", "k8s.io/client-go/kubernetes/scheme", "k8s.io/client-go/plugin/pkg/client/auth/gcp", + "k8s.io/client-go/rest", + "k8s.io/client-go/tools/clientcmd/api", "k8s.io/client-go/tools/remotecommand", "k8s.io/code-generator/cmd/client-gen", "k8s.io/code-generator/cmd/conversion-gen", diff --git a/README.md b/README.md index 1238738ec..050f6b7fa 100644 --- a/README.md +++ b/README.md @@ -104,11 +104,55 @@ cp deploy/keycloak_provision /tmp/keycloak_provision ``` This file is added to a Docker image, thus this step isn't required when deploying an operator image. +## E2E Tests +`e2e` directory contains end-to-end tests that create a custom resource, operator deployment, required RBAC. +Pre-reqs to run e3e tests: +* a running OpenShift instance (3.11+) +* current oc/kubectl context as a cluster admin user +### How to build tests binary +``` +OOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o $GOPATH/src/github.com/eclipse/che-operator/run-tests $GOPATH/src/github.com/eclipse/che-operator/e2e/*.go +``` +Or you can build in a container: + +``` +docker run -ti -v /tmp:/tmp -v ${OPERATOR_REPO}:/opt/app-root/src/go/src/github.com/eclipse/che-operator registry.access.redhat.com/devtools/go-toolset-rhel7:1.11.5-3 sh -c "OOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /tmp/run-tests /opt/app-root/src/go/src/github.com/eclipse/che-operator/e2e/*.go" +cp /tmp/run-tests ${OPERATOR_REPO}/run-tests +``` + +### How to run tests + +The resulted binary is created in the root of the repo. Make sure it is run from this location since it uses relative paths to yamls that are then deserialized. +There's a script `run-okd-local.sh` which is more of a CI thing, however, if you can run `oc cluster up` in your environment, you are unlikely to have any issues. + +``` +./run-tests +``` + +Tests create a number of k8s/OpenShift objects and generally assume that a fresh installation of OpenShift is available. +TODO: handle AlreadyExists errors to either remove che namespace or create a new one with a unique name. + +### What do tests check? + +#### Installation of Che/CRW + +A custom resource is created, which signals the operator to deploy Che/CRW with default settings. + +#### Configuration changes in runtime + +Once an successful installation of Che/CRW is verified, tests patch custom resource to: + +* enable oAuth +* enable TLS mode + +Subsequent checks verify that the installation is reconfigured, for example uses secure routes or ConfigMap has the right Login-with-OpenShift values + +TODO: add more scenarios diff --git a/deploy/cluster_role_binding.yaml b/deploy/cluster_role_binding.yaml new file mode 100644 index 000000000..79f7500f5 --- /dev/null +++ b/deploy/cluster_role_binding.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: che-operator +subjects: + - kind: ServiceAccount + name: che-operator + namespace: che +roleRef: + kind: ClusterRole + name: che-operator + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/e2e/config.go b/e2e/config.go new file mode 100644 index 000000000..61626bc9b --- /dev/null +++ b/e2e/config.go @@ -0,0 +1,145 @@ +// +// Copyright (c) 2012-2019 Red Hat, Inc. +// This program and the accompanying materials are made +// available under the terms of the Eclipse Public License 2.0 +// which is available at https://www.eclipse.org/legal/epl-2.0/ +// +// SPDX-License-Identifier: EPL-2.0 +// +// Contributors: +// Red Hat, Inc. - initial API and implementation +// +package main + +import ( + orgv1 "github.com/eclipse/che-operator/pkg/apis/org/v1" + oauth "github.com/openshift/api/oauth/v1" + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +var ( + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + AddToScheme = SchemeBuilder.AddToScheme + clientSet, err = getClientSet() + oauthClientSet, _ = getOAuthClientSet() + client = GetK8Config() + SchemeGroupVersion = schema.GroupVersion{Group: groupName, Version: orgv1.SchemeGroupVersion.Version} +) + +type k8s struct { + clientset kubernetes.Interface +} + +type CRClient struct { + restClient rest.Interface +} + +type OauthClient struct { + restClient rest.Interface +} + +func GetK8Config() *k8s { + cfg, err := config.GetConfig() + if err != nil { + logrus.Errorf(err.Error()) + } + client := k8s{} + client.clientset, err = kubernetes.NewForConfig(cfg) + + if err != nil { + logrus.Errorf(err.Error()) + } + return &client +} + +func getClientSet() (clientSet *CRClient, err error) { + cfg, err := config.GetConfig() + if err != nil { + logrus.Errorf(err.Error()) + } + client := k8s{} + client.clientset, err = kubernetes.NewForConfig(cfg) + clientSet, err = newForConfig(cfg) + if err != nil { + return nil, err + } + return clientSet, nil +} + +func getOAuthClientSet() (clientSet *OauthClient, err error) { + cfg, err := config.GetConfig() + if err != nil { + logrus.Errorf(err.Error()) + } + client := k8s{} + client.clientset, err = kubernetes.NewForConfig(cfg) + clientSet, err = newOAuthConfig(cfg) + if err != nil { + return nil, err + } + return clientSet, nil +} + +func getCR() (*orgv1.CheCluster, error) { + + result := orgv1.CheCluster{} + opts := metav1.ListOptions{} + err = clientSet.restClient. + Get(). + Namespace(namespace). + Resource(kind). + Name(crName). + VersionedParams(&opts, scheme.ParameterCodec). + Do(). + Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func newForConfig(c *rest.Config) (*CRClient, error) { + config := *c + config.ContentConfig.GroupVersion = &schema.GroupVersion{Group: groupName, Version: orgv1.SchemeGroupVersion.Version} + //config.ContentConfig.GroupVersion = &schema.GroupVersion{Group: oauth.GroupName, Version: oauth.SchemeGroupVersion.Version} + config.APIPath = "/apis" + config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs} + config.UserAgent = rest.DefaultKubernetesUserAgent() + client, err := rest.RESTClientFor(&config) + if err != nil { + return nil, err + } + + return &CRClient{restClient: client}, nil +} + +func newOAuthConfig(c *rest.Config) (*OauthClient, error) { + config := *c + config.ContentConfig.GroupVersion = &schema.GroupVersion{Group: oauth.GroupName, Version: oauth.SchemeGroupVersion.Version} + config.APIPath = "/apis" + config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs} + config.UserAgent = rest.DefaultKubernetesUserAgent() + client, err := rest.RESTClientFor(&config) + if err != nil { + return nil, err + } + + return &OauthClient{restClient: client}, nil +} + +func addKnownTypes(scheme *runtime.Scheme) (error) { + scheme.AddKnownTypes(SchemeGroupVersion, + &orgv1.CheCluster{}, + &orgv1.CheClusterList{}, + ) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/e2e/create.go b/e2e/create.go new file mode 100644 index 000000000..070c91d5f --- /dev/null +++ b/e2e/create.go @@ -0,0 +1,143 @@ +// +// Copyright (c) 2012-2019 Red Hat, Inc. +// This program and the accompanying materials are made +// available under the terms of the Eclipse Public License 2.0 +// which is available at https://www.eclipse.org/legal/epl-2.0/ +// +// SPDX-License-Identifier: EPL-2.0 +// +// Contributors: +// Red Hat, Inc. - initial API and implementation +// +package main + +import ( + orgv1 "github.com/eclipse/che-operator/pkg/apis/org/v1" + "github.com/sirupsen/logrus" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbac "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func createOperatorServiceAccount(operatorServiceAccount *corev1.ServiceAccount) (err error) { + + operatorServiceAccount, err = client.clientset.CoreV1().ServiceAccounts(namespace).Create(operatorServiceAccount) + if err != nil { + logrus.Fatalf("Failed to create service account %s: %s", operatorServiceAccount.Name, err) + return err + } + return nil + +} + +func createOperatorServiceAccountRole(operatorServiceAccountRole *rbac.Role) (err error) { + + operatorServiceAccountRole, err = client.clientset.RbacV1().Roles(namespace).Create(operatorServiceAccountRole) + if err != nil { + logrus.Fatalf("Failed to create role %s: %s", operatorServiceAccountRole.Name, err) + return err + } + return nil + +} + +func createOperatorServiceAccountClusterRole(operatorServiceAccountClusterRole *rbac.ClusterRole) (err error) { + + operatorServiceAccountClusterRole, err = client.clientset.RbacV1().ClusterRoles().Create(operatorServiceAccountClusterRole) + if err != nil && ! errors.IsAlreadyExists(err) { + logrus.Fatalf("Failed to create role %s: %s", operatorServiceAccountClusterRole.Name, err) + return err + } + return nil + +} + +func createOperatorServiceAccountRoleBinding(operatorServiceAccountRoleBinding *rbac.RoleBinding) (err error) { + + operatorServiceAccountRoleBinding, err = client.clientset.RbacV1().RoleBindings(namespace).Create(operatorServiceAccountRoleBinding) + if err != nil { + logrus.Fatalf("Failed to create role %s: %s", operatorServiceAccountRoleBinding.Name, err) + return err + } + return nil + +} + +func createOperatorServiceAccountClusterRoleBinding(operatorServiceAccountClusterRoleBinding *rbac.ClusterRoleBinding) (err error) { + + operatorServiceAccountClusterRoleBinding, err = client.clientset.RbacV1().ClusterRoleBindings().Create(operatorServiceAccountClusterRoleBinding) + if err != nil && !errors.IsAlreadyExists(err) { + logrus.Fatalf("Failed to create role %s: %s", operatorServiceAccountClusterRoleBinding.Name, err) + return err + } + return nil + +} + +func deployOperator(deployment *appsv1.Deployment) (err error) { + + deployment, err = client.clientset.AppsV1().Deployments(namespace).Create(deployment) + if err != nil { + logrus.Fatalf("Failed to create deployment %s: %s", deployment.Name, err) + return err + } + return nil + +} + +func newNamespace() (ns *corev1.Namespace){ + + return &corev1.Namespace{ + + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: corev1.SchemeGroupVersion.Version, + }, + ObjectMeta: metav1.ObjectMeta{ + Name:namespace, + }, + + } +} + +func createNamespace(ns *corev1.Namespace) (err error) { + + ns, err = client.clientset.CoreV1().Namespaces().Create(ns) + if err != nil { + logrus.Fatalf("Failed to create namespace %s: %s", ns.Name, err) + return err + } + return nil + +} + +func newCheCluster() (cr *orgv1.CheCluster) { + cr = &orgv1.CheCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: crName, + }, + TypeMeta: metav1.TypeMeta{ + Kind: kind, + }, + } + return cr +} + +func createCR() (err error) { + result := orgv1.CheCluster{} + cheCluster := newCheCluster() + err = clientSet.restClient. + Post(). + Namespace(namespace). + Resource(kind). + Name(crName). + Body(cheCluster). + Do(). + Into(&result) + if err != nil { + return err + } + return nil +} diff --git a/e2e/delete.go b/e2e/delete.go new file mode 100644 index 000000000..8baa3c390 --- /dev/null +++ b/e2e/delete.go @@ -0,0 +1,24 @@ +// +// Copyright (c) 2012-2019 Red Hat, Inc. +// This program and the accompanying materials are made +// available under the terms of the Eclipse Public License 2.0 +// which is available at https://www.eclipse.org/legal/epl-2.0/ +// +// SPDX-License-Identifier: EPL-2.0 +// +// Contributors: +// Red Hat, Inc. - initial API and implementation +// +package main + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +) + +func deleteNamespace() (err error) { + if err := client.clientset.CoreV1().Namespaces().Delete(namespace, &metav1.DeleteOptions{}); err != nil { + return err + } + return nil +} diff --git a/e2e/deserialize.go b/e2e/deserialize.go new file mode 100644 index 000000000..c8274272c --- /dev/null +++ b/e2e/deserialize.go @@ -0,0 +1,143 @@ +// +// Copyright (c) 2012-2019 Red Hat, Inc. +// This program and the accompanying materials are made +// available under the terms of the Eclipse Public License 2.0 +// which is available at https://www.eclipse.org/legal/epl-2.0/ +// +// SPDX-License-Identifier: EPL-2.0 +// +// Contributors: +// Red Hat, Inc. - initial API and implementation +// +package main + +import ( + "github.com/sirupsen/logrus" + "io/ioutil" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbac "k8s.io/api/rbac/v1" + "k8s.io/client-go/kubernetes/scheme" + "path/filepath" +) + +func deserializeOperatorDeployment() (operatorDeployment *appsv1.Deployment, err error) { + fileLocation, err := filepath.Abs("deploy/operator-local.yaml") + if err != nil { + logrus.Fatalf("Failed to locate operator deployment yaml, %s", err) + } + file, err := ioutil.ReadFile(fileLocation) + if err != nil { + logrus.Errorf("Failed to locate operator deployment yaml, %s", err) + } + deployment := string(file) + decode := scheme.Codecs.UniversalDeserializer().Decode + object, _, err := decode([]byte(deployment), nil, nil) + if err != nil { + logrus.Errorf("Failed to deserialize yaml %s", err) + return nil, err + } + operatorDeployment = object.(*appsv1.Deployment) + return operatorDeployment, nil +} + +func deserializeOperatorServiceAccount() (operatorServiceAccount *corev1.ServiceAccount, err error) { + fileLocation, err := filepath.Abs("deploy/service_account.yaml") + if err != nil { + logrus.Fatalf("Failed to locate operator service account yaml, %s", err) + } + file, err := ioutil.ReadFile(fileLocation) + if err != nil { + logrus.Errorf("Failed to locate operator service account yaml, %s", err) + } + sa := string(file) + decode := scheme.Codecs.UniversalDeserializer().Decode + object, _, err := decode([]byte(sa), nil, nil) + if err != nil { + logrus.Errorf("Failed to deserialize yaml %s", err) + return nil, err + } + operatorServiceAccount = object.(*corev1.ServiceAccount) + return operatorServiceAccount, nil +} + +func deserializeOperatorRole() (operatorServiceAccountRole *rbac.Role, err error) { + fileLocation, err := filepath.Abs("deploy/role.yaml") + if err != nil { + logrus.Fatalf("Failed to locate operator service account role yaml, %s", err) + } + file, err := ioutil.ReadFile(fileLocation) + if err != nil { + logrus.Errorf("Failed to locate operator service account role yaml, %s", err) + } + role := string(file) + decode := scheme.Codecs.UniversalDeserializer().Decode + object, _, err := decode([]byte(role), nil, nil) + if err != nil { + logrus.Errorf("Failed to deserialize yaml %s", err) + return nil, err + } + operatorServiceAccountRole = object.(*rbac.Role) + return operatorServiceAccountRole, nil +} + +func deserializeOperatorClusterRole() (operatorServiceAccountClusterRole *rbac.ClusterRole, err error) { + fileLocation, err := filepath.Abs("deploy/cluster_role.yaml") + if err != nil { + logrus.Fatalf("Failed to locate operator service account cluster role yaml, %s", err) + } + file, err := ioutil.ReadFile(fileLocation) + if err != nil { + logrus.Errorf("Failed to locate operator service account cluster role yaml, %s", err) + } + role := string(file) + decode := scheme.Codecs.UniversalDeserializer().Decode + object, _, err := decode([]byte(role), nil, nil) + if err != nil { + logrus.Errorf("Failed to deserialize yaml %s", err) + return nil, err + } + operatorServiceAccountClusterRole = object.(*rbac.ClusterRole) + return operatorServiceAccountClusterRole, nil +} + +func deserializeOperatorRoleBinding() (operatorServiceAccountRoleBinding *rbac.RoleBinding, err error) { + fileLocation, err := filepath.Abs("deploy/role_binding.yaml") + if err != nil { + logrus.Fatalf("Failed to locate operator service account role binding yaml, %s", err) + } + file, err := ioutil.ReadFile(fileLocation) + if err != nil { + logrus.Errorf("Failed to locate operator service account role binding yaml, %s", err) + } + roleBinding := string(file) + decode := scheme.Codecs.UniversalDeserializer().Decode + object, _, err := decode([]byte(roleBinding), nil, nil) + if err != nil { + logrus.Errorf("Failed to deserialize yaml %s", err) + return nil, err + } + operatorServiceAccountRoleBinding = object.(*rbac.RoleBinding) + return operatorServiceAccountRoleBinding, nil +} + + +func deserializeOperatorClusterRoleBinding() (operatorServiceAccountClusterRoleBinding *rbac.ClusterRoleBinding, err error) { + fileLocation, err := filepath.Abs("deploy/cluster_role_binding.yaml") + if err != nil { + logrus.Fatalf("Failed to locate operator service account role binding yaml, %s", err) + } + file, err := ioutil.ReadFile(fileLocation) + if err != nil { + logrus.Errorf("Failed to locate operator service account role binding yaml, %s", err) + } + roleBinding := string(file) + decode := scheme.Codecs.UniversalDeserializer().Decode + object, _, err := decode([]byte(roleBinding), nil, nil) + if err != nil { + logrus.Errorf("Failed to deserialize yaml %s", err) + return nil, err + } + operatorServiceAccountClusterRoleBinding = object.(*rbac.ClusterRoleBinding) + return operatorServiceAccountClusterRoleBinding, nil +} \ No newline at end of file diff --git a/e2e/get.go b/e2e/get.go new file mode 100644 index 000000000..016a466e9 --- /dev/null +++ b/e2e/get.go @@ -0,0 +1,38 @@ +// +// Copyright (c) 2012-2019 Red Hat, Inc. +// This program and the accompanying materials are made +// available under the terms of the Eclipse Public License 2.0 +// which is available at https://www.eclipse.org/legal/epl-2.0/ +// +// SPDX-License-Identifier: EPL-2.0 +// +// Contributors: +// Red Hat, Inc. - initial API and implementation +// +package main + +import ( + oauth "github.com/openshift/api/oauth/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) +func getOauthClient(name string)(oAuthClient *oauth.OAuthClient, err error) { + oAuthClient = &oauth.OAuthClient{} + err = oauthClientSet.restClient.Get().Name(name).Resource("oauthclients").Do().Into(oAuthClient) + if err != nil && errors.IsNotFound(err) { + return nil, err + } + return oAuthClient,nil +} + + +func getConfigMap(cmName string) (cm *corev1.ConfigMap, err error) { + + cm, err = client.clientset.CoreV1().ConfigMaps(namespace).Get(cmName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return cm, nil + +} \ No newline at end of file diff --git a/e2e/patch.go b/e2e/patch.go new file mode 100644 index 000000000..a359da194 --- /dev/null +++ b/e2e/patch.go @@ -0,0 +1,45 @@ +// +// Copyright (c) 2012-2019 Red Hat, Inc. +// This program and the accompanying materials are made +// available under the terms of the Eclipse Public License 2.0 +// which is available at https://www.eclipse.org/legal/epl-2.0/ +// +// SPDX-License-Identifier: EPL-2.0 +// +// Contributors: +// Red Hat, Inc. - initial API and implementation +// +package main + +import ( + "encoding/json" + "github.com/sirupsen/logrus" + api "k8s.io/apimachinery/pkg/types" +) + +func patchCustomResource(path string, value bool) (err error) { + + type PatchSpec struct { + Operation string `json:"op"` + Path string `json:"path"` + Value bool `json:"value"` + } + + fields := make([]PatchSpec, 1) + fields[0].Operation = "replace" + fields[0].Path = path + fields[0].Value = value + patchBytes, err := json.Marshal(fields) + if err != nil { + logrus.Errorf("Failed to marchall fields %s", err) + return err + } + _, err = clientSet.restClient.Patch(api.JSONPatchType).Name(crName).Namespace(namespace).Resource(kind).Body(patchBytes).Do().Get() + + if err != nil { + logrus.Errorf("Failed to patch CR: %s", err) + return err + } + + return nil +} \ No newline at end of file diff --git a/e2e/run-okd-tests.sh b/e2e/run-okd-tests.sh new file mode 100755 index 000000000..3b2897e8e --- /dev/null +++ b/e2e/run-okd-tests.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Copyright (c) 2012-2018 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +# Contributors: +# Red Hat, Inc. - initial API and implementation +set -e +# download oc +echo "Download oc client" +wget https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz -O /tmp/oc.tar && tar -xvf /tmp/oc.tar -C /tmp --strip-components=1 + +# start OKD +echo "Starting OKD 3.11" +cd /tmp +sudo rm -rf openshift.local.clusterup +./oc cluster up --public-hostname=172.17.0.1 --routing-suffix=172.17.0.1.nip.io +oc login -u system:admin +./oc adm policy add-cluster-role-to-user cluster-admin developer +./oc login -u developer -p password +sleep 10 +echo "Registering a custom resource definition" +./oc apply -f ${OPERATOR_REPO}/deploy/crds/org_v1_che_crd.yaml + +# generate self signed cert +echo "Generating self signed certificate" +openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -subj '/CN=*.172.17.0.1.nip.io' -nodes +cat cert.pem key.pem > ca.crt + +# replace default router cert +echo "Updating OpenShift router tls secret" +./oc project default +./oc secrets new router-certs tls.crt=ca.crt tls.key=key.pem -o json --type='kubernetes.io/tls' --confirm | oc replace -f - +echo "Initiating a new router deployment" +sleep 10 +./oc rollout latest dc/router -n=default + +echo "Compiling tests binary" +docker run -ti -v /tmp:/tmp -v ${OPERATOR_REPO}:/opt/app-root/src/go/src/github.com/eclipse/che-operator registry.access.redhat.com/devtools/go-toolset-rhel7:1.11.5-3 sh -c "OOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /tmp/run-tests /opt/app-root/src/go/src/github.com/eclipse/che-operator/e2e/*.go" +cp /tmp/run-tests ${OPERATOR_REPO}/run-tests + + +cd ${OPERATOR_REPO} +echo "Building operator docker image..." +docker build -t che/operator . +echo "Running tests..." +./run-tests \ No newline at end of file diff --git a/e2e/tests.go b/e2e/tests.go new file mode 100644 index 000000000..b3685c7b8 --- /dev/null +++ b/e2e/tests.go @@ -0,0 +1,194 @@ +// +// Copyright (c) 2012-2019 Red Hat, Inc. +// This program and the accompanying materials are made +// available under the terms of the Eclipse Public License 2.0 +// which is available at https://www.eclipse.org/legal/epl-2.0/ +// +// SPDX-License-Identifier: EPL-2.0 +// +// Contributors: +// Red Hat, Inc. - initial API and implementation +// +package main + +import ( + orgv1 "github.com/eclipse/che-operator/pkg/apis/org/v1" + "github.com/eclipse/che-operator/pkg/controller/che" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd/api" + "log" +) + +var ( + crName = "eclipse-che" + kind = "checlusters" + groupName = "org.eclipse.che" + namespace = "che" +) + +func main() { + logrus.Info("Starting CHE/CRW operator e2e tests") + logrus.Info("A running OCP instance and cluster-admin login are required") + logrus.Info("Adding CRD to schema") + if err := orgv1.SchemeBuilder.AddToScheme(scheme.Scheme); err != nil { + logrus.Fatalf("Failed to add CRD to scheme") + } + apiScheme := runtime.NewScheme() + if err := api.AddToScheme(apiScheme); err != nil { + logrus.Fatalf("Failed to add CRD to scheme") + } + logrus.Info("CRD successfully added to schema") + + + logrus.Infof("Creating a new namespace: %s", namespace) + ns := newNamespace() + if err := createNamespace(ns); err != nil { + logrus.Fatalf("Failed to create a namespace %s: %s", ns.Name, err) + } + + logrus.Info("Creating a new CR") + err = createCR() + if err != nil { + logrus.Fatalf("Failed to create %s CR: %s", crName, err) + } + logrus.Info("CR has been successfully created") + + logrus.Infof("Getting CR %s to verify it has been successfully created", crName) + cheCluster, err := getCR() + if err != nil { + logrus.Fatalf("An error occurred: %s", err) + } + logrus.Infof("CR found: name: %s", cheCluster.Name) + + logrus.Info("Creating a service account for operator deployment") + + operatorServiceAccount, err := deserializeOperatorServiceAccount() + if err := createOperatorServiceAccount(operatorServiceAccount); err != nil { + logrus.Fatalf("Failed to create Operator service account: %s", err) + } + + logrus.Info("Creating role for operator service account") + + operatorServiceAccountRole, err := deserializeOperatorRole() + if err := createOperatorServiceAccountRole(operatorServiceAccountRole); err != nil { + logrus.Fatalf("Failed to create Operator service account role: %s", err) + + } + + logrus.Info("Creating RoleBinding") + operatorServiceAccountRoleBinding, err := deserializeOperatorRoleBinding() + if err := createOperatorServiceAccountRoleBinding(operatorServiceAccountRoleBinding); err != nil { + logrus.Fatalf("Failed to create Operator service account role binding: %s", err) + + } + + logrus.Info("Deploying operator") + operatorDeployment, err := deserializeOperatorDeployment() + if err := deployOperator(operatorDeployment); err != nil { + logrus.Fatalf("Failed to create Operator deployment: %s", err) + } + + logrus.Info("Waiting for CR Available status. Timeout 6 min") + deployed, err := VerifyCheRunning(che.AvailableStatus) + if deployed { + logrus.Info("Installation succeeded") + } + + + // reconfigure CR to enable TLS support + logrus.Info("Patching CR with TLS enabled. This should cause a new Che deployment") + patchPath := "/spec/server/tlsSupport" + if err := patchCustomResource(patchPath, true); err != nil { + logrus.Fatalf("An error occurred while patching CR %s", err) + } + + // check if a CR status has changed to Rolling update in progress + redeployed, err := VerifyCheRunning(che.RollingUpdateInProgressStatus) + if redeployed { + logrus.Info("New deployment triggered") + } + + // wait for Available status + logrus.Info("Waiting for CR Available status. Timeout 6 min") + deployed, err = VerifyCheRunning(che.AvailableStatus) + if deployed { + logrus.Info("Installation succeeded") + } + + // create clusterRole and clusterRoleBinding to let operator service account create oAuthclients + logrus.Info("Creating cluster role for operator service account") + + operatorServiceAccountClusterRole, err := deserializeOperatorClusterRole() + if err := createOperatorServiceAccountClusterRole(operatorServiceAccountClusterRole); err != nil { + logrus.Fatalf("Failed to create Operator service account cluster role: %s", err) + + } + + logrus.Info("Creating RoleBinding") + operatorServiceAccountClusterRoleBinding, err := deserializeOperatorClusterRoleBinding() + if err := createOperatorServiceAccountClusterRoleBinding(operatorServiceAccountClusterRoleBinding); err != nil { + logrus.Fatalf("Failed to create Operator service account cluster role binding: %s", err) + + } + + // reconfigure CR to enable login with OpenShift + logrus.Info("Patching CR with oAuth enabled. This should cause a new Che deployment") + patchPath = "/spec/auth/openShiftoAuth" + if err := patchCustomResource(patchPath, true); err != nil { + logrus.Fatalf("An error occurred while patching CR %s", err) + } + + // check if a CR status has changed to Rolling update in progress + redeployed, err = VerifyCheRunning(che.RollingUpdateInProgressStatus) + if redeployed { + logrus.Info("New deployment triggered") + } + + // wait for Available status + logrus.Info("Waiting for CR Available status. Timeout 6 min") + deployed, err = VerifyCheRunning(che.AvailableStatus) + if deployed { + logrus.Info("Installation succeeded") + } + + // check if oAuthClient has been created + cr, err := getCR() + if err != nil { + logrus.Fatalf("Failed to get CR: %s", err) + } + oAuthClientName := cr.Spec.Auth.OauthClientName + _, err = getOauthClient(oAuthClientName) + if err != nil { + logrus.Fatalf("oAuthclient %s not found", oAuthClientName) + } + logrus.Infof("Checking if oauthclient %s has been created", oAuthClientName) + + // verify oathclient name is set in che ConfigMap + cm, err := getConfigMap("che") + if err != nil { + log.Fatalf("Failed to get ConfigMap: %s", err) + } + expectedIdentityProvider := "openshift-v3" + actualIdentityProvider := cm.Data["CHE_INFRA_OPENSHIFT_OAUTH__IDENTITY__PROVIDER"] + expectedWorkspaceProject := "" + actualWorkspaceProject := cm.Data["CHE_INFRA_OPENSHIFT_PROJECT"] + + logrus.Info("Checking if identity provider is added to configmap") + + if expectedIdentityProvider != actualIdentityProvider { + logrus.Fatalf("Test failed. Expecting identity provider: %s, got: %s", expectedIdentityProvider, actualIdentityProvider) + } + + logrus.Info("Checking if workspace project is empty in CM") + if expectedWorkspaceProject != actualWorkspaceProject { + logrus.Fatalf("Test failed. Expecting identity provider: %s, got: %s", expectedWorkspaceProject, actualWorkspaceProject) + } + + // cleanup + logrus.Infof("Tests passed. Deleting namespace %s", namespace) + if err := deleteNamespace(); err != nil { + logrus.Errorf("Failed to delete namespace %s: %s", namespace, err) + } +} diff --git a/e2e/watch.go b/e2e/watch.go new file mode 100644 index 000000000..7c3e29dca --- /dev/null +++ b/e2e/watch.go @@ -0,0 +1,38 @@ +// +// Copyright (c) 2012-2019 Red Hat, Inc. +// This program and the accompanying materials are made +// available under the terms of the Eclipse Public License 2.0 +// which is available at https://www.eclipse.org/legal/epl-2.0/ +// +// SPDX-License-Identifier: EPL-2.0 +// +// Contributors: +// Red Hat, Inc. - initial API and implementation +// +package main + +import ( + "errors" + "time" +) + +func VerifyCheRunning(status string) (deployed bool, err error) { + + timeout := time.After(6 * time.Minute) + tick := time.Tick(10 * time.Second) + for { + select { + case <-timeout: + return false, errors.New("timed out") + case <-tick: + customResource, _ := getCR() + if customResource.Status.CheClusterRunning != status { + + } else { + return true, nil + } + } + } +} + + diff --git a/pkg/controller/che/che_controller.go b/pkg/controller/che/che_controller.go index ef61829b3..6414c94e5 100644 --- a/pkg/controller/che/che_controller.go +++ b/pkg/controller/che/che_controller.go @@ -207,18 +207,20 @@ func (r *ReconcileChe) Reconcile(request reconcile.Request) (reconcile.Result, e if err != nil { logrus.Errorf("An error occurred when detecting current infra: %s", err) } - // create a secret with router tls cert if self signed certs are in use - // requires cluster admin privileges - selfSignedCert := instance.Spec.Server.SelfSignedCert - if isOpenShift && selfSignedCert { - crt, err := k8sclient.GetDefaultRouterCert("openshift-ingress") - if err != nil { - logrus.Errorf("Default router tls secret not found. Self signed cert isn't added") - return reconcile.Result{}, err - } else { - secret := deploy.NewSecret(instance, "self-signed-certificate", crt) - if err := r.CreateNewSecret(instance, secret); err != nil { + // create a secret with router tls cert + if isOpenShift { + secret := &corev1.Secret{} + if err := r.client.Get(context.TODO(), types.NamespacedName{Name: "self-signed-certificate", Namespace: instance.Namespace}, secret); + err != nil && errors.IsNotFound(err) { + crt, err := r.GetRouterTlsCrt(instance) + if err != nil { + logrus.Errorf("Default router tls secret not found. Self signed cert isn't added") return reconcile.Result{}, err + } else { + secret := deploy.NewSecret(instance, "self-signed-certificate", crt) + if err := r.CreateNewSecret(instance, secret); err != nil { + return reconcile.Result{}, err + } } } } diff --git a/pkg/controller/che/k8s_helpers.go b/pkg/controller/che/k8s_helpers.go index 23d1c567c..085fe4d2c 100644 --- a/pkg/controller/che/k8s_helpers.go +++ b/pkg/controller/che/k8s_helpers.go @@ -13,6 +13,11 @@ package che import ( "bytes" + "context" + "crypto/tls" + "encoding/pem" + orgv1 "github.com/eclipse/che-operator/pkg/apis/org/v1" + "github.com/eclipse/che-operator/pkg/deploy" "github.com/eclipse/che-operator/pkg/util" "github.com/sirupsen/logrus" "io" @@ -22,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" + "net/http" "sigs.k8s.io/controller-runtime/pkg/client/config" ) @@ -183,24 +189,24 @@ func (cl *k8s) GetDeploymentStatus(name string, ns string) (scaled bool) { } } } - dc, _ := cl.clientset.AppsV1().Deployments(ns).Get(name, metav1.GetOptions{}) - if dc.Status.AvailableReplicas != 1 { - logrus.Errorf("Failed to verify a successful %s deployment", name) - eventList := cl.GetEvents(name, ns).Items - for i := range eventList { - logrus.Errorf("Event message: %v", eventList[i].Message) - } - deploymentPod, err := cl.GetDeploymentPod(name, ns) - if err != nil { - return false - } - cl.GetPodLogs(deploymentPod, ns) - logrus.Errorf("Command to get deployment logs: kubectl logs deployment/%s -n=%s", name, ns) - logrus.Errorf("Get k8s events: kubectl get events "+ - "--field-selector "+ - "involvedObject.name=$(kubectl get pods -l=component=%s -n=%s"+ - " -o=jsonpath='{.items[0].metadata.name}') -n=%s", name, ns, ns) + dc, _ := cl.clientset.AppsV1().Deployments(ns).Get(name, metav1.GetOptions{}) + if dc.Status.AvailableReplicas != 1 { + logrus.Errorf("Failed to verify a successful %s deployment", name) + eventList := cl.GetEvents(name, ns).Items + for i := range eventList { + logrus.Errorf("Event message: %v", eventList[i].Message) + } + deploymentPod, err := cl.GetDeploymentPod(name, ns) + if err != nil { return false + } + cl.GetPodLogs(deploymentPod, ns) + logrus.Errorf("Command to get deployment logs: kubectl logs deployment/%s -n=%s", name, ns) + logrus.Errorf("Get k8s events: kubectl get events "+ + "--field-selector "+ + "involvedObject.name=$(kubectl get pods -l=component=%s -n=%s"+ + " -o=jsonpath='{.items[0].metadata.name}') -n=%s", name, ns, ns) + return false } return true } @@ -245,20 +251,42 @@ func (cl *k8s) GetDeploymentPod(name string, ns string) (podName string, err err return podName, nil } -// GetDefaultRouterCert retrieves secret with OpenShift router certificate and extracts it -// The cert is then used to create self-signed-certificate secret consumed by CheCluster server and workspaces -func (cl *k8s) GetDefaultRouterCert(ns string) (crt []byte, err error) { - options := metav1.GetOptions{} - secret, err := cl.clientset.CoreV1().Secrets(ns).Get("router-certs-default", options) - if err != nil { - // in 3.11 it's default namespace and router-certs secret - secret, err = cl.clientset.CoreV1().Secrets("default").Get("router-certs", options) - if err != nil { - logrus.Errorf("Failed to get a secret in both namespace %s and default: %s", ns, err) - return nil, err - } +// GetRouterTlsCrt creates a test TLS route and gets it to extract certificate chain +// There's an easier way which is to read tls secret in default (3.11) or openshift-ingress (4.0) namespace +// which however requires extra privileges for operator service account +func (r *ReconcileChe) GetRouterTlsCrt(instance *orgv1.CheCluster) (certificate []byte, err error) { + testRoute := deploy.NewTlsRoute(instance, "test", "test") + logrus.Infof("Creating a test route %s to extract routes crt", testRoute.Name) + if err := r.CreateNewRoute(instance, testRoute); err != nil { + logrus.Errorf("Failed to create test route %s: %s", testRoute.Name, err) + return nil, err } - secretData := secret.Data - crt = secretData["tls.crt"] - return crt, nil + url := "https://" + testRoute.Spec.Host + http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + client := &http.Client{} + req, err := http.NewRequest("GET", url, nil) + resp, err := client.Do(req) + if err != nil { + logrus.Errorf("An error occurred when reaching test TLS route: %s", err) + if r.tests { + fakeCrt := make([]byte, 5) + return fakeCrt, nil + } + return nil, err + } + + for i := range resp.TLS.PeerCertificates { + cert := resp.TLS.PeerCertificates[i].Raw + crt := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: cert, + }) + certificate = append(certificate, crt...) + } + logrus.Infof("Deleting a test route %s to extract routes crt", testRoute.Name) + if err := r.client.Delete(context.TODO(), testRoute); err != nil { + logrus.Errorf("Failed to delete test route %s: %s", testRoute.Name, err) + } + + return certificate, nil } diff --git a/pkg/deploy/exec_commands.go b/pkg/deploy/exec_commands.go index 3063e768b..d9bbd5009 100644 --- a/pkg/deploy/exec_commands.go +++ b/pkg/deploy/exec_commands.go @@ -48,7 +48,7 @@ func GetKeycloakProvisionCommand(cr *orgv1.CheCluster, cheHost string) (command } file, err := ioutil.ReadFile("/tmp/keycloak_provision") if err != nil { - logrus.Errorf("Failed to find keycloak entrypoint file %s", err) + logrus.Errorf("Failed to locate keycloak entrypoint file: %s", err) } keycloakTheme := "che" realmDisplayName := "Eclipse Che" diff --git a/pkg/deploy/labels.go b/pkg/deploy/labels.go index 4677ea7c2..8a5825328 100644 --- a/pkg/deploy/labels.go +++ b/pkg/deploy/labels.go @@ -13,10 +13,11 @@ package deploy import ( orgv1 "github.com/eclipse/che-operator/pkg/apis/org/v1" + "github.com/eclipse/che-operator/pkg/util" ) func GetLabels(cr *orgv1.CheCluster, component string) (labels map[string]string) { - cheFlavor := cr.Spec.Server.CheFlavor + cheFlavor := util.GetValue(cr.Spec.Server.CheFlavor, DefaultCheFlavor) labels = map[string]string{"app": cheFlavor, "component": component} return labels }