Auto fetch crt. Introduce e2e tests (#8)

pull/9/head
Eugene Ivantsov 2019-04-05 16:17:01 +03:00 committed by GitHub
parent e112fad833
commit ad6b2a3a21
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 957 additions and 46 deletions

2
.gitignore vendored
View File

@ -4,7 +4,7 @@
# Temporary Build Files
tmp/_output
tmp/_test
run-tests
# Created by https://www.gitignore.io/api/go,vim,emacs,visualstudiocode

3
Gopkg.lock generated
View File

@ -903,6 +903,7 @@
"k8s.io/apimachinery/pkg/fields",
"k8s.io/apimachinery/pkg/runtime",
"k8s.io/apimachinery/pkg/runtime/schema",
"k8s.io/apimachinery/pkg/runtime/serializer",
"k8s.io/apimachinery/pkg/types",
"k8s.io/apimachinery/pkg/util/intstr",
"k8s.io/apimachinery/pkg/watch",
@ -911,6 +912,8 @@
"k8s.io/client-go/kubernetes/fake",
"k8s.io/client-go/kubernetes/scheme",
"k8s.io/client-go/plugin/pkg/client/auth/gcp",
"k8s.io/client-go/rest",
"k8s.io/client-go/tools/clientcmd/api",
"k8s.io/client-go/tools/remotecommand",
"k8s.io/code-generator/cmd/client-gen",
"k8s.io/code-generator/cmd/conversion-gen",

View File

@ -104,11 +104,55 @@ cp deploy/keycloak_provision /tmp/keycloak_provision
```
This file is added to a Docker image, thus this step isn't required when deploying an operator image.
## E2E Tests
`e2e` directory contains end-to-end tests that create a custom resource, operator deployment, required RBAC.
Pre-reqs to run e3e tests:
* a running OpenShift instance (3.11+)
* current oc/kubectl context as a cluster admin user
### How to build tests binary
```
OOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o $GOPATH/src/github.com/eclipse/che-operator/run-tests $GOPATH/src/github.com/eclipse/che-operator/e2e/*.go
```
Or you can build in a container:
```
docker run -ti -v /tmp:/tmp -v ${OPERATOR_REPO}:/opt/app-root/src/go/src/github.com/eclipse/che-operator registry.access.redhat.com/devtools/go-toolset-rhel7:1.11.5-3 sh -c "OOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /tmp/run-tests /opt/app-root/src/go/src/github.com/eclipse/che-operator/e2e/*.go"
cp /tmp/run-tests ${OPERATOR_REPO}/run-tests
```
### How to run tests
The resulted binary is created in the root of the repo. Make sure it is run from this location since it uses relative paths to yamls that are then deserialized.
There's a script `run-okd-local.sh` which is more of a CI thing, however, if you can run `oc cluster up` in your environment, you are unlikely to have any issues.
```
./run-tests
```
Tests create a number of k8s/OpenShift objects and generally assume that a fresh installation of OpenShift is available.
TODO: handle AlreadyExists errors to either remove che namespace or create a new one with a unique name.
### What do tests check?
#### Installation of Che/CRW
A custom resource is created, which signals the operator to deploy Che/CRW with default settings.
#### Configuration changes in runtime
Once an successful installation of Che/CRW is verified, tests patch custom resource to:
* enable oAuth
* enable TLS mode
Subsequent checks verify that the installation is reconfigured, for example uses secure routes or ConfigMap has the right Login-with-OpenShift values
TODO: add more scenarios

View File

@ -0,0 +1,12 @@
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: che-operator
subjects:
- kind: ServiceAccount
name: che-operator
namespace: che
roleRef:
kind: ClusterRole
name: che-operator
apiGroup: rbac.authorization.k8s.io

145
e2e/config.go Normal file
View File

@ -0,0 +1,145 @@
//
// Copyright (c) 2012-2019 Red Hat, Inc.
// This program and the accompanying materials are made
// available under the terms of the Eclipse Public License 2.0
// which is available at https://www.eclipse.org/legal/epl-2.0/
//
// SPDX-License-Identifier: EPL-2.0
//
// Contributors:
// Red Hat, Inc. - initial API and implementation
//
package main
import (
orgv1 "github.com/eclipse/che-operator/pkg/apis/org/v1"
oauth "github.com/openshift/api/oauth/v1"
"github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client/config"
)
var (
SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
AddToScheme = SchemeBuilder.AddToScheme
clientSet, err = getClientSet()
oauthClientSet, _ = getOAuthClientSet()
client = GetK8Config()
SchemeGroupVersion = schema.GroupVersion{Group: groupName, Version: orgv1.SchemeGroupVersion.Version}
)
type k8s struct {
clientset kubernetes.Interface
}
type CRClient struct {
restClient rest.Interface
}
type OauthClient struct {
restClient rest.Interface
}
func GetK8Config() *k8s {
cfg, err := config.GetConfig()
if err != nil {
logrus.Errorf(err.Error())
}
client := k8s{}
client.clientset, err = kubernetes.NewForConfig(cfg)
if err != nil {
logrus.Errorf(err.Error())
}
return &client
}
func getClientSet() (clientSet *CRClient, err error) {
cfg, err := config.GetConfig()
if err != nil {
logrus.Errorf(err.Error())
}
client := k8s{}
client.clientset, err = kubernetes.NewForConfig(cfg)
clientSet, err = newForConfig(cfg)
if err != nil {
return nil, err
}
return clientSet, nil
}
func getOAuthClientSet() (clientSet *OauthClient, err error) {
cfg, err := config.GetConfig()
if err != nil {
logrus.Errorf(err.Error())
}
client := k8s{}
client.clientset, err = kubernetes.NewForConfig(cfg)
clientSet, err = newOAuthConfig(cfg)
if err != nil {
return nil, err
}
return clientSet, nil
}
func getCR() (*orgv1.CheCluster, error) {
result := orgv1.CheCluster{}
opts := metav1.ListOptions{}
err = clientSet.restClient.
Get().
Namespace(namespace).
Resource(kind).
Name(crName).
VersionedParams(&opts, scheme.ParameterCodec).
Do().
Into(&result)
if err != nil {
return nil, err
}
return &result, nil
}
func newForConfig(c *rest.Config) (*CRClient, error) {
config := *c
config.ContentConfig.GroupVersion = &schema.GroupVersion{Group: groupName, Version: orgv1.SchemeGroupVersion.Version}
//config.ContentConfig.GroupVersion = &schema.GroupVersion{Group: oauth.GroupName, Version: oauth.SchemeGroupVersion.Version}
config.APIPath = "/apis"
config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs}
config.UserAgent = rest.DefaultKubernetesUserAgent()
client, err := rest.RESTClientFor(&config)
if err != nil {
return nil, err
}
return &CRClient{restClient: client}, nil
}
func newOAuthConfig(c *rest.Config) (*OauthClient, error) {
config := *c
config.ContentConfig.GroupVersion = &schema.GroupVersion{Group: oauth.GroupName, Version: oauth.SchemeGroupVersion.Version}
config.APIPath = "/apis"
config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs}
config.UserAgent = rest.DefaultKubernetesUserAgent()
client, err := rest.RESTClientFor(&config)
if err != nil {
return nil, err
}
return &OauthClient{restClient: client}, nil
}
func addKnownTypes(scheme *runtime.Scheme) (error) {
scheme.AddKnownTypes(SchemeGroupVersion,
&orgv1.CheCluster{},
&orgv1.CheClusterList{},
)
metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
return nil
}

143
e2e/create.go Normal file
View File

@ -0,0 +1,143 @@
//
// Copyright (c) 2012-2019 Red Hat, Inc.
// This program and the accompanying materials are made
// available under the terms of the Eclipse Public License 2.0
// which is available at https://www.eclipse.org/legal/epl-2.0/
//
// SPDX-License-Identifier: EPL-2.0
//
// Contributors:
// Red Hat, Inc. - initial API and implementation
//
package main
import (
orgv1 "github.com/eclipse/che-operator/pkg/apis/org/v1"
"github.com/sirupsen/logrus"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func createOperatorServiceAccount(operatorServiceAccount *corev1.ServiceAccount) (err error) {
operatorServiceAccount, err = client.clientset.CoreV1().ServiceAccounts(namespace).Create(operatorServiceAccount)
if err != nil {
logrus.Fatalf("Failed to create service account %s: %s", operatorServiceAccount.Name, err)
return err
}
return nil
}
func createOperatorServiceAccountRole(operatorServiceAccountRole *rbac.Role) (err error) {
operatorServiceAccountRole, err = client.clientset.RbacV1().Roles(namespace).Create(operatorServiceAccountRole)
if err != nil {
logrus.Fatalf("Failed to create role %s: %s", operatorServiceAccountRole.Name, err)
return err
}
return nil
}
func createOperatorServiceAccountClusterRole(operatorServiceAccountClusterRole *rbac.ClusterRole) (err error) {
operatorServiceAccountClusterRole, err = client.clientset.RbacV1().ClusterRoles().Create(operatorServiceAccountClusterRole)
if err != nil && ! errors.IsAlreadyExists(err) {
logrus.Fatalf("Failed to create role %s: %s", operatorServiceAccountClusterRole.Name, err)
return err
}
return nil
}
func createOperatorServiceAccountRoleBinding(operatorServiceAccountRoleBinding *rbac.RoleBinding) (err error) {
operatorServiceAccountRoleBinding, err = client.clientset.RbacV1().RoleBindings(namespace).Create(operatorServiceAccountRoleBinding)
if err != nil {
logrus.Fatalf("Failed to create role %s: %s", operatorServiceAccountRoleBinding.Name, err)
return err
}
return nil
}
func createOperatorServiceAccountClusterRoleBinding(operatorServiceAccountClusterRoleBinding *rbac.ClusterRoleBinding) (err error) {
operatorServiceAccountClusterRoleBinding, err = client.clientset.RbacV1().ClusterRoleBindings().Create(operatorServiceAccountClusterRoleBinding)
if err != nil && !errors.IsAlreadyExists(err) {
logrus.Fatalf("Failed to create role %s: %s", operatorServiceAccountClusterRoleBinding.Name, err)
return err
}
return nil
}
func deployOperator(deployment *appsv1.Deployment) (err error) {
deployment, err = client.clientset.AppsV1().Deployments(namespace).Create(deployment)
if err != nil {
logrus.Fatalf("Failed to create deployment %s: %s", deployment.Name, err)
return err
}
return nil
}
func newNamespace() (ns *corev1.Namespace){
return &corev1.Namespace{
TypeMeta: metav1.TypeMeta{
Kind: "Namespace",
APIVersion: corev1.SchemeGroupVersion.Version,
},
ObjectMeta: metav1.ObjectMeta{
Name:namespace,
},
}
}
func createNamespace(ns *corev1.Namespace) (err error) {
ns, err = client.clientset.CoreV1().Namespaces().Create(ns)
if err != nil {
logrus.Fatalf("Failed to create namespace %s: %s", ns.Name, err)
return err
}
return nil
}
func newCheCluster() (cr *orgv1.CheCluster) {
cr = &orgv1.CheCluster{
ObjectMeta: metav1.ObjectMeta{
Name: crName,
},
TypeMeta: metav1.TypeMeta{
Kind: kind,
},
}
return cr
}
func createCR() (err error) {
result := orgv1.CheCluster{}
cheCluster := newCheCluster()
err = clientSet.restClient.
Post().
Namespace(namespace).
Resource(kind).
Name(crName).
Body(cheCluster).
Do().
Into(&result)
if err != nil {
return err
}
return nil
}

24
e2e/delete.go Normal file
View File

@ -0,0 +1,24 @@
//
// Copyright (c) 2012-2019 Red Hat, Inc.
// This program and the accompanying materials are made
// available under the terms of the Eclipse Public License 2.0
// which is available at https://www.eclipse.org/legal/epl-2.0/
//
// SPDX-License-Identifier: EPL-2.0
//
// Contributors:
// Red Hat, Inc. - initial API and implementation
//
package main
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func deleteNamespace() (err error) {
if err := client.clientset.CoreV1().Namespaces().Delete(namespace, &metav1.DeleteOptions{}); err != nil {
return err
}
return nil
}

143
e2e/deserialize.go Normal file
View File

@ -0,0 +1,143 @@
//
// Copyright (c) 2012-2019 Red Hat, Inc.
// This program and the accompanying materials are made
// available under the terms of the Eclipse Public License 2.0
// which is available at https://www.eclipse.org/legal/epl-2.0/
//
// SPDX-License-Identifier: EPL-2.0
//
// Contributors:
// Red Hat, Inc. - initial API and implementation
//
package main
import (
"github.com/sirupsen/logrus"
"io/ioutil"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
"k8s.io/client-go/kubernetes/scheme"
"path/filepath"
)
func deserializeOperatorDeployment() (operatorDeployment *appsv1.Deployment, err error) {
fileLocation, err := filepath.Abs("deploy/operator-local.yaml")
if err != nil {
logrus.Fatalf("Failed to locate operator deployment yaml, %s", err)
}
file, err := ioutil.ReadFile(fileLocation)
if err != nil {
logrus.Errorf("Failed to locate operator deployment yaml, %s", err)
}
deployment := string(file)
decode := scheme.Codecs.UniversalDeserializer().Decode
object, _, err := decode([]byte(deployment), nil, nil)
if err != nil {
logrus.Errorf("Failed to deserialize yaml %s", err)
return nil, err
}
operatorDeployment = object.(*appsv1.Deployment)
return operatorDeployment, nil
}
func deserializeOperatorServiceAccount() (operatorServiceAccount *corev1.ServiceAccount, err error) {
fileLocation, err := filepath.Abs("deploy/service_account.yaml")
if err != nil {
logrus.Fatalf("Failed to locate operator service account yaml, %s", err)
}
file, err := ioutil.ReadFile(fileLocation)
if err != nil {
logrus.Errorf("Failed to locate operator service account yaml, %s", err)
}
sa := string(file)
decode := scheme.Codecs.UniversalDeserializer().Decode
object, _, err := decode([]byte(sa), nil, nil)
if err != nil {
logrus.Errorf("Failed to deserialize yaml %s", err)
return nil, err
}
operatorServiceAccount = object.(*corev1.ServiceAccount)
return operatorServiceAccount, nil
}
func deserializeOperatorRole() (operatorServiceAccountRole *rbac.Role, err error) {
fileLocation, err := filepath.Abs("deploy/role.yaml")
if err != nil {
logrus.Fatalf("Failed to locate operator service account role yaml, %s", err)
}
file, err := ioutil.ReadFile(fileLocation)
if err != nil {
logrus.Errorf("Failed to locate operator service account role yaml, %s", err)
}
role := string(file)
decode := scheme.Codecs.UniversalDeserializer().Decode
object, _, err := decode([]byte(role), nil, nil)
if err != nil {
logrus.Errorf("Failed to deserialize yaml %s", err)
return nil, err
}
operatorServiceAccountRole = object.(*rbac.Role)
return operatorServiceAccountRole, nil
}
func deserializeOperatorClusterRole() (operatorServiceAccountClusterRole *rbac.ClusterRole, err error) {
fileLocation, err := filepath.Abs("deploy/cluster_role.yaml")
if err != nil {
logrus.Fatalf("Failed to locate operator service account cluster role yaml, %s", err)
}
file, err := ioutil.ReadFile(fileLocation)
if err != nil {
logrus.Errorf("Failed to locate operator service account cluster role yaml, %s", err)
}
role := string(file)
decode := scheme.Codecs.UniversalDeserializer().Decode
object, _, err := decode([]byte(role), nil, nil)
if err != nil {
logrus.Errorf("Failed to deserialize yaml %s", err)
return nil, err
}
operatorServiceAccountClusterRole = object.(*rbac.ClusterRole)
return operatorServiceAccountClusterRole, nil
}
func deserializeOperatorRoleBinding() (operatorServiceAccountRoleBinding *rbac.RoleBinding, err error) {
fileLocation, err := filepath.Abs("deploy/role_binding.yaml")
if err != nil {
logrus.Fatalf("Failed to locate operator service account role binding yaml, %s", err)
}
file, err := ioutil.ReadFile(fileLocation)
if err != nil {
logrus.Errorf("Failed to locate operator service account role binding yaml, %s", err)
}
roleBinding := string(file)
decode := scheme.Codecs.UniversalDeserializer().Decode
object, _, err := decode([]byte(roleBinding), nil, nil)
if err != nil {
logrus.Errorf("Failed to deserialize yaml %s", err)
return nil, err
}
operatorServiceAccountRoleBinding = object.(*rbac.RoleBinding)
return operatorServiceAccountRoleBinding, nil
}
func deserializeOperatorClusterRoleBinding() (operatorServiceAccountClusterRoleBinding *rbac.ClusterRoleBinding, err error) {
fileLocation, err := filepath.Abs("deploy/cluster_role_binding.yaml")
if err != nil {
logrus.Fatalf("Failed to locate operator service account role binding yaml, %s", err)
}
file, err := ioutil.ReadFile(fileLocation)
if err != nil {
logrus.Errorf("Failed to locate operator service account role binding yaml, %s", err)
}
roleBinding := string(file)
decode := scheme.Codecs.UniversalDeserializer().Decode
object, _, err := decode([]byte(roleBinding), nil, nil)
if err != nil {
logrus.Errorf("Failed to deserialize yaml %s", err)
return nil, err
}
operatorServiceAccountClusterRoleBinding = object.(*rbac.ClusterRoleBinding)
return operatorServiceAccountClusterRoleBinding, nil
}

38
e2e/get.go Normal file
View File

@ -0,0 +1,38 @@
//
// Copyright (c) 2012-2019 Red Hat, Inc.
// This program and the accompanying materials are made
// available under the terms of the Eclipse Public License 2.0
// which is available at https://www.eclipse.org/legal/epl-2.0/
//
// SPDX-License-Identifier: EPL-2.0
//
// Contributors:
// Red Hat, Inc. - initial API and implementation
//
package main
import (
oauth "github.com/openshift/api/oauth/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func getOauthClient(name string)(oAuthClient *oauth.OAuthClient, err error) {
oAuthClient = &oauth.OAuthClient{}
err = oauthClientSet.restClient.Get().Name(name).Resource("oauthclients").Do().Into(oAuthClient)
if err != nil && errors.IsNotFound(err) {
return nil, err
}
return oAuthClient,nil
}
func getConfigMap(cmName string) (cm *corev1.ConfigMap, err error) {
cm, err = client.clientset.CoreV1().ConfigMaps(namespace).Get(cmName, metav1.GetOptions{})
if err != nil {
return nil, err
}
return cm, nil
}

45
e2e/patch.go Normal file
View File

@ -0,0 +1,45 @@
//
// Copyright (c) 2012-2019 Red Hat, Inc.
// This program and the accompanying materials are made
// available under the terms of the Eclipse Public License 2.0
// which is available at https://www.eclipse.org/legal/epl-2.0/
//
// SPDX-License-Identifier: EPL-2.0
//
// Contributors:
// Red Hat, Inc. - initial API and implementation
//
package main
import (
"encoding/json"
"github.com/sirupsen/logrus"
api "k8s.io/apimachinery/pkg/types"
)
func patchCustomResource(path string, value bool) (err error) {
type PatchSpec struct {
Operation string `json:"op"`
Path string `json:"path"`
Value bool `json:"value"`
}
fields := make([]PatchSpec, 1)
fields[0].Operation = "replace"
fields[0].Path = path
fields[0].Value = value
patchBytes, err := json.Marshal(fields)
if err != nil {
logrus.Errorf("Failed to marchall fields %s", err)
return err
}
_, err = clientSet.restClient.Patch(api.JSONPatchType).Name(crName).Namespace(namespace).Resource(kind).Body(patchBytes).Do().Get()
if err != nil {
logrus.Errorf("Failed to patch CR: %s", err)
return err
}
return nil
}

51
e2e/run-okd-tests.sh Executable file
View File

@ -0,0 +1,51 @@
#!/bin/bash
#
# Copyright (c) 2012-2018 Red Hat, Inc.
# This program and the accompanying materials are made
# available under the terms of the Eclipse Public License 2.0
# which is available at https://www.eclipse.org/legal/epl-2.0/
#
# SPDX-License-Identifier: EPL-2.0
#
# Contributors:
# Red Hat, Inc. - initial API and implementation
set -e
# download oc
echo "Download oc client"
wget https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz -O /tmp/oc.tar && tar -xvf /tmp/oc.tar -C /tmp --strip-components=1
# start OKD
echo "Starting OKD 3.11"
cd /tmp
sudo rm -rf openshift.local.clusterup
./oc cluster up --public-hostname=172.17.0.1 --routing-suffix=172.17.0.1.nip.io
oc login -u system:admin
./oc adm policy add-cluster-role-to-user cluster-admin developer
./oc login -u developer -p password
sleep 10
echo "Registering a custom resource definition"
./oc apply -f ${OPERATOR_REPO}/deploy/crds/org_v1_che_crd.yaml
# generate self signed cert
echo "Generating self signed certificate"
openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -subj '/CN=*.172.17.0.1.nip.io' -nodes
cat cert.pem key.pem > ca.crt
# replace default router cert
echo "Updating OpenShift router tls secret"
./oc project default
./oc secrets new router-certs tls.crt=ca.crt tls.key=key.pem -o json --type='kubernetes.io/tls' --confirm | oc replace -f -
echo "Initiating a new router deployment"
sleep 10
./oc rollout latest dc/router -n=default
echo "Compiling tests binary"
docker run -ti -v /tmp:/tmp -v ${OPERATOR_REPO}:/opt/app-root/src/go/src/github.com/eclipse/che-operator registry.access.redhat.com/devtools/go-toolset-rhel7:1.11.5-3 sh -c "OOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /tmp/run-tests /opt/app-root/src/go/src/github.com/eclipse/che-operator/e2e/*.go"
cp /tmp/run-tests ${OPERATOR_REPO}/run-tests
cd ${OPERATOR_REPO}
echo "Building operator docker image..."
docker build -t che/operator .
echo "Running tests..."
./run-tests

194
e2e/tests.go Normal file
View File

@ -0,0 +1,194 @@
//
// Copyright (c) 2012-2019 Red Hat, Inc.
// This program and the accompanying materials are made
// available under the terms of the Eclipse Public License 2.0
// which is available at https://www.eclipse.org/legal/epl-2.0/
//
// SPDX-License-Identifier: EPL-2.0
//
// Contributors:
// Red Hat, Inc. - initial API and implementation
//
package main
import (
orgv1 "github.com/eclipse/che-operator/pkg/apis/org/v1"
"github.com/eclipse/che-operator/pkg/controller/che"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/clientcmd/api"
"log"
)
var (
crName = "eclipse-che"
kind = "checlusters"
groupName = "org.eclipse.che"
namespace = "che"
)
func main() {
logrus.Info("Starting CHE/CRW operator e2e tests")
logrus.Info("A running OCP instance and cluster-admin login are required")
logrus.Info("Adding CRD to schema")
if err := orgv1.SchemeBuilder.AddToScheme(scheme.Scheme); err != nil {
logrus.Fatalf("Failed to add CRD to scheme")
}
apiScheme := runtime.NewScheme()
if err := api.AddToScheme(apiScheme); err != nil {
logrus.Fatalf("Failed to add CRD to scheme")
}
logrus.Info("CRD successfully added to schema")
logrus.Infof("Creating a new namespace: %s", namespace)
ns := newNamespace()
if err := createNamespace(ns); err != nil {
logrus.Fatalf("Failed to create a namespace %s: %s", ns.Name, err)
}
logrus.Info("Creating a new CR")
err = createCR()
if err != nil {
logrus.Fatalf("Failed to create %s CR: %s", crName, err)
}
logrus.Info("CR has been successfully created")
logrus.Infof("Getting CR %s to verify it has been successfully created", crName)
cheCluster, err := getCR()
if err != nil {
logrus.Fatalf("An error occurred: %s", err)
}
logrus.Infof("CR found: name: %s", cheCluster.Name)
logrus.Info("Creating a service account for operator deployment")
operatorServiceAccount, err := deserializeOperatorServiceAccount()
if err := createOperatorServiceAccount(operatorServiceAccount); err != nil {
logrus.Fatalf("Failed to create Operator service account: %s", err)
}
logrus.Info("Creating role for operator service account")
operatorServiceAccountRole, err := deserializeOperatorRole()
if err := createOperatorServiceAccountRole(operatorServiceAccountRole); err != nil {
logrus.Fatalf("Failed to create Operator service account role: %s", err)
}
logrus.Info("Creating RoleBinding")
operatorServiceAccountRoleBinding, err := deserializeOperatorRoleBinding()
if err := createOperatorServiceAccountRoleBinding(operatorServiceAccountRoleBinding); err != nil {
logrus.Fatalf("Failed to create Operator service account role binding: %s", err)
}
logrus.Info("Deploying operator")
operatorDeployment, err := deserializeOperatorDeployment()
if err := deployOperator(operatorDeployment); err != nil {
logrus.Fatalf("Failed to create Operator deployment: %s", err)
}
logrus.Info("Waiting for CR Available status. Timeout 6 min")
deployed, err := VerifyCheRunning(che.AvailableStatus)
if deployed {
logrus.Info("Installation succeeded")
}
// reconfigure CR to enable TLS support
logrus.Info("Patching CR with TLS enabled. This should cause a new Che deployment")
patchPath := "/spec/server/tlsSupport"
if err := patchCustomResource(patchPath, true); err != nil {
logrus.Fatalf("An error occurred while patching CR %s", err)
}
// check if a CR status has changed to Rolling update in progress
redeployed, err := VerifyCheRunning(che.RollingUpdateInProgressStatus)
if redeployed {
logrus.Info("New deployment triggered")
}
// wait for Available status
logrus.Info("Waiting for CR Available status. Timeout 6 min")
deployed, err = VerifyCheRunning(che.AvailableStatus)
if deployed {
logrus.Info("Installation succeeded")
}
// create clusterRole and clusterRoleBinding to let operator service account create oAuthclients
logrus.Info("Creating cluster role for operator service account")
operatorServiceAccountClusterRole, err := deserializeOperatorClusterRole()
if err := createOperatorServiceAccountClusterRole(operatorServiceAccountClusterRole); err != nil {
logrus.Fatalf("Failed to create Operator service account cluster role: %s", err)
}
logrus.Info("Creating RoleBinding")
operatorServiceAccountClusterRoleBinding, err := deserializeOperatorClusterRoleBinding()
if err := createOperatorServiceAccountClusterRoleBinding(operatorServiceAccountClusterRoleBinding); err != nil {
logrus.Fatalf("Failed to create Operator service account cluster role binding: %s", err)
}
// reconfigure CR to enable login with OpenShift
logrus.Info("Patching CR with oAuth enabled. This should cause a new Che deployment")
patchPath = "/spec/auth/openShiftoAuth"
if err := patchCustomResource(patchPath, true); err != nil {
logrus.Fatalf("An error occurred while patching CR %s", err)
}
// check if a CR status has changed to Rolling update in progress
redeployed, err = VerifyCheRunning(che.RollingUpdateInProgressStatus)
if redeployed {
logrus.Info("New deployment triggered")
}
// wait for Available status
logrus.Info("Waiting for CR Available status. Timeout 6 min")
deployed, err = VerifyCheRunning(che.AvailableStatus)
if deployed {
logrus.Info("Installation succeeded")
}
// check if oAuthClient has been created
cr, err := getCR()
if err != nil {
logrus.Fatalf("Failed to get CR: %s", err)
}
oAuthClientName := cr.Spec.Auth.OauthClientName
_, err = getOauthClient(oAuthClientName)
if err != nil {
logrus.Fatalf("oAuthclient %s not found", oAuthClientName)
}
logrus.Infof("Checking if oauthclient %s has been created", oAuthClientName)
// verify oathclient name is set in che ConfigMap
cm, err := getConfigMap("che")
if err != nil {
log.Fatalf("Failed to get ConfigMap: %s", err)
}
expectedIdentityProvider := "openshift-v3"
actualIdentityProvider := cm.Data["CHE_INFRA_OPENSHIFT_OAUTH__IDENTITY__PROVIDER"]
expectedWorkspaceProject := ""
actualWorkspaceProject := cm.Data["CHE_INFRA_OPENSHIFT_PROJECT"]
logrus.Info("Checking if identity provider is added to configmap")
if expectedIdentityProvider != actualIdentityProvider {
logrus.Fatalf("Test failed. Expecting identity provider: %s, got: %s", expectedIdentityProvider, actualIdentityProvider)
}
logrus.Info("Checking if workspace project is empty in CM")
if expectedWorkspaceProject != actualWorkspaceProject {
logrus.Fatalf("Test failed. Expecting identity provider: %s, got: %s", expectedWorkspaceProject, actualWorkspaceProject)
}
// cleanup
logrus.Infof("Tests passed. Deleting namespace %s", namespace)
if err := deleteNamespace(); err != nil {
logrus.Errorf("Failed to delete namespace %s: %s", namespace, err)
}
}

38
e2e/watch.go Normal file
View File

@ -0,0 +1,38 @@
//
// Copyright (c) 2012-2019 Red Hat, Inc.
// This program and the accompanying materials are made
// available under the terms of the Eclipse Public License 2.0
// which is available at https://www.eclipse.org/legal/epl-2.0/
//
// SPDX-License-Identifier: EPL-2.0
//
// Contributors:
// Red Hat, Inc. - initial API and implementation
//
package main
import (
"errors"
"time"
)
func VerifyCheRunning(status string) (deployed bool, err error) {
timeout := time.After(6 * time.Minute)
tick := time.Tick(10 * time.Second)
for {
select {
case <-timeout:
return false, errors.New("timed out")
case <-tick:
customResource, _ := getCR()
if customResource.Status.CheClusterRunning != status {
} else {
return true, nil
}
}
}
}

View File

@ -207,18 +207,20 @@ func (r *ReconcileChe) Reconcile(request reconcile.Request) (reconcile.Result, e
if err != nil {
logrus.Errorf("An error occurred when detecting current infra: %s", err)
}
// create a secret with router tls cert if self signed certs are in use
// requires cluster admin privileges
selfSignedCert := instance.Spec.Server.SelfSignedCert
if isOpenShift && selfSignedCert {
crt, err := k8sclient.GetDefaultRouterCert("openshift-ingress")
if err != nil {
logrus.Errorf("Default router tls secret not found. Self signed cert isn't added")
return reconcile.Result{}, err
} else {
secret := deploy.NewSecret(instance, "self-signed-certificate", crt)
if err := r.CreateNewSecret(instance, secret); err != nil {
// create a secret with router tls cert
if isOpenShift {
secret := &corev1.Secret{}
if err := r.client.Get(context.TODO(), types.NamespacedName{Name: "self-signed-certificate", Namespace: instance.Namespace}, secret);
err != nil && errors.IsNotFound(err) {
crt, err := r.GetRouterTlsCrt(instance)
if err != nil {
logrus.Errorf("Default router tls secret not found. Self signed cert isn't added")
return reconcile.Result{}, err
} else {
secret := deploy.NewSecret(instance, "self-signed-certificate", crt)
if err := r.CreateNewSecret(instance, secret); err != nil {
return reconcile.Result{}, err
}
}
}
}

View File

@ -13,6 +13,11 @@ package che
import (
"bytes"
"context"
"crypto/tls"
"encoding/pem"
orgv1 "github.com/eclipse/che-operator/pkg/apis/org/v1"
"github.com/eclipse/che-operator/pkg/deploy"
"github.com/eclipse/che-operator/pkg/util"
"github.com/sirupsen/logrus"
"io"
@ -22,6 +27,7 @@ import (
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"net/http"
"sigs.k8s.io/controller-runtime/pkg/client/config"
)
@ -183,24 +189,24 @@ func (cl *k8s) GetDeploymentStatus(name string, ns string) (scaled bool) {
}
}
}
dc, _ := cl.clientset.AppsV1().Deployments(ns).Get(name, metav1.GetOptions{})
if dc.Status.AvailableReplicas != 1 {
logrus.Errorf("Failed to verify a successful %s deployment", name)
eventList := cl.GetEvents(name, ns).Items
for i := range eventList {
logrus.Errorf("Event message: %v", eventList[i].Message)
}
deploymentPod, err := cl.GetDeploymentPod(name, ns)
if err != nil {
return false
}
cl.GetPodLogs(deploymentPod, ns)
logrus.Errorf("Command to get deployment logs: kubectl logs deployment/%s -n=%s", name, ns)
logrus.Errorf("Get k8s events: kubectl get events "+
"--field-selector "+
"involvedObject.name=$(kubectl get pods -l=component=%s -n=%s"+
" -o=jsonpath='{.items[0].metadata.name}') -n=%s", name, ns, ns)
dc, _ := cl.clientset.AppsV1().Deployments(ns).Get(name, metav1.GetOptions{})
if dc.Status.AvailableReplicas != 1 {
logrus.Errorf("Failed to verify a successful %s deployment", name)
eventList := cl.GetEvents(name, ns).Items
for i := range eventList {
logrus.Errorf("Event message: %v", eventList[i].Message)
}
deploymentPod, err := cl.GetDeploymentPod(name, ns)
if err != nil {
return false
}
cl.GetPodLogs(deploymentPod, ns)
logrus.Errorf("Command to get deployment logs: kubectl logs deployment/%s -n=%s", name, ns)
logrus.Errorf("Get k8s events: kubectl get events "+
"--field-selector "+
"involvedObject.name=$(kubectl get pods -l=component=%s -n=%s"+
" -o=jsonpath='{.items[0].metadata.name}') -n=%s", name, ns, ns)
return false
}
return true
}
@ -245,20 +251,42 @@ func (cl *k8s) GetDeploymentPod(name string, ns string) (podName string, err err
return podName, nil
}
// GetDefaultRouterCert retrieves secret with OpenShift router certificate and extracts it
// The cert is then used to create self-signed-certificate secret consumed by CheCluster server and workspaces
func (cl *k8s) GetDefaultRouterCert(ns string) (crt []byte, err error) {
options := metav1.GetOptions{}
secret, err := cl.clientset.CoreV1().Secrets(ns).Get("router-certs-default", options)
if err != nil {
// in 3.11 it's default namespace and router-certs secret
secret, err = cl.clientset.CoreV1().Secrets("default").Get("router-certs", options)
if err != nil {
logrus.Errorf("Failed to get a secret in both namespace %s and default: %s", ns, err)
return nil, err
}
// GetRouterTlsCrt creates a test TLS route and gets it to extract certificate chain
// There's an easier way which is to read tls secret in default (3.11) or openshift-ingress (4.0) namespace
// which however requires extra privileges for operator service account
func (r *ReconcileChe) GetRouterTlsCrt(instance *orgv1.CheCluster) (certificate []byte, err error) {
testRoute := deploy.NewTlsRoute(instance, "test", "test")
logrus.Infof("Creating a test route %s to extract routes crt", testRoute.Name)
if err := r.CreateNewRoute(instance, testRoute); err != nil {
logrus.Errorf("Failed to create test route %s: %s", testRoute.Name, err)
return nil, err
}
secretData := secret.Data
crt = secretData["tls.crt"]
return crt, nil
url := "https://" + testRoute.Spec.Host
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
client := &http.Client{}
req, err := http.NewRequest("GET", url, nil)
resp, err := client.Do(req)
if err != nil {
logrus.Errorf("An error occurred when reaching test TLS route: %s", err)
if r.tests {
fakeCrt := make([]byte, 5)
return fakeCrt, nil
}
return nil, err
}
for i := range resp.TLS.PeerCertificates {
cert := resp.TLS.PeerCertificates[i].Raw
crt := pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: cert,
})
certificate = append(certificate, crt...)
}
logrus.Infof("Deleting a test route %s to extract routes crt", testRoute.Name)
if err := r.client.Delete(context.TODO(), testRoute); err != nil {
logrus.Errorf("Failed to delete test route %s: %s", testRoute.Name, err)
}
return certificate, nil
}

View File

@ -48,7 +48,7 @@ func GetKeycloakProvisionCommand(cr *orgv1.CheCluster, cheHost string) (command
}
file, err := ioutil.ReadFile("/tmp/keycloak_provision")
if err != nil {
logrus.Errorf("Failed to find keycloak entrypoint file %s", err)
logrus.Errorf("Failed to locate keycloak entrypoint file: %s", err)
}
keycloakTheme := "che"
realmDisplayName := "Eclipse Che"

View File

@ -13,10 +13,11 @@ package deploy
import (
orgv1 "github.com/eclipse/che-operator/pkg/apis/org/v1"
"github.com/eclipse/che-operator/pkg/util"
)
func GetLabels(cr *orgv1.CheCluster, component string) (labels map[string]string) {
cheFlavor := cr.Spec.Server.CheFlavor
cheFlavor := util.GetValue(cr.Spec.Server.CheFlavor, DefaultCheFlavor)
labels = map[string]string{"app": cheFlavor, "component": component}
return labels
}