seg_yinzy
/
che-operator
mirror of https://github.com/eclipse-che/che-operator.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: Support node selector and pod tolerations in devworkspaces (#1301) 

Fixes #20884
* Upgrade to devworkspace operator with support for pod tolerations
* Implement additional logic in usernamespace controller to sync config
  from checluster CR to ns annotations understood by the dwo
* Add new fields to CheCluster CRD v1 and v2alpha1
* added support for the new fields in conversion methods between v1 and v2alpha1
pull/1308/head
Lukas Krejci 2022-02-01 17:51:35 +01:00 committed by GitHub
parent c1844de883
commit 12da7adeeb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 556 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
api/conversion.go
View File

@ -13,10 +13,14 @@
package org
import (
"encoding/json"
"strings"
v1 "github.com/eclipse-che/che-operator/api/v1"
"github.com/eclipse-che/che-operator/api/v2alpha1"
"github.com/eclipse-che/che-operator/pkg/deploy"
"github.com/eclipse-che/che-operator/pkg/util"
corev1 "k8s.io/api/core/v1"
"k8s.io/utils/pointer"
"sigs.k8s.io/yaml"
)
@ -72,6 +76,10 @@ func V1ToV2alpha1(v1 *v1.CheCluster, v2 *v2alpha1.CheCluster) error {
v1toV2alpha1_GatewayConfigLabels(v1, v2)
v1ToV2alpha1_WorkspaceDomainEndpointsBaseDomain(v1, v2)
v1ToV2alpha1_WorkspaceDomainEndpointsTlsSecretName(v1, v2)
v1ToV2alpha1_WorkspacePodNodeSelector(v1, v2)
if err := v1ToV2alpha1_WorkspacePodTolerations(v1, v2); err != nil {
return err
}
v1ToV2alpha1_K8sIngressAnnotations(v1, v2)
// we don't need to store the serialized v2 on a v2 object
@ -111,6 +119,8 @@ func V2alpha1ToV1(v2 *v2alpha1.CheCluster, v1Obj *v1.CheCluster) error {
v2alpha1ToV1_GatewayConfigLabels(v1Obj, v2)
v2alpha1ToV1_WorkspaceDomainEndpointsBaseDomain(v1Obj, v2)
v2alpha1ToV1_WorkspaceDomainEndpointsTlsSecretName(v1Obj, v2)
v2alpha1ToV1_WorkspacePodNodeSelector(v1Obj, v2)
v2alpha1ToV1_WorkspacePodTolerations(v1Obj, v2)
v2alpha1ToV1_K8sIngressAnnotations(v1Obj, v2)
// we don't need to store the serialized v1 on a v1 object
@ -129,9 +139,9 @@ func v1ToV2alpha1_Host(v1 *v1.CheCluster, v2 *v2alpha1.CheCluster) {
func v1ToV2alpha1_WorkspaceDomainEndpointsBaseDomain(v1 *v1.CheCluster, v2 *v2alpha1.CheCluster) {
if util.IsOpenShift {
v2.Spec.WorkspaceDomainEndpoints.BaseDomain = v1.Spec.Server.CustomCheProperties[routeDomainSuffixPropertyKey]
v2.Spec.Workspaces.DomainEndpoints.BaseDomain = v1.Spec.Server.CustomCheProperties[routeDomainSuffixPropertyKey]
} else {
v2.Spec.WorkspaceDomainEndpoints.BaseDomain = v1.Spec.K8s.IngressDomain
v2.Spec.Workspaces.DomainEndpoints.BaseDomain = v1.Spec.K8s.IngressDomain
}
}
@ -140,10 +150,46 @@ func v1ToV2alpha1_WorkspaceDomainEndpointsTlsSecretName(v1 *v1.CheCluster, v2 *v
// Because we're dealing with endpoints, let's try to use the secret on Kubernetes and nothing (e.g. the default cluster cert on OpenShift)
// which is in line with the logic of the Che server.
if !util.IsOpenShift {
v2.Spec.WorkspaceDomainEndpoints.TlsSecretName = v1.Spec.K8s.TlsSecretName
v2.Spec.Workspaces.DomainEndpoints.TlsSecretName = v1.Spec.K8s.TlsSecretName
}
}
func v1ToV2alpha1_WorkspacePodNodeSelector(v1 *v1.CheCluster, v2 *v2alpha1.CheCluster) {
selector := v1.Spec.Server.WorkspacePodNodeSelector
if len(selector) == 0 {
prop := v1.Spec.Server.CustomCheProperties["CHE_WORKSPACE_POD_NODE__SELECTOR"]
if prop != "" {
selector = map[string]string{}
kvs := strings.Split(prop, ",")
for _, pair := range kvs {
kv := strings.Split(pair, "=")
if len(kv) == 2 {
selector[kv[0]] = kv[1]
}
}
}
}
v2.Spec.Workspaces.PodNodeSelector = selector
}
func v1ToV2alpha1_WorkspacePodTolerations(v1 *v1.CheCluster, v2 *v2alpha1.CheCluster) error {
tolerations := v1.Spec.Server.WorkspacePodTolerations
if len(tolerations) == 0 {
prop := v1.Spec.Server.CustomCheProperties["CHE_WORKSPACE_POD_TOLERATIONS__JSON"]
if prop != "" {
tols := []corev1.Toleration{}
if err := json.Unmarshal([]byte(prop), &tols); err != nil {
return err
}
tolerations = tols
}
}
v2.Spec.Workspaces.PodTolerations = tolerations
return nil
}
func v1ToV2alpha1_GatewayEnabled(v1 *v1.CheCluster, v2 *v2alpha1.CheCluster) {
// On Kubernetes, we can have single-host realized using ingresses (that use the same host but different paths).
// This is actually not supported on DWCO where we always use the gateway for that. So here, we actually just
@ -197,21 +243,29 @@ func v2alpha1ToV1_WorkspaceDomainEndpointsBaseDomain(v1 *v1.CheCluster, v2 *v2al
if v1.Spec.Server.CustomCheProperties == nil {
v1.Spec.Server.CustomCheProperties = map[string]string{}
}
if len(v2.Spec.WorkspaceDomainEndpoints.BaseDomain) > 0 {
v1.Spec.Server.CustomCheProperties[routeDomainSuffixPropertyKey] = v2.Spec.WorkspaceDomainEndpoints.BaseDomain
if len(v2.Spec.Workspaces.DomainEndpoints.BaseDomain) > 0 {
v1.Spec.Server.CustomCheProperties[routeDomainSuffixPropertyKey] = v2.Spec.Workspaces.DomainEndpoints.BaseDomain
}
} else {
v1.Spec.K8s.IngressDomain = v2.Spec.WorkspaceDomainEndpoints.BaseDomain
v1.Spec.K8s.IngressDomain = v2.Spec.Workspaces.DomainEndpoints.BaseDomain
}
}
func v2alpha1ToV1_WorkspaceDomainEndpointsTlsSecretName(v1 *v1.CheCluster, v2 *v2alpha1.CheCluster) {
// see the comments in the v1 to v2alpha1 conversion method
if !util.IsOpenShift {
v1.Spec.K8s.TlsSecretName = v2.Spec.WorkspaceDomainEndpoints.TlsSecretName
v1.Spec.K8s.TlsSecretName = v2.Spec.Workspaces.DomainEndpoints.TlsSecretName
}
}
func v2alpha1ToV1_WorkspacePodNodeSelector(v1 *v1.CheCluster, v2 *v2alpha1.CheCluster) {
v1.Spec.Server.WorkspacePodNodeSelector = v2.Spec.Workspaces.PodNodeSelector
}
func v2alpha1ToV1_WorkspacePodTolerations(v1 *v1.CheCluster, v2 *v2alpha1.CheCluster) {
v1.Spec.Server.WorkspacePodTolerations = v2.Spec.Workspaces.PodTolerations
}
func v2alpha1ToV1_GatewayImage(v1 *v1.CheCluster, v2 *v2alpha1.CheCluster) {
v1.Spec.Server.SingleHostGatewayImage = v2.Spec.Gateway.Image
}

85
api/conversion_test.go
View File

@ -13,9 +13,12 @@
package org
import (
"encoding/json"
"reflect"
"testing"
corev1 "k8s.io/api/core/v1"
"github.com/stretchr/testify/assert"
"github.com/che-incubator/kubernetes-image-puller-operator/api/v1alpha1"
@ -30,6 +33,19 @@ import (
)
func TestV1ToV2alpha1(t *testing.T) {
tolerations := []corev1.Toleration{
{
Key: "a",
Operator: corev1.TolerationOpEqual,
Value: "b",
},
}
tolBytes, err := json.Marshal(tolerations)
assert.NoError(t, err)
tolerationStr := string(tolBytes)
v1Obj := v1.CheCluster{
ObjectMeta: metav1.ObjectMeta{
Name: "che-cluster",
@ -71,6 +87,8 @@ func TestV1ToV2alpha1(t *testing.T) {
},
CustomCheProperties: map[string]string{
"CHE_INFRA_OPENSHIFT_ROUTE_HOST_DOMAIN__SUFFIX": "routeDomain",
"CHE_WORKSPACE_POD_TOLERATIONS__JSON": tolerationStr,
"CHE_WORKSPACE_POD_NODE__SELECTOR": "a=b,c=d",
},
},
Storage: v1.CheClusterSpecStorage{
@ -144,8 +162,8 @@ func TestV1ToV2alpha1(t *testing.T) {
t.Error(err)
}
if v2.Spec.WorkspaceDomainEndpoints.BaseDomain != "ingressDomain" {
t.Errorf("Unexpected v2.Spec.WorkspaceDomainEndpoints.BaseDomain: %s", v2.Spec.WorkspaceDomainEndpoints.BaseDomain)
if v2.Spec.Workspaces.DomainEndpoints.BaseDomain != "ingressDomain" {
t.Errorf("Unexpected v2.Spec.Workspaces.DomainEndpoints.BaseDomain: %s", v2.Spec.Workspaces.DomainEndpoints.BaseDomain)
}
})
})
@ -158,8 +176,8 @@ func TestV1ToV2alpha1(t *testing.T) {
t.Error(err)
}
if v2.Spec.WorkspaceDomainEndpoints.BaseDomain != "routeDomain" {
t.Errorf("Unexpected v2.Spec.WorkspaceWorkspaceDomainEndpoints.BaseDomainBaseDomain: %s", v2.Spec.WorkspaceDomainEndpoints.BaseDomain)
if v2.Spec.Workspaces.DomainEndpoints.BaseDomain != "routeDomain" {
t.Errorf("Unexpected v2.Spec.Workspaces.DomainEndpoints.BaseDomain: %s", v2.Spec.Workspaces.DomainEndpoints.BaseDomain)
}
})
})
@ -172,7 +190,7 @@ func TestV1ToV2alpha1(t *testing.T) {
t.Error(err)
}
if v2.Spec.WorkspaceDomainEndpoints.TlsSecretName != "k8sSecret" {
if v2.Spec.Workspaces.DomainEndpoints.TlsSecretName != "k8sSecret" {
t.Errorf("Unexpected TlsSecretName")
}
})
@ -186,7 +204,7 @@ func TestV1ToV2alpha1(t *testing.T) {
t.Error(err)
}
if v2.Spec.WorkspaceDomainEndpoints.TlsSecretName != "" {
if v2.Spec.Workspaces.DomainEndpoints.TlsSecretName != "" {
t.Errorf("Unexpected TlsSecretName")
}
})
@ -246,6 +264,18 @@ func TestV1ToV2alpha1(t *testing.T) {
t.Errorf("Unexpected Spec.Gateway.ConfigLabels: %v", cmp.Diff(v1Obj.Spec.Server.SingleHostGatewayConfigMapLabels, v2.Spec.Gateway.ConfigLabels))
}
})
t.Run("WorkspacePodSelector", func(t *testing.T) {
v2 := &v2alpha1.CheCluster{}
assert.NoError(t, V1ToV2alpha1(&v1Obj, v2))
assert.Equal(t, map[string]string{"a": "b", "c": "d"}, v2.Spec.Workspaces.PodNodeSelector)
})
t.Run("WorkspacePodTolerations", func(t *testing.T) {
v2 := &v2alpha1.CheCluster{}
assert.NoError(t, V1ToV2alpha1(&v1Obj, v2))
assert.Equal(t, tolerations, v2.Spec.Workspaces.PodTolerations)
})
}
func TestV2alpha1ToV1(t *testing.T) {
@ -259,10 +289,20 @@ func TestV2alpha1ToV1(t *testing.T) {
},
Spec: v2alpha1.CheClusterSpec{
Enabled: pointer.BoolPtr(true),
WorkspaceDomainEndpoints: v2alpha1.WorkspaceDomainEndpoints{
Workspaces: v2alpha1.Workspaces{
DomainEndpoints: v2alpha1.DomainEndpoints{
BaseDomain: "baseDomain",
TlsSecretName: "workspaceSecret",
},
PodNodeSelector: map[string]string{"a": "b", "c": "d"},
PodTolerations: []corev1.Toleration{
{
Key: "a",
Operator: corev1.TolerationOpEqual,
Value: "b",
},
},
},
Gateway: v2alpha1.CheGatewaySpec{
Host: "v2Host",
Enabled: pointer.BoolPtr(true),
@ -369,7 +409,7 @@ func TestV2alpha1ToV1(t *testing.T) {
onFakeOpenShift(func() {
v1 := &v1.CheCluster{}
v2apha := v2Obj.DeepCopy()
v2apha.Spec.WorkspaceDomainEndpoints.BaseDomain = ""
v2apha.Spec.Workspaces.DomainEndpoints.BaseDomain = ""
err := V2alpha1ToV1(v2apha, v1)
if err != nil {
t.Error(err)
@ -454,6 +494,23 @@ func TestV2alpha1ToV1(t *testing.T) {
t.Errorf("Unexpected SingleHostGatewayConfigMapLabels: %s", v1.Spec.Server.SingleHostGatewayConfigMapLabels)
}
})
t.Run("WorkspacePodNodeSelector", func(t *testing.T) {
v1 := &v1.CheCluster{}
assert.NoError(t, V2alpha1ToV1(&v2Obj, v1))
assert.Equal(t, map[string]string{"a": "b", "c": "d"}, v1.Spec.Server.WorkspacePodNodeSelector)
})
t.Run("WorkspacePodTolerations", func(t *testing.T) {
v1 := &v1.CheCluster{}
assert.NoError(t, V2alpha1ToV1(&v2Obj, v1))
assert.Equal(t, []corev1.Toleration{{
Key: "a",
Operator: corev1.TolerationOpEqual,
Value: "b",
}}, v1.Spec.Server.WorkspacePodTolerations)
})
}
func TestFullCircleV1(t *testing.T) {
@ -506,10 +563,10 @@ func TestFullCircleV1(t *testing.T) {
}
v2Obj := v2alpha1.CheCluster{}
V1ToV2alpha1(&v1Obj, &v2Obj)
assert.NoError(t, V1ToV2alpha1(&v1Obj, &v2Obj))
convertedV1 := v1.CheCluster{}
V2alpha1ToV1(&v2Obj, &convertedV1)
assert.NoError(t, V2alpha1ToV1(&v2Obj, &convertedV1))
assert.Empty(t, convertedV1.Annotations[v1StorageAnnotation])
assert.NotEmpty(t, convertedV1.Annotations[v2alpha1StorageAnnotation])
@ -531,10 +588,12 @@ func TestFullCircleV2(t *testing.T) {
},
Spec: v2alpha1.CheClusterSpec{
Enabled: pointer.BoolPtr(true),
WorkspaceDomainEndpoints: v2alpha1.WorkspaceDomainEndpoints{
Workspaces: v2alpha1.Workspaces{
DomainEndpoints: v2alpha1.DomainEndpoints{
BaseDomain: "baseDomain",
TlsSecretName: "workspaceSecret",
},
},
Gateway: v2alpha1.CheGatewaySpec{
Host: "v2Host",
Enabled: pointer.BoolPtr(true),
@ -555,10 +614,10 @@ func TestFullCircleV2(t *testing.T) {
}
v1Obj := v1.CheCluster{}
V2alpha1ToV1(&v2Obj, &v1Obj)
assert.NoError(t, V2alpha1ToV1(&v2Obj, &v1Obj))
convertedV2 := v2alpha1.CheCluster{}
V1ToV2alpha1(&v1Obj, &convertedV2)
assert.NoError(t, V1ToV2alpha1(&v1Obj, &convertedV2))
assert.Empty(t, convertedV2.Annotations[v2alpha1StorageAnnotation])
assert.NotEmpty(t, convertedV2.Annotations[v1StorageAnnotation])

4
api/v1/checluster_types.go
View File

@ -353,6 +353,10 @@ type CheClusterSpecServer struct {
// Default plug-ins applied to Devworkspaces.
// +optional
WorkspacesDefaultPlugins []WorkspacesDefaultPlugins `json:"workspacesDefaultPlugins,omitempty"`
// The node selector that limits the nodes that can run the workspace pods.
WorkspacePodNodeSelector map[string]string `json:"workspacePodNodeSelector,omitempty"`
// The pod tolerations put on the workspace pods to limit where the workspace pods can run.
WorkspacePodTolerations []corev1.Toleration `json:"workspacePodTolerations,omitempty"`
}
// +k8s:openapi-gen=true

15
api/v1/zz_generated.deepcopy.go
View File

@ -17,6 +17,7 @@
package v1
import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
runtime "k8s.io/apimachinery/pkg/runtime"
)
@ -255,6 +256,20 @@ func (in *CheClusterSpecServer) DeepCopyInto(out *CheClusterSpecServer) {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.WorkspacePodNodeSelector != nil {
in, out := &in.WorkspacePodNodeSelector, &out.WorkspacePodNodeSelector
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.WorkspacePodTolerations != nil {
in, out := &in.WorkspacePodTolerations, &out.WorkspacePodTolerations
*out = make([]corev1.Toleration, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CheClusterSpecServer.

20
api/v2alpha1/checluster_types.go
View File

@ -13,6 +13,7 @@
package v2alpha1
import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
)
@ -23,9 +24,8 @@ type CheClusterSpec struct {
// If false, Che is disabled and does not resolve the devworkspaces with the che routingClass.
Enabled *bool `json:"enabled,omitempty"`
// Configuration of the workspace endpoints that are exposed on separate domains, as opposed to the subpaths
// of the gateway.
WorkspaceDomainEndpoints WorkspaceDomainEndpoints `json:"workspaceDomainEndpoints,omitempty"`
// Workspaces contains configuration affecting the behavior of workspaces.
Workspaces Workspaces `json:"workspaces"`
// Gateway contains the configuration of the gateway used for workspace endpoint routing.
Gateway CheGatewaySpec `json:"gateway,omitempty"`
@ -34,7 +34,19 @@ type CheClusterSpec struct {
K8s CheClusterSpecK8s `json:"k8s,omitempty"`
}
type WorkspaceDomainEndpoints struct {
type Workspaces struct {
// Configuration of the workspace endpoints that are exposed on separate domains, as opposed to the subpaths
// of the gateway.
DomainEndpoints DomainEndpoints `json:"domainEndpoints,omitempty"`
// The node selector that limits the nodes that can run the workspace pods.
PodNodeSelector map[string]string `json:"podNodeSelector,omitempty"`
// The pod tolerations put on the workspace pods to limit where the workspace pods can run.
PodTolerations []corev1.Toleration `json:"podTolerations,omitempty"`
}
type DomainEndpoints struct {
// The workspace endpoints that need to be deployed on a subdomain will be deployed on subdomains of this base domain.
// This is mandatory on Kubernetes. On OpenShift, an attempt is made to automatically figure out the base domain of
// the routes. The resolved value of this property is written to the status.

41
api/v2alpha1/zz_generated.deepcopy.go
View File

@ -17,6 +17,7 @@
package v2alpha1
import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
runtime "k8s.io/apimachinery/pkg/runtime"
)
@ -88,7 +89,7 @@ func (in *CheClusterSpec) DeepCopyInto(out *CheClusterSpec) {
*out = new(bool)
**out = **in
}
out.WorkspaceDomainEndpoints = in.WorkspaceDomainEndpoints
in.Workspaces.DeepCopyInto(&out.Workspaces)
in.Gateway.DeepCopyInto(&out.Gateway)
in.K8s.DeepCopyInto(&out.K8s)
}
@ -168,16 +169,46 @@ func (in *CheGatewaySpec) DeepCopy() *CheGatewaySpec {
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *WorkspaceDomainEndpoints) DeepCopyInto(out *WorkspaceDomainEndpoints) {
func (in *DomainEndpoints) DeepCopyInto(out *DomainEndpoints) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkspaceDomainEndpoints.
func (in *WorkspaceDomainEndpoints) DeepCopy() *WorkspaceDomainEndpoints {
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DomainEndpoints.
func (in *DomainEndpoints) DeepCopy() *DomainEndpoints {
if in == nil {
return nil
}
out := new(WorkspaceDomainEndpoints)
out := new(DomainEndpoints)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Workspaces) DeepCopyInto(out *Workspaces) {
*out = *in
out.DomainEndpoints = in.DomainEndpoints
if in.PodNodeSelector != nil {
in, out := &in.PodNodeSelector, &out.PodNodeSelector
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.PodTolerations != nil {
in, out := &in.PodTolerations, &out.PodTolerations
*out = make([]v1.Toleration, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Workspaces.
func (in *Workspaces) DeepCopy() *Workspaces {
if in == nil {
return nil
}
out := new(Workspaces)
in.DeepCopyInto(out)
return out
}

4
bundle/next/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml
View File

@ -75,7 +75,7 @@ metadata:
operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
repository: https://github.com/eclipse-che/che-operator
support: Eclipse Foundation
name: eclipse-che-preview-openshift.v7.43.0-419.next
name: eclipse-che-preview-openshift.v7.43.0-420.next
namespace: placeholder
spec:
apiservicedefinitions: {}
@ -1288,4 +1288,4 @@ spec:
maturity: stable
provider:
name: Eclipse Foundation
version: 7.43.0-419.next
version: 7.43.0-420.next

49
bundle/next/eclipse-che-preview-openshift/manifests/org_v1_che_crd.yaml
View File

@ -1019,6 +1019,55 @@ spec:
placeholders, such as che-workspace-<username>. In that case,
a new namespace will be created for each user or workspace.
type: string
workspacePodNodeSelector:
additionalProperties:
type: string
description: The node selector that limits the nodes that can
run the workspace pods.
type: object
workspacePodTolerations:
description: The pod tolerations put on the workspace pods to
limit where the workspace pods can run.
items:
description: The pod this Toleration is attached to tolerates
any taint that matches the triple <key,value,effect> using
the matching operator <operator>.
properties:
effect:
description: Effect indicates the taint effect to match.
Empty means match all taint effects. When specified,
allowed values are NoSchedule, PreferNoSchedule and
NoExecute.
type: string
key:
description: Key is the taint key that the toleration
applies to. Empty means match all taint keys. If the
key is empty, operator must be Exists; this combination
means to match all values and all keys.
type: string
operator:
description: Operator represents a key's relationship
to the value. Valid operators are Exists and Equal.
Defaults to Equal. Exists is equivalent to wildcard
for value, so that a pod can tolerate all taints of
a particular category.
type: string
tolerationSeconds:
description: TolerationSeconds represents the period of
time the toleration (which must be of effect NoExecute,
otherwise this field is ignored) tolerates the taint.
By default, it is not set, which means tolerate the
taint forever (do not evict). Zero and negative values
will be treated as 0 (evict immediately) by the system.
format: int64
type: integer
value:
description: Value is the taint value the toleration matches
to. If the operator is Exists, the value should be empty,
otherwise just a regular string.
type: string
type: object
type: array
workspacesDefaultPlugins:
description: Default plug-ins applied to Devworkspaces.
items:

47
config/crd/bases/org_v1_che_crd-v1beta1.yaml
View File

@ -980,6 +980,53 @@ spec:
placeholders, such as che-workspace-<username>. In that case,
a new namespace will be created for each user or workspace.
type: string
workspacePodNodeSelector:
additionalProperties:
type: string
description: The node selector that limits the nodes that can run
the workspace pods.
type: object
workspacePodTolerations:
description: The pod tolerations put on the workspace pods to limit
where the workspace pods can run.
items:
description: The pod this Toleration is attached to tolerates
any taint that matches the triple <key,value,effect> using the
matching operator <operator>.
properties:
effect:
description: Effect indicates the taint effect to match. Empty
means match all taint effects. When specified, allowed values
are NoSchedule, PreferNoSchedule and NoExecute.
type: string
key:
description: Key is the taint key that the toleration applies
to. Empty means match all taint keys. If the key is empty,
operator must be Exists; this combination means to match
all values and all keys.
type: string
operator:
description: Operator represents a key's relationship to the
value. Valid operators are Exists and Equal. Defaults to
Equal. Exists is equivalent to wildcard for value, so that
a pod can tolerate all taints of a particular category.
type: string
tolerationSeconds:
description: TolerationSeconds represents the period of time
the toleration (which must be of effect NoExecute, otherwise
this field is ignored) tolerates the taint. By default,
it is not set, which means tolerate the taint forever (do
not evict). Zero and negative values will be treated as
0 (evict immediately) by the system.
format: int64
type: integer
value:
description: Value is the taint value the toleration matches
to. If the operator is Exists, the value should be empty,
otherwise just a regular string.
type: string
type: object
type: array
workspacesDefaultPlugins:
description: Default plug-ins applied to Devworkspaces.
items:

49
config/crd/bases/org_v1_che_crd.yaml
View File

@ -1015,6 +1015,55 @@ spec:
placeholders, such as che-workspace-<username>. In that case,
a new namespace will be created for each user or workspace.
type: string
workspacePodNodeSelector:
additionalProperties:
type: string
description: The node selector that limits the nodes that can
run the workspace pods.
type: object
workspacePodTolerations:
description: The pod tolerations put on the workspace pods to
limit where the workspace pods can run.
items:
description: The pod this Toleration is attached to tolerates
any taint that matches the triple <key,value,effect> using
the matching operator <operator>.
properties:
effect:
description: Effect indicates the taint effect to match.
Empty means match all taint effects. When specified,
allowed values are NoSchedule, PreferNoSchedule and
NoExecute.
type: string
key:
description: Key is the taint key that the toleration
applies to. Empty means match all taint keys. If the
key is empty, operator must be Exists; this combination
means to match all values and all keys.
type: string
operator:
description: Operator represents a key's relationship
to the value. Valid operators are Exists and Equal.
Defaults to Equal. Exists is equivalent to wildcard
for value, so that a pod can tolerate all taints of
a particular category.
type: string
tolerationSeconds:
description: TolerationSeconds represents the period of
time the toleration (which must be of effect NoExecute,
otherwise this field is ignored) tolerates the taint.
By default, it is not set, which means tolerate the
taint forever (do not evict). Zero and negative values
will be treated as 0 (evict immediately) by the system.
format: int64
type: integer
value:
description: Value is the taint value the toleration matches
to. If the operator is Exists, the value should be empty,
otherwise just a regular string.
type: string
type: object
type: array
workspacesDefaultPlugins:
description: Default plug-ins applied to Devworkspaces.
items:

4
controllers/devworkspace/controller.go
View File

@ -209,7 +209,7 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
// control of gateway creation
changed = false
workspaceBaseDomain := current.Spec.WorkspaceDomainEndpoints.BaseDomain
workspaceBaseDomain := current.Spec.Workspaces.DomainEndpoints.BaseDomain
if workspaceBaseDomain == "" {
workspaceBaseDomain, err = r.detectOpenShiftRouteBaseDomain(current)
@ -277,7 +277,7 @@ func (r *CheClusterReconciler) validate(cluster *v2alpha1.CheCluster) error {
if !util.IsOpenShift {
// The validation error messages must correspond to the storage version of the resource, which is currently
// v1...
if cluster.Spec.WorkspaceDomainEndpoints.BaseDomain == "" {
if cluster.Spec.Workspaces.DomainEndpoints.BaseDomain == "" {
validationErrors = append(validationErrors, "spec.k8s.ingressDomain must be specified")
}
}

24
controllers/devworkspace/controller_test.go
View File

@ -100,10 +100,12 @@ func TestNoCustomResourceSharedWhenReconcilingNonExistent(t *testing.T) {
Host: "over.the.rainbow",
Enabled: pointer.BoolPtr(false),
},
WorkspaceDomainEndpoints: v2alpha1.WorkspaceDomainEndpoints{
Workspaces: v2alpha1.Workspaces{
DomainEndpoints: v2alpha1.DomainEndpoints{
BaseDomain: "down.on.earth",
},
},
},
}))
_, err = reconciler.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Name: managerName, Namespace: ns}})
@ -137,10 +139,12 @@ func TestAddsCustomResourceToSharedMapOnCreate(t *testing.T) {
Host: "over.the.rainbow",
Enabled: pointer.BoolPtr(false),
},
WorkspaceDomainEndpoints: v2alpha1.WorkspaceDomainEndpoints{
Workspaces: v2alpha1.Workspaces{
DomainEndpoints: v2alpha1.DomainEndpoints{
BaseDomain: "down.on.earth",
},
},
},
}))
reconciler := CheClusterReconciler{client: cl, scheme: scheme, syncer: sync.New(cl, scheme)}
@ -186,10 +190,12 @@ func TestUpdatesCustomResourceInSharedMapOnUpdate(t *testing.T) {
Enabled: pointer.BoolPtr(false),
Host: "over.the.rainbow",
},
WorkspaceDomainEndpoints: v2alpha1.WorkspaceDomainEndpoints{
Workspaces: v2alpha1.Workspaces{
DomainEndpoints: v2alpha1.DomainEndpoints{
BaseDomain: "down.on.earth",
},
},
},
}))
reconciler := CheClusterReconciler{client: cl, scheme: scheme, syncer: sync.New(cl, scheme)}
@ -288,10 +294,12 @@ func TestRemovesCustomResourceFromSharedMapOnDelete(t *testing.T) {
Host: "over.the.rainbow",
Enabled: pointer.BoolPtr(false),
},
WorkspaceDomainEndpoints: v2alpha1.WorkspaceDomainEndpoints{
Workspaces: v2alpha1.Workspaces{
DomainEndpoints: v2alpha1.DomainEndpoints{
BaseDomain: "down.on.earth",
},
},
},
}))
reconciler := CheClusterReconciler{client: cl, scheme: scheme, syncer: sync.New(cl, scheme)}
@ -347,10 +355,12 @@ func TestCustomResourceFinalization(t *testing.T) {
Gateway: v2alpha1.CheGatewaySpec{
Host: "over.the.rainbow",
},
WorkspaceDomainEndpoints: v2alpha1.WorkspaceDomainEndpoints{
Workspaces: v2alpha1.Workspaces{
DomainEndpoints: v2alpha1.DomainEndpoints{
BaseDomain: "down.on.earth",
},
},
},
}),
&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
@ -457,10 +467,12 @@ func TestExternalGatewayDetection(t *testing.T) {
Namespace: ns,
},
Spec: v2alpha1.CheClusterSpec{
WorkspaceDomainEndpoints: v2alpha1.WorkspaceDomainEndpoints{
Workspaces: v2alpha1.Workspaces{
DomainEndpoints: v2alpha1.DomainEndpoints{
BaseDomain: "down.on.earth",
},
},
},
}
onKubernetes(func() {

3
controllers/devworkspace/solver/che_routing.go
View File

@ -19,13 +19,14 @@ import (
"strconv"
"strings"
dw "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
"github.com/eclipse-che/che-operator/pkg/util"
"github.com/eclipse-che/che-operator/pkg/deploy/gateway"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/util/intstr"
dw "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
dwo "github.com/devfile/devworkspace-operator/apis/controller/v1alpha1"
"github.com/devfile/devworkspace-operator/controllers/controller/devworkspacerouting/solvers"
"github.com/devfile/devworkspace-operator/pkg/common"

16
controllers/devworkspace/solver/che_routing_test.go
View File

@ -113,10 +113,12 @@ func getSpecObjects(t *testing.T, routing *dwo.DevWorkspaceRouting) (client.Clie
Gateway: v2alpha1.CheGatewaySpec{
Host: "over.the.rainbow",
},
WorkspaceDomainEndpoints: v2alpha1.WorkspaceDomainEndpoints{
Workspaces: v2alpha1.Workspaces{
DomainEndpoints: v2alpha1.DomainEndpoints{
BaseDomain: "down.on.earth",
},
},
},
}, routing)
}
@ -756,9 +758,11 @@ func TestUsesIngressAnnotationsForWorkspaceEndpointIngresses(t *testing.T) {
Gateway: v2alpha1.CheGatewaySpec{
Host: "over.the.rainbow",
},
WorkspaceDomainEndpoints: v2alpha1.WorkspaceDomainEndpoints{
Workspaces: v2alpha1.Workspaces{
DomainEndpoints: v2alpha1.DomainEndpoints{
BaseDomain: "down.on.earth",
},
},
K8s: v2alpha1.CheClusterSpecK8s{
IngressAnnotations: map[string]string{
"a": "b",
@ -798,11 +802,13 @@ func TestUsesCustomCertificateForWorkspaceEndpointIngresses(t *testing.T) {
Gateway: v2alpha1.CheGatewaySpec{
Host: "beyond.comprehension",
},
WorkspaceDomainEndpoints: v2alpha1.WorkspaceDomainEndpoints{
Workspaces: v2alpha1.Workspaces{
DomainEndpoints: v2alpha1.DomainEndpoints{
BaseDomain: "almost.trivial",
TlsSecretName: "tlsSecret",
},
},
},
}
_, _, objs := getSpecObjectsForManager(t, mgr, subdomainDevWorkspaceRouting(), &corev1.Secret{
@ -877,11 +883,13 @@ func TestUsesCustomCertificateForWorkspaceEndpointRoutes(t *testing.T) {
Gateway: v2alpha1.CheGatewaySpec{
Host: "beyond.comprehension",
},
WorkspaceDomainEndpoints: v2alpha1.WorkspaceDomainEndpoints{
Workspaces: v2alpha1.Workspaces{
DomainEndpoints: v2alpha1.DomainEndpoints{
BaseDomain: "almost.trivial",
TlsSecretName: "tlsSecret",
},
},
},
}
_, _, objs := getSpecObjectsForManager(t, mgr, subdomainDevWorkspaceRouting(), &corev1.Secret{

8
controllers/devworkspace/solver/endpoint_exposer.go
View File

@ -66,9 +66,9 @@ func (e *RouteExposer) initFrom(ctx context.Context, cl client.Client, cluster *
e.baseDomain = cluster.Status.WorkspaceBaseDomain
e.devWorkspaceID = routing.Spec.DevWorkspaceId
if cluster.Spec.WorkspaceDomainEndpoints.TlsSecretName != "" {
if cluster.Spec.Workspaces.DomainEndpoints.TlsSecretName != "" {
secret := &corev1.Secret{}
err := cl.Get(ctx, client.ObjectKey{Name: cluster.Spec.WorkspaceDomainEndpoints.TlsSecretName, Namespace: cluster.Namespace}, secret)
err := cl.Get(ctx, client.ObjectKey{Name: cluster.Spec.Workspaces.DomainEndpoints.TlsSecretName, Namespace: cluster.Namespace}, secret)
if err != nil {
return err
}
@ -85,7 +85,7 @@ func (e *IngressExposer) initFrom(ctx context.Context, cl client.Client, cluster
e.devWorkspaceID = routing.Spec.DevWorkspaceId
e.ingressAnnotations = ingressAnnotations
if cluster.Spec.WorkspaceDomainEndpoints.TlsSecretName != "" {
if cluster.Spec.Workspaces.DomainEndpoints.TlsSecretName != "" {
tlsSecretName := routing.Spec.DevWorkspaceId + "-endpoints"
e.tlsSecretName = tlsSecretName
@ -95,7 +95,7 @@ func (e *IngressExposer) initFrom(ctx context.Context, cl client.Client, cluster
err := cl.Get(ctx, client.ObjectKey{Name: tlsSecretName, Namespace: routing.Namespace}, secret)
if errors.IsNotFound(err) {
secret = &corev1.Secret{}
err = cl.Get(ctx, client.ObjectKey{Name: cluster.Spec.WorkspaceDomainEndpoints.TlsSecretName, Namespace: cluster.Namespace}, secret)
err = cl.Get(ctx, client.ObjectKey{Name: cluster.Spec.Workspaces.DomainEndpoints.TlsSecretName, Namespace: cluster.Namespace}, secret)
if err != nil {
return err
}

68
controllers/usernamespace/controller.go
View File

@ -14,6 +14,9 @@ package usernamespace
import (
"context"
"encoding/json"
"github.com/devfile/devworkspace-operator/pkg/infrastructure"
"github.com/eclipse-che/che-operator/pkg/deploy/tls"
"github.com/eclipse-che/che-operator/pkg/util"
@ -42,6 +45,10 @@ import (
const (
userSettingsComponentLabelValue = "user-settings"
// we're define these here because we're forced to use an older version
// of devworkspace operator as our dependency due to different go version
nodeSelectorAnnotation = "controller.devfile.io/node-selector"
podTolerationsAnnotation = "controller.devfile.io/pod-tolerations"
)
type CheUserNamespaceReconciler struct {
@ -233,6 +240,11 @@ func (r *CheUserNamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Req
return ctrl.Result{}, err
}
if err = r.reconcileNodeSelectorAndTolerations(ctx, req.Name, checluster, deployContext); err != nil {
logrus.Errorf("Failed to reconcile the workspace pod node selector and tolerations in namespace '%s': %v", req.Name, err)
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
@ -473,6 +485,62 @@ func (r *CheUserNamespaceReconciler) reconcileGitTlsCertificate(ctx context.Cont
return err
}
func (r *CheUserNamespaceReconciler) reconcileNodeSelectorAndTolerations(ctx context.Context, targetNs string, checluster *v2alpha1.CheCluster, deployContext *deploy.DeployContext) error {
var ns client.Object
if infrastructure.IsOpenShift() {
ns = &projectv1.Project{}
} else {
ns = &corev1.Namespace{}
}
if err := r.client.Get(ctx, client.ObjectKey{Name: targetNs}, ns); err != nil {
return err
}
nodeSelector := ""
tolerations := ""
if len(checluster.Spec.Workspaces.PodNodeSelector) != 0 {
serialized, err := json.Marshal(checluster.Spec.Workspaces.PodNodeSelector)
if err != nil {
return err
}
nodeSelector = string(serialized)
}
if len(checluster.Spec.Workspaces.PodTolerations) != 0 {
serialized, err := json.Marshal(checluster.Spec.Workspaces.PodTolerations)
if err != nil {
return err
}
tolerations = string(serialized)
}
annos := ns.GetAnnotations()
if annos == nil {
annos = map[string]string{}
}
if len(nodeSelector) == 0 {
delete(annos, nodeSelectorAnnotation)
} else {
annos[nodeSelectorAnnotation] = nodeSelector
}
if len(tolerations) == 0 {
delete(annos, podTolerationsAnnotation)
} else {
annos[podTolerationsAnnotation] = tolerations
}
ns.SetAnnotations(annos)
return r.client.Update(ctx, ns)
}
func prefixedName(checluster *v2alpha1.CheCluster, name string) string {
return checluster.Name + "-" + checluster.Namespace + "-" + name
}

35
controllers/usernamespace/controller_test.go
View File

@ -14,6 +14,7 @@ package usernamespace
import (
"context"
"encoding/json"
"sync"
"testing"
@ -63,6 +64,19 @@ func setupCheCluster(t *testing.T, ctx context.Context, cl client.Client, scheme
CustomCheProperties: map[string]string{
"CHE_INFRA_OPENSHIFT_ROUTE_HOST_DOMAIN__SUFFIX": "root-domain",
},
WorkspacePodNodeSelector: map[string]string{"a": "b", "c": "d"},
WorkspacePodTolerations: []corev1.Toleration{
{
Key: "a",
Operator: corev1.TolerationOpEqual,
Value: "b",
},
{
Key: "c",
Operator: corev1.TolerationOpEqual,
Value: "d",
},
},
},
DevWorkspace: v1.CheClusterSpecDevWorkspace{
Enable: true,
@ -273,7 +287,21 @@ func TestMatchingCheClusterCanBeSelectedUsingLabels(t *testing.T) {
}
func TestCreatesDataInNamespace(t *testing.T) {
test := func(t *testing.T, infraType infrastructure.Type, namespace metav1.Object, objs ...runtime.Object) {
expectedPodTolerations, err := json.Marshal([]corev1.Toleration{
{
Key: "a",
Operator: corev1.TolerationOpEqual,
Value: "b",
},
{
Key: "c",
Operator: corev1.TolerationOpEqual,
Value: "d",
},
})
assert.NoError(t, err)
test := func(t *testing.T, infraType infrastructure.Type, namespace client.Object, objs ...runtime.Object) {
ctx := context.TODO()
allObjs := append(objs, namespace.(runtime.Object))
scheme, cl, r := setup(infraType, allObjs...)
@ -323,6 +351,11 @@ func TestCreatesDataInNamespace(t *testing.T) {
assert.Equal(t, "true", gitTlsConfig.Labels[constants.DevWorkspaceWatchConfigMapLabel])
assert.Equal(t, "the.host.of.git", gitTlsConfig.Data["host"])
assert.Equal(t, "the public certificate of the.host.of.git", gitTlsConfig.Data["certificate"])
updatedNs := namespace.DeepCopyObject().(client.Object)
assert.NoError(t, cl.Get(ctx, client.ObjectKeyFromObject(namespace), updatedNs))
assert.Equal(t, `{"a":"b","c":"d"}`, updatedNs.GetAnnotations()[nodeSelectorAnnotation])
assert.Equal(t, string(expectedPodTolerations), updatedNs.GetAnnotations()[podTolerationsAnnotation])
}
t.Run("k8s", func(t *testing.T) {

49
helmcharts/next/crds/org_v1_che_crd.yaml
View File

@ -1015,6 +1015,55 @@ spec:
placeholders, such as che-workspace-<username>. In that case,
a new namespace will be created for each user or workspace.
type: string
workspacePodNodeSelector:
additionalProperties:
type: string
description: The node selector that limits the nodes that can
run the workspace pods.
type: object
workspacePodTolerations:
description: The pod tolerations put on the workspace pods to
limit where the workspace pods can run.
items:
description: The pod this Toleration is attached to tolerates
any taint that matches the triple <key,value,effect> using
the matching operator <operator>.
properties:
effect:
description: Effect indicates the taint effect to match.
Empty means match all taint effects. When specified,
allowed values are NoSchedule, PreferNoSchedule and
NoExecute.
type: string
key:
description: Key is the taint key that the toleration
applies to. Empty means match all taint keys. If the
key is empty, operator must be Exists; this combination
means to match all values and all keys.
type: string
operator:
description: Operator represents a key's relationship
to the value. Valid operators are Exists and Equal.
Defaults to Equal. Exists is equivalent to wildcard
for value, so that a pod can tolerate all taints of
a particular category.
type: string
tolerationSeconds:
description: TolerationSeconds represents the period of
time the toleration (which must be of effect NoExecute,
otherwise this field is ignored) tolerates the taint.
By default, it is not set, which means tolerate the
taint forever (do not evict). Zero and negative values
will be treated as 0 (evict immediately) by the system.
format: int64
type: integer
value:
description: Value is the taint value the toleration matches
to. If the operator is Exists, the value should be empty,
otherwise just a regular string.
type: string
type: object
type: array
workspacesDefaultPlugins:
description: Default plug-ins applied to Devworkspaces.
items: