diff --git a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties index e3e1153a51..b293762cad 100644 --- a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties +++ b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties @@ -686,6 +686,12 @@ che.infra.kubernetes.async.storage.image=quay.io/eclipse/che-workspace-data-sync # key=value pairs, e.g: disktype=ssd,cpu=xlarge,foo=bar che.workspace.pod.node_selector=NULL +# Optionally configures tolerations for workspace pod. Format is a string representing a JSON Array of taint tolerations, +# or `NULL` to disable it. The objects contained in the array have to follow this +# link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#toleration-v1-core[spec]. +# Example: [{"effect":"NoExecute","key":"aNodeTaint","operator":"Equal","value":"aValue"}] +che.workspace.pod.tolerations_json=NULL + # The timeout for the Asynchronous Storage Pod shutdown after stopping the last used workspace. # Value less or equal to 0 interpreted as disabling shutdown ability. che.infra.kubernetes.async.storage.shutdown_timeout_min=120 diff --git a/deploy/kubernetes/helm/che/templates/configmap.yaml b/deploy/kubernetes/helm/che/templates/configmap.yaml index f422d9cee5..f8620bfbf3 100644 --- a/deploy/kubernetes/helm/che/templates/configmap.yaml +++ b/deploy/kubernetes/helm/che/templates/configmap.yaml @@ -174,3 +174,6 @@ data: {{- if .Values.cheServerSecureExposerJwtProxyImage }} CHE_SERVER_SECURE__EXPOSER_JWTPROXY_IMAGE: {{ .Values.cheServerSecureExposerJwtProxyImage | quote }} {{- end }} +{{- if .Values.cheWorkspacePodTolerations }} + CHE_WORKSPACE_POD_TOLERATIONS__JSON: {{ .Values.cheWorkspacePodTolerations | toJson | quote }} +{{- end }} diff --git a/deploy/kubernetes/helm/che/values.yaml b/deploy/kubernetes/helm/che/values.yaml index 1f2d5bdeb0..b7133cffc1 100644 --- a/deploy/kubernetes/helm/che/values.yaml +++ b/deploy/kubernetes/helm/che/values.yaml @@ -16,6 +16,11 @@ cheWorkspaceHttpProxy: "" cheWorkspaceHttpsProxy: "" cheWorkspaceNoProxy: "" +#cheWorkspacePodTolerations: +# - key: "a.node.taint" +# operator: "Equal" +# value: "aValue" +# effect: "NoExecute" cheImage: quay.io/eclipse/che-server:nightly cheImagePullPolicy: Always cheKeycloakRealm: "che" diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisioner.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisioner.java index 2cb16c1b19..01d03f18f1 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisioner.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisioner.java @@ -36,6 +36,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ServiceAcco import org.eclipse.che.workspace.infrastructure.kubernetes.provision.SshKeysProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TlsProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TlsProvisionerProvider; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TolerationsProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.UniqueNamesProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.VcsSslCertificateProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.EnvVarsConverter; @@ -78,6 +79,7 @@ public interface KubernetesEnvironmentProvisioner matchLabels = new HashMap<>(); diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/TolerationsProvisioner.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/TolerationsProvisioner.java new file mode 100644 index 0000000000..7286bb1dee --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/TolerationsProvisioner.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.provision; + +import static java.util.Collections.emptyList; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.fabric8.kubernetes.api.model.Toleration; +import java.util.List; +import javax.inject.Inject; +import javax.inject.Named; +import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.commons.annotation.Nullable; +import org.eclipse.che.inject.ConfigurationException; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; + +/** Provisions tolerations into workspace pod spec. */ +public class TolerationsProvisioner implements ConfigurationProvisioner { + + private final List tolerations; + + @Inject + public TolerationsProvisioner( + @Nullable @Named("che.workspace.pod.tolerations_json") String tolerationsProperty) + throws ConfigurationException { + try { + ObjectMapper jsonMapper = new ObjectMapper(); + this.tolerations = + tolerationsProperty != null + ? jsonMapper.readValue(tolerationsProperty, new TypeReference>() {}) + : emptyList(); + } catch (JsonProcessingException e) { + throw new ConfigurationException( + "che.workspace.pod.tolerations_json contains an invalid JSON string", e); + } + } + + @Override + public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity) + throws InfrastructureException { + if (!tolerations.isEmpty()) { + k8sEnv.getPodsData().values().forEach(d -> d.getSpec().setTolerations(tolerations)); + } + } +} diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisionerTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisionerTest.java index dbb69619cf..686e691d87 100644 --- a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisionerTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisionerTest.java @@ -35,6 +35,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ServiceAcco import org.eclipse.che.workspace.infrastructure.kubernetes.provision.SshKeysProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TlsProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TlsProvisionerProvider; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TolerationsProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.UniqueNamesProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.VcsSslCertificateProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.EnvVarsConverter; @@ -84,6 +85,7 @@ public class KubernetesEnvironmentProvisionerTest { @Mock private PreviewUrlExposer previewUrlExposer; @Mock private VcsSslCertificateProvisioner vcsSslCertificateProvisioner; @Mock private NodeSelectorProvisioner nodeSelectorProvisioner; + @Mock private TolerationsProvisioner tolerationsProvisioner; @Mock private KubernetesTrustedCAProvisioner trustedCAProvisioner; @Mock private GatewayRouterProvisioner gatewayRouterProvisioner; @@ -111,6 +113,7 @@ public class KubernetesEnvironmentProvisionerTest { imagePullSecretProvisioner, proxySettingsProvisioner, nodeSelectorProvisioner, + tolerationsProvisioner, asyncStorageProvisioner, asyncStoragePodObserver, serviceAccountProvisioner, @@ -131,6 +134,7 @@ public class KubernetesEnvironmentProvisionerTest { restartPolicyRewriter, ramLimitProvisioner, nodeSelectorProvisioner, + tolerationsProvisioner, securityContextProvisioner, podTerminationGracePeriodProvisioner, externalServerIngressTlsProvisioner, @@ -156,6 +160,7 @@ public class KubernetesEnvironmentProvisionerTest { provisionOrder.verify(ramLimitProvisioner).provision(eq(k8sEnv), eq(runtimeIdentity)); provisionOrder.verify(nodeSelectorProvisioner).provision(eq(k8sEnv), eq(runtimeIdentity)); + provisionOrder.verify(tolerationsProvisioner).provision(eq(k8sEnv), eq(runtimeIdentity)); provisionOrder .verify(externalServerIngressTlsProvisioner) .provision(eq(k8sEnv), eq(runtimeIdentity)); diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/PodMergerTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/PodMergerTest.java index 121a781e7f..7b4d5f638f 100644 --- a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/PodMergerTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/PodMergerTest.java @@ -28,6 +28,7 @@ import io.fabric8.kubernetes.api.model.PodSecurityContextBuilder; import io.fabric8.kubernetes.api.model.PodSpec; import io.fabric8.kubernetes.api.model.PodSpecBuilder; import io.fabric8.kubernetes.api.model.PodTemplateSpec; +import io.fabric8.kubernetes.api.model.Toleration; import io.fabric8.kubernetes.api.model.VolumeBuilder; import io.fabric8.kubernetes.api.model.apps.Deployment; import java.util.Arrays; @@ -118,6 +119,7 @@ public class PodMergerTest { .withVolumes(new VolumeBuilder().withName("v1").build()) .withNodeSelector(Map.of("foo1", "bar1")) .withImagePullSecrets(new LocalObjectReferenceBuilder().withName("secret1").build()) + .withTolerations(new Toleration("Effect", "key", "operator", 0L, "value1")) .build(); podSpec1.setAdditionalProperty("add1", 1L); PodData podData1 = new PodData(podSpec1, new ObjectMetaBuilder().build()); @@ -129,6 +131,7 @@ public class PodMergerTest { .withVolumes(new VolumeBuilder().withName("v2").build()) .withNodeSelector(Map.of("foo2", "bar2")) .withImagePullSecrets(new LocalObjectReferenceBuilder().withName("secret2").build()) + .withTolerations(new Toleration("Effect", "key", "operator", 0L, "value2")) .build(); podSpec2.setAdditionalProperty("add2", 2L); PodData podData2 = new PodData(podSpec2, new ObjectMetaBuilder().build()); @@ -448,5 +451,6 @@ public class PodMergerTest { .getAdditionalProperties() .entrySet() .containsAll(toCheck.getAdditionalProperties().entrySet())); + assertTrue(source.getTolerations().containsAll(toCheck.getTolerations())); } } diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/TolerationsProvisionerTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/TolerationsProvisionerTest.java new file mode 100644 index 0000000000..7c64bf7722 --- /dev/null +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/TolerationsProvisionerTest.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.provision; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.fabric8.kubernetes.api.model.ContainerBuilder; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodBuilder; +import io.fabric8.kubernetes.api.model.Toleration; +import java.util.Collections; +import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.inject.ConfigurationException; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.mockito.Mock; +import org.mockito.testng.MockitoTestNGListener; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Listeners; +import org.testng.annotations.Test; + +@Listeners(MockitoTestNGListener.class) +public class TolerationsProvisionerTest { + + @Mock private RuntimeIdentity runtimeId; + private KubernetesEnvironment k8sEnv; + + private TolerationsProvisioner provisioner; + + @BeforeMethod + public void setUp() { + k8sEnv = KubernetesEnvironment.builder().build(); + } + + @Test + public void shouldAddTolerationsIntoAllPods() throws Exception { + // given + k8sEnv.addPod(createPod("pod")); + k8sEnv.addPod(createPod("pod2")); + Toleration expectedToleration = + new Toleration("NoExecute", "a.node.taint", "Equal", 0L, "aValue"); + ObjectMapper objMapper = new ObjectMapper(); + String json = objMapper.writeValueAsString(Collections.singletonList(expectedToleration)); + provisioner = new TolerationsProvisioner(json); + + // when + provisioner.provision(k8sEnv, runtimeId); + + // then + for (Pod pod : k8sEnv.getPodsCopy().values()) { + assertEquals(pod.getSpec().getTolerations().size(), 1); + assertEquals(pod.getSpec().getTolerations().get(0), expectedToleration); + } + } + + @Test + public void shouldOmitEmptyTolerations() throws Exception { + // given + k8sEnv.addPod(createPod("pod")); + k8sEnv.addPod(createPod("pod2")); + + provisioner = new TolerationsProvisioner(null); + + // when + provisioner.provision(k8sEnv, runtimeId); + + // then + for (Pod pod : k8sEnv.getPodsCopy().values()) { + assertTrue(pod.getSpec().getTolerations().isEmpty()); + } + } + + @Test(expectedExceptions = ConfigurationException.class) + public void shouldFailOnInvalidTolerationsJson() throws Exception { + // given + provisioner = new TolerationsProvisioner("an invalid json string"); + } + + private Pod createPod(String podName) { + return new PodBuilder() + .withNewMetadata() + .withName(podName) + .endMetadata() + .withNewSpec() + .withTolerations() + .withInitContainers(new ContainerBuilder().build()) + .withContainers(new ContainerBuilder().build(), new ContainerBuilder().build()) + .endSpec() + .build(); + } +} diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisioner.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisioner.java index 2258d4ad2e..70e53170ac 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisioner.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisioner.java @@ -35,6 +35,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ServiceAcco import org.eclipse.che.workspace.infrastructure.kubernetes.provision.SshKeysProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TlsProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TlsProvisionerProvider; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TolerationsProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.UniqueNamesProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.VcsSslCertificateProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.EnvVarsConverter; @@ -75,6 +76,7 @@ public class OpenShiftEnvironmentProvisioner private final ImagePullSecretProvisioner imagePullSecretProvisioner; private final ProxySettingsProvisioner proxySettingsProvisioner; private final NodeSelectorProvisioner nodeSelectorProvisioner; + private final TolerationsProvisioner tolerationsProvisioner; private final AsyncStoragePodInterceptor asyncStoragePodInterceptor; private final ServiceAccountProvisioner serviceAccountProvisioner; private final AsyncStorageProvisioner asyncStorageProvisioner; @@ -102,6 +104,7 @@ public class OpenShiftEnvironmentProvisioner ImagePullSecretProvisioner imagePullSecretProvisioner, ProxySettingsProvisioner proxySettingsProvisioner, NodeSelectorProvisioner nodeSelectorProvisioner, + TolerationsProvisioner tolerationsProvisioner, AsyncStorageProvisioner asyncStorageProvisioner, AsyncStoragePodInterceptor asyncStoragePodInterceptor, ServiceAccountProvisioner serviceAccountProvisioner, @@ -126,6 +129,7 @@ public class OpenShiftEnvironmentProvisioner this.imagePullSecretProvisioner = imagePullSecretProvisioner; this.proxySettingsProvisioner = proxySettingsProvisioner; this.nodeSelectorProvisioner = nodeSelectorProvisioner; + this.tolerationsProvisioner = tolerationsProvisioner; this.asyncStorageProvisioner = asyncStorageProvisioner; this.asyncStoragePodInterceptor = asyncStoragePodInterceptor; this.serviceAccountProvisioner = serviceAccountProvisioner; @@ -166,6 +170,7 @@ public class OpenShiftEnvironmentProvisioner routeTlsProvisioner.provision(osEnv, identity); resourceLimitRequestProvisioner.provision(osEnv, identity); nodeSelectorProvisioner.provision(osEnv, identity); + tolerationsProvisioner.provision(osEnv, identity); podTerminationGracePeriodProvisioner.provision(osEnv, identity); imagePullSecretProvisioner.provision(osEnv, identity); proxySettingsProvisioner.provision(osEnv, identity); diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisionerTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisionerTest.java index 2d5245b4ad..26c8f5ede4 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisionerTest.java +++ b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisionerTest.java @@ -31,6 +31,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ProxySettin import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ServiceAccountProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.SshKeysProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TlsProvisionerProvider; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TolerationsProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.VcsSslCertificateProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.EnvVarsConverter; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.limits.ram.ContainerResourceProvisioner; @@ -79,6 +80,7 @@ public class OpenShiftEnvironmentProvisionerTest { @Mock private OpenShiftPreviewUrlExposer previewUrlEndpointsProvisioner; @Mock private VcsSslCertificateProvisioner vcsSslCertificateProvisioner; @Mock private NodeSelectorProvisioner nodeSelectorProvisioner; + @Mock private TolerationsProvisioner tolerationsProvisioner; @Mock private GatewayRouterProvisioner gatewayRouterProvisioner; @Mock private DeploymentMetadataProvisioner deploymentMetadataProvisioner; @Mock private OpenshiftTrustedCAProvisioner trustedCAProvisioner; @@ -105,6 +107,7 @@ public class OpenShiftEnvironmentProvisionerTest { imagePullSecretProvisioner, proxySettingsProvisioner, nodeSelectorProvisioner, + tolerationsProvisioner, asyncStorageProvisioner, asyncStoragePodObserver, serviceAccountProvisioner, @@ -127,6 +130,7 @@ public class OpenShiftEnvironmentProvisionerTest { restartPolicyRewriter, ramLimitProvisioner, nodeSelectorProvisioner, + tolerationsProvisioner, podTerminationGracePeriodProvisioner, imagePullSecretProvisioner, proxySettingsProvisioner, @@ -153,6 +157,7 @@ public class OpenShiftEnvironmentProvisionerTest { provisionOrder.verify(tlsRouteProvisioner).provision(eq(osEnv), eq(runtimeIdentity)); provisionOrder.verify(ramLimitProvisioner).provision(eq(osEnv), eq(runtimeIdentity)); provisionOrder.verify(nodeSelectorProvisioner).provision(eq(osEnv), eq(runtimeIdentity)); + provisionOrder.verify(tolerationsProvisioner).provision(eq(osEnv), eq(runtimeIdentity)); provisionOrder .verify(podTerminationGracePeriodProvisioner) .provision(eq(osEnv), eq(runtimeIdentity));