seg_yinzy
/
che-server
mirror of https://github.com/eclipse-che/che-server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adding k8s tolerations to workspace pods (#18691) 

* Added TolerationsProvisioner to support toleration in workspace pods

Signed-off-by: cccs-eric <eric.ladouceur@cyber.gc.ca>
7.26.x
cccs-eric 2021-01-18 08:09:49 -05:00 committed by GitHub
parent 9461a5dabe
commit fa06d947e7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 198 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties
View File

@ -686,6 +686,12 @@ che.infra.kubernetes.async.storage.image=quay.io/eclipse/che-workspace-data-sync
# key=value pairs, e.g: disktype=ssd,cpu=xlarge,foo=bar
che.workspace.pod.node_selector=NULL
# Optionally configures tolerations for workspace pod. Format is a string representing a JSON Array of taint tolerations,
# or `NULL` to disable it. The objects contained in the array have to follow this
# link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#toleration-v1-core[spec].
# Example: [{"effect":"NoExecute","key":"aNodeTaint","operator":"Equal","value":"aValue"}]
che.workspace.pod.tolerations_json=NULL
# The timeout for the Asynchronous Storage Pod shutdown after stopping the last used workspace.
# Value less or equal to 0 interpreted as disabling shutdown ability.
che.infra.kubernetes.async.storage.shutdown_timeout_min=120

3
deploy/kubernetes/helm/che/templates/configmap.yaml
View File

@ -174,3 +174,6 @@ data:
{{- if .Values.cheServerSecureExposerJwtProxyImage }}
CHE_SERVER_SECURE__EXPOSER_JWTPROXY_IMAGE: {{ .Values.cheServerSecureExposerJwtProxyImage | quote }}
{{- end }}
{{- if .Values.cheWorkspacePodTolerations }}
CHE_WORKSPACE_POD_TOLERATIONS__JSON: {{ .Values.cheWorkspacePodTolerations | toJson | quote }}
{{- end }}

5
deploy/kubernetes/helm/che/values.yaml
View File

@ -16,6 +16,11 @@
cheWorkspaceHttpProxy: ""
cheWorkspaceHttpsProxy: ""
cheWorkspaceNoProxy: ""
#cheWorkspacePodTolerations:
# - key: "a.node.taint"
# operator: "Equal"
# value: "aValue"
# effect: "NoExecute"
cheImage: quay.io/eclipse/che-server:nightly
cheImagePullPolicy: Always
cheKeycloakRealm: "che"

5
infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisioner.java
View File

@ -36,6 +36,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ServiceAcco
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.SshKeysProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TlsProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TlsProvisionerProvider;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TolerationsProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.UniqueNamesProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.VcsSslCertificateProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.EnvVarsConverter;
@ -78,6 +79,7 @@ public interface KubernetesEnvironmentProvisioner<T extends KubernetesEnvironmen
private final ImagePullSecretProvisioner imagePullSecretProvisioner;
private final ProxySettingsProvisioner proxySettingsProvisioner;
private final NodeSelectorProvisioner nodeSelectorProvisioner;
private final TolerationsProvisioner tolerationsProvisioner;
private final AsyncStorageProvisioner asyncStorageProvisioner;
private final AsyncStoragePodInterceptor asyncStoragePodInterceptor;
private final ServiceAccountProvisioner serviceAccountProvisioner;
@ -105,6 +107,7 @@ public interface KubernetesEnvironmentProvisioner<T extends KubernetesEnvironmen
ImagePullSecretProvisioner imagePullSecretProvisioner,
ProxySettingsProvisioner proxySettingsProvisioner,
NodeSelectorProvisioner nodeSelectorProvisioner,
TolerationsProvisioner tolerationsProvisioner,
AsyncStorageProvisioner asyncStorageProvisioner,
AsyncStoragePodInterceptor asyncStoragePodInterceptor,
ServiceAccountProvisioner serviceAccountProvisioner,
@ -129,6 +132,7 @@ public interface KubernetesEnvironmentProvisioner<T extends KubernetesEnvironmen
this.imagePullSecretProvisioner = imagePullSecretProvisioner;
this.proxySettingsProvisioner = proxySettingsProvisioner;
this.nodeSelectorProvisioner = nodeSelectorProvisioner;
this.tolerationsProvisioner = tolerationsProvisioner;
this.asyncStorageProvisioner = asyncStorageProvisioner;
this.asyncStoragePodInterceptor = asyncStoragePodInterceptor;
this.serviceAccountProvisioner = serviceAccountProvisioner;
@ -169,6 +173,7 @@ public interface KubernetesEnvironmentProvisioner<T extends KubernetesEnvironmen
restartPolicyRewriter.provision(k8sEnv, identity);
resourceLimitRequestProvisioner.provision(k8sEnv, identity);
nodeSelectorProvisioner.provision(k8sEnv, identity);
tolerationsProvisioner.provision(k8sEnv, identity);
externalServerTlsProvisioner.provision(k8sEnv, identity);
securityContextProvisioner.provision(k8sEnv, identity);
podTerminationGracePeriodProvisioner.provision(k8sEnv, identity);

2
infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/PodMerger.java
View File

@ -154,6 +154,8 @@ public class PodMerger {
// if there are entries with such keys then values will be overridden
baseSpec.getAdditionalProperties().putAll(podData.getSpec().getAdditionalProperties());
// add tolerations to baseSpec if any
baseSpec.getTolerations().addAll(podData.getSpec().getTolerations());
}
Map<String, String> matchLabels = new HashMap<>();

57
infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/TolerationsProvisioner.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.provision;
import static java.util.Collections.emptyList;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.fabric8.kubernetes.api.model.Toleration;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Named;
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
import org.eclipse.che.commons.annotation.Nullable;
import org.eclipse.che.inject.ConfigurationException;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
/** Provisions tolerations into workspace pod spec. */
public class TolerationsProvisioner implements ConfigurationProvisioner {
private final List<Toleration> tolerations;
@Inject
public TolerationsProvisioner(
@Nullable @Named("che.workspace.pod.tolerations_json") String tolerationsProperty)
throws ConfigurationException {
try {
ObjectMapper jsonMapper = new ObjectMapper();
this.tolerations =
tolerationsProperty != null
? jsonMapper.readValue(tolerationsProperty, new TypeReference<List<Toleration>>() {})
: emptyList();
} catch (JsonProcessingException e) {
throw new ConfigurationException(
"che.workspace.pod.tolerations_json contains an invalid JSON string", e);
}
}
@Override
public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity)
throws InfrastructureException {
if (!tolerations.isEmpty()) {
k8sEnv.getPodsData().values().forEach(d -> d.getSpec().setTolerations(tolerations));
}
}
}

5
infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisionerTest.java
View File

@ -35,6 +35,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ServiceAcco
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.SshKeysProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TlsProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TlsProvisionerProvider;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TolerationsProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.UniqueNamesProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.VcsSslCertificateProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.EnvVarsConverter;
@ -84,6 +85,7 @@ public class KubernetesEnvironmentProvisionerTest {
@Mock private PreviewUrlExposer previewUrlExposer;
@Mock private VcsSslCertificateProvisioner vcsSslCertificateProvisioner;
@Mock private NodeSelectorProvisioner nodeSelectorProvisioner;
@Mock private TolerationsProvisioner tolerationsProvisioner;
@Mock private KubernetesTrustedCAProvisioner trustedCAProvisioner;
@Mock private GatewayRouterProvisioner gatewayRouterProvisioner;
@ -111,6 +113,7 @@ public class KubernetesEnvironmentProvisionerTest {
imagePullSecretProvisioner,
proxySettingsProvisioner,
nodeSelectorProvisioner,
tolerationsProvisioner,
asyncStorageProvisioner,
asyncStoragePodObserver,
serviceAccountProvisioner,
@ -131,6 +134,7 @@ public class KubernetesEnvironmentProvisionerTest {
restartPolicyRewriter,
ramLimitProvisioner,
nodeSelectorProvisioner,
tolerationsProvisioner,
securityContextProvisioner,
podTerminationGracePeriodProvisioner,
externalServerIngressTlsProvisioner,
@ -156,6 +160,7 @@ public class KubernetesEnvironmentProvisionerTest {
provisionOrder.verify(ramLimitProvisioner).provision(eq(k8sEnv), eq(runtimeIdentity));
provisionOrder.verify(nodeSelectorProvisioner).provision(eq(k8sEnv), eq(runtimeIdentity));
provisionOrder.verify(tolerationsProvisioner).provision(eq(k8sEnv), eq(runtimeIdentity));
provisionOrder
.verify(externalServerIngressTlsProvisioner)
.provision(eq(k8sEnv), eq(runtimeIdentity));

4
infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/PodMergerTest.java
View File

@ -28,6 +28,7 @@ import io.fabric8.kubernetes.api.model.PodSecurityContextBuilder;
import io.fabric8.kubernetes.api.model.PodSpec;
import io.fabric8.kubernetes.api.model.PodSpecBuilder;
import io.fabric8.kubernetes.api.model.PodTemplateSpec;
import io.fabric8.kubernetes.api.model.Toleration;
import io.fabric8.kubernetes.api.model.VolumeBuilder;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import java.util.Arrays;
@ -118,6 +119,7 @@ public class PodMergerTest {
.withVolumes(new VolumeBuilder().withName("v1").build())
.withNodeSelector(Map.of("foo1", "bar1"))
.withImagePullSecrets(new LocalObjectReferenceBuilder().withName("secret1").build())
.withTolerations(new Toleration("Effect", "key", "operator", 0L, "value1"))
.build();
podSpec1.setAdditionalProperty("add1", 1L);
PodData podData1 = new PodData(podSpec1, new ObjectMetaBuilder().build());
@ -129,6 +131,7 @@ public class PodMergerTest {
.withVolumes(new VolumeBuilder().withName("v2").build())
.withNodeSelector(Map.of("foo2", "bar2"))
.withImagePullSecrets(new LocalObjectReferenceBuilder().withName("secret2").build())
.withTolerations(new Toleration("Effect", "key", "operator", 0L, "value2"))
.build();
podSpec2.setAdditionalProperty("add2", 2L);
PodData podData2 = new PodData(podSpec2, new ObjectMetaBuilder().build());
@ -448,5 +451,6 @@ public class PodMergerTest {
.getAdditionalProperties()
.entrySet()
.containsAll(toCheck.getAdditionalProperties().entrySet()));
assertTrue(source.getTolerations().containsAll(toCheck.getTolerations()));
}
}

101
infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/TolerationsProvisionerTest.java Normal file
View File

@ -0,0 +1,101 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.provision;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.fabric8.kubernetes.api.model.ContainerBuilder;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.PodBuilder;
import io.fabric8.kubernetes.api.model.Toleration;
import java.util.Collections;
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
import org.eclipse.che.inject.ConfigurationException;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
import org.mockito.Mock;
import org.mockito.testng.MockitoTestNGListener;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;
@Listeners(MockitoTestNGListener.class)
public class TolerationsProvisionerTest {
@Mock private RuntimeIdentity runtimeId;
private KubernetesEnvironment k8sEnv;
private TolerationsProvisioner provisioner;
@BeforeMethod
public void setUp() {
k8sEnv = KubernetesEnvironment.builder().build();
}
@Test
public void shouldAddTolerationsIntoAllPods() throws Exception {
// given
k8sEnv.addPod(createPod("pod"));
k8sEnv.addPod(createPod("pod2"));
Toleration expectedToleration =
new Toleration("NoExecute", "a.node.taint", "Equal", 0L, "aValue");
ObjectMapper objMapper = new ObjectMapper();
String json = objMapper.writeValueAsString(Collections.singletonList(expectedToleration));
provisioner = new TolerationsProvisioner(json);
// when
provisioner.provision(k8sEnv, runtimeId);
// then
for (Pod pod : k8sEnv.getPodsCopy().values()) {
assertEquals(pod.getSpec().getTolerations().size(), 1);
assertEquals(pod.getSpec().getTolerations().get(0), expectedToleration);
}
}
@Test
public void shouldOmitEmptyTolerations() throws Exception {
// given
k8sEnv.addPod(createPod("pod"));
k8sEnv.addPod(createPod("pod2"));
provisioner = new TolerationsProvisioner(null);
// when
provisioner.provision(k8sEnv, runtimeId);
// then
for (Pod pod : k8sEnv.getPodsCopy().values()) {
assertTrue(pod.getSpec().getTolerations().isEmpty());
}
}
@Test(expectedExceptions = ConfigurationException.class)
public void shouldFailOnInvalidTolerationsJson() throws Exception {
// given
provisioner = new TolerationsProvisioner("an invalid json string");
}
private Pod createPod(String podName) {
return new PodBuilder()
.withNewMetadata()
.withName(podName)
.endMetadata()
.withNewSpec()
.withTolerations()
.withInitContainers(new ContainerBuilder().build())
.withContainers(new ContainerBuilder().build(), new ContainerBuilder().build())
.endSpec()
.build();
}
}

5
infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisioner.java
View File

@ -35,6 +35,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ServiceAcco
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.SshKeysProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TlsProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TlsProvisionerProvider;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TolerationsProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.UniqueNamesProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.VcsSslCertificateProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.EnvVarsConverter;
@ -75,6 +76,7 @@ public class OpenShiftEnvironmentProvisioner
private final ImagePullSecretProvisioner imagePullSecretProvisioner;
private final ProxySettingsProvisioner proxySettingsProvisioner;
private final NodeSelectorProvisioner nodeSelectorProvisioner;
private final TolerationsProvisioner tolerationsProvisioner;
private final AsyncStoragePodInterceptor asyncStoragePodInterceptor;
private final ServiceAccountProvisioner serviceAccountProvisioner;
private final AsyncStorageProvisioner asyncStorageProvisioner;
@ -102,6 +104,7 @@ public class OpenShiftEnvironmentProvisioner
ImagePullSecretProvisioner imagePullSecretProvisioner,
ProxySettingsProvisioner proxySettingsProvisioner,
NodeSelectorProvisioner nodeSelectorProvisioner,
TolerationsProvisioner tolerationsProvisioner,
AsyncStorageProvisioner asyncStorageProvisioner,
AsyncStoragePodInterceptor asyncStoragePodInterceptor,
ServiceAccountProvisioner serviceAccountProvisioner,
@ -126,6 +129,7 @@ public class OpenShiftEnvironmentProvisioner
this.imagePullSecretProvisioner = imagePullSecretProvisioner;
this.proxySettingsProvisioner = proxySettingsProvisioner;
this.nodeSelectorProvisioner = nodeSelectorProvisioner;
this.tolerationsProvisioner = tolerationsProvisioner;
this.asyncStorageProvisioner = asyncStorageProvisioner;
this.asyncStoragePodInterceptor = asyncStoragePodInterceptor;
this.serviceAccountProvisioner = serviceAccountProvisioner;
@ -166,6 +170,7 @@ public class OpenShiftEnvironmentProvisioner
routeTlsProvisioner.provision(osEnv, identity);
resourceLimitRequestProvisioner.provision(osEnv, identity);
nodeSelectorProvisioner.provision(osEnv, identity);
tolerationsProvisioner.provision(osEnv, identity);
podTerminationGracePeriodProvisioner.provision(osEnv, identity);
imagePullSecretProvisioner.provision(osEnv, identity);
proxySettingsProvisioner.provision(osEnv, identity);

5
infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisionerTest.java
View File

@ -31,6 +31,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ProxySettin
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ServiceAccountProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.SshKeysProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TlsProvisionerProvider;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TolerationsProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.VcsSslCertificateProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.EnvVarsConverter;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.limits.ram.ContainerResourceProvisioner;
@ -79,6 +80,7 @@ public class OpenShiftEnvironmentProvisionerTest {
@Mock private OpenShiftPreviewUrlExposer previewUrlEndpointsProvisioner;
@Mock private VcsSslCertificateProvisioner vcsSslCertificateProvisioner;
@Mock private NodeSelectorProvisioner nodeSelectorProvisioner;
@Mock private TolerationsProvisioner tolerationsProvisioner;
@Mock private GatewayRouterProvisioner gatewayRouterProvisioner;
@Mock private DeploymentMetadataProvisioner deploymentMetadataProvisioner;
@Mock private OpenshiftTrustedCAProvisioner trustedCAProvisioner;
@ -105,6 +107,7 @@ public class OpenShiftEnvironmentProvisionerTest {
imagePullSecretProvisioner,
proxySettingsProvisioner,
nodeSelectorProvisioner,
tolerationsProvisioner,
asyncStorageProvisioner,
asyncStoragePodObserver,
serviceAccountProvisioner,
@ -127,6 +130,7 @@ public class OpenShiftEnvironmentProvisionerTest {
restartPolicyRewriter,
ramLimitProvisioner,
nodeSelectorProvisioner,
tolerationsProvisioner,
podTerminationGracePeriodProvisioner,
imagePullSecretProvisioner,
proxySettingsProvisioner,
@ -153,6 +157,7 @@ public class OpenShiftEnvironmentProvisionerTest {
provisionOrder.verify(tlsRouteProvisioner).provision(eq(osEnv), eq(runtimeIdentity));
provisionOrder.verify(ramLimitProvisioner).provision(eq(osEnv), eq(runtimeIdentity));
provisionOrder.verify(nodeSelectorProvisioner).provision(eq(osEnv), eq(runtimeIdentity));
provisionOrder.verify(tolerationsProvisioner).provision(eq(osEnv), eq(runtimeIdentity));
provisionOrder
.verify(podTerminationGracePeriodProvisioner)
.provision(eq(osEnv), eq(runtimeIdentity));