From cb43481e5432db75f4245fa3cd9caeba546a8dac Mon Sep 17 00:00:00 2001 From: Oleksandr Garagatyi Date: Mon, 5 Feb 2018 15:45:03 +0200 Subject: [PATCH] CHE-5908 Allow to customize ingress controller specific annotations for ingresses Signed-off-by: Oleksandr Garagatyi --- .../webapp/WEB-INF/classes/che/che.properties | 1 + .../init/modules/kubernetes/Deploy Che.md | 20 ++++++++ .../kubernetes/files/che-kubernetes.yaml | 8 +++ .../kubernetes/KubernetesInfraModule.java | 7 +++ .../provision/server/ServersConverter.java | 13 ++++- .../server/IngressAnnotationsProvider.java | 49 +++++++++++++++++++ .../server/KubernetesServerExposer.java | 21 ++++++-- .../server/KubernetesServerExposerTest.java | 7 ++- .../server/OpenShiftServerExposer.java | 3 +- 9 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/IngressAnnotationsProvider.java diff --git a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties index 7b2a914629..ec672be758 100644 --- a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties +++ b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties @@ -394,6 +394,7 @@ che.infra.kubernetes.pvc.access_mode=ReadWriteOnce # then OpenShift infrastructure will reconfigure installer to use first available from this range che.infra.kubernetes.installer_server_min_port=10000 che.infra.kubernetes.installer_server_max_port=20000 +che.infra.kubernetes.ingress.annotations_json=NULL # Defines security context for pods that will be created by Kubernetes Infra # diff --git a/dockerfiles/init/modules/kubernetes/Deploy Che.md b/dockerfiles/init/modules/kubernetes/Deploy Che.md index 6b3b7a0fda..0e0c297b1d 100644 --- a/dockerfiles/init/modules/kubernetes/Deploy Che.md +++ b/dockerfiles/init/modules/kubernetes/Deploy Che.md @@ -2,9 +2,29 @@ Tested on minikube with vm provider Virtualbox. Note that Che with workspaces requires quite a lot of RAM. Initial tests were done with 10GB, but it is definitely more than it is needed to start Che and couple of workspaces. + IP of VM is supposed to be `192.168.99.100`. `nip.io` is also used for handling hosts resolution. If you have another IP or DNS replace these values in k8s.yml file. +Services are exposed using ingress controller approach. +We added ingress annotations to customize ingress controller behavior - +not to break websocket connections. +In particular testing environment was setup with NginX ingress controller 0.9.0-beta.17. +So we added annotations specific to this implementation and version: +- ingress.kubernetes.io/rewrite-target: / +- ingress.kubernetes.io/proxy-read-timeout: "3600" +- ingress.kubernetes.io/proxy-connect-timeout: "3600" + +If you use another ingress controller implementation or version you need to customize +Che master ingress and value of environment variable `CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON` stored in ConfigMap. +Value of the map should be expressed as a stringified JSON. For example most recent NginX controller uses other annotations: +- nginx.ingress.kubernetes.io/rewrite-target +- nginx.ingress.kubernetes.io/proxy-read-timeout +- nginx.ingress.kubernetes.io/proxy-connect-timeout +- nginx.ingress.kubernetes.io/ssl-redirect + +And environment variable would be: `'{"ingress.kubernetes.io/rewrite-target": "/","ingress.kubernetes.io/ssl-redirect": "false","ingress.kubernetes.io/proxy-connect-timeout": "3600","ingress.kubernetes.io/proxy-read-timeout": "3600"}'` + ###Prerequisites: - Ingress controller is running. Note: you can start it on minikube with `minikube addons enable ingress`. - Currently Che workspaces work with NginX ingress controller only. Note: it is default ingress controller on minikube. diff --git a/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml b/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml index e5035c53da..c99b9b1c81 100644 --- a/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml +++ b/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml @@ -67,12 +67,15 @@ items: CHE_PREDEFINED_STACKS_RELOAD__ON__START: "false" JAVA_OPTS: "-XX:MaxRAMFraction=2 -XX:+UseParallelGC -XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=20 -XX:GCTimeRatio=4 -XX:AdaptiveSizePolicyWeight=90 -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -Dsun.zip.disableMemoryMapping=true -Xms20m " CHE_WORKSPACE_AUTO_START: "false" + CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON: '{"ingress.kubernetes.io/rewrite-target": "/","ingress.kubernetes.io/ssl-redirect": "false","ingress.kubernetes.io/proxy-connect-timeout": "3600","ingress.kubernetes.io/proxy-read-timeout": "3600"}' - apiVersion: extensions/v1beta1 kind: Ingress metadata: name: che-ingress annotations: ingress.kubernetes.io/rewrite-target: / + ingress.kubernetes.io/proxy-read-timeout: "3600" + ingress.kubernetes.io/proxy-connect-timeout: "3600" spec: rules: - host: 192.168.99.100.nip.io @@ -234,6 +237,11 @@ items: configMapKeyRef: key: CHE_WORKSPACE_AUTO_START name: che + - name: CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON + valueFrom: + configMapKeyRef: + key: CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON + name: che image: eclipse/che-server:nightly imagePullPolicy: Always livenessProbe: diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java index 7a422e1d7b..596077ada5 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java @@ -14,9 +14,11 @@ import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc. import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.UniqueWorkspacePVCStrategy.UNIQUE_STRATEGY; import com.google.inject.AbstractModule; +import com.google.inject.TypeLiteral; import com.google.inject.assistedinject.FactoryModuleBuilder; import com.google.inject.multibindings.MapBinder; import com.google.inject.multibindings.Multibinder; +import java.util.Map; import org.eclipse.che.api.workspace.server.spi.RuntimeInfrastructure; import org.eclipse.che.api.workspace.server.spi.environment.InternalEnvironmentFactory; import org.eclipse.che.api.workspace.server.spi.provision.env.CheApiEnvVarProvider; @@ -34,6 +36,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.Workspa import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.KubernetesCheApiEnvVarProvider; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.LogsRootEnvVariableProvider; +import org.eclipse.che.workspace.infrastructure.kubernetes.server.IngressAnnotationsProvider; /** @author Sergii Leshchenko */ public class KubernetesInfraModule extends AbstractModule { @@ -65,5 +68,9 @@ public class KubernetesInfraModule extends AbstractModule { Multibinder envVarProviders = Multibinder.newSetBinder(binder(), EnvVarProvider.class); envVarProviders.addBinding().to(LogsRootEnvVariableProvider.class); + + bind(new TypeLiteral>() {}) + .annotatedWith(com.google.inject.name.Names.named("infra.kubernetes.ingress.annotations")) + .toProvider(IngressAnnotationsProvider.class); } } diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/server/ServersConverter.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/server/ServersConverter.java index 59e43e95e0..511dccf468 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/server/ServersConverter.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/server/ServersConverter.java @@ -14,6 +14,8 @@ import io.fabric8.kubernetes.api.model.Container; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodSpec; import java.util.Map; +import javax.inject.Inject; +import javax.inject.Named; import javax.inject.Singleton; import org.eclipse.che.api.core.model.workspace.config.ServerConfig; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; @@ -36,6 +38,14 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServ @Singleton public class ServersConverter implements ConfigurationProvisioner { + private final Map ingressAnnotations; + + @Inject + public ServersConverter( + @Named("infra.kubernetes.ingress.annotations") Map ingressAnnotations) { + this.ingressAnnotations = ingressAnnotations; + } + @Override public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity) throws InfrastructureException { @@ -47,7 +57,8 @@ public class ServersConverter implements ConfigurationProvisioner { InternalMachineConfig machineConfig = k8sEnv.getMachines().get(machineName); if (!machineConfig.getServers().isEmpty()) { KubernetesServerExposer kubernetesServerExposer = - new KubernetesServerExposer<>(machineName, podConfig, containerConfig, k8sEnv); + new KubernetesServerExposer<>( + ingressAnnotations, machineName, podConfig, containerConfig, k8sEnv); kubernetesServerExposer.expose(machineConfig.getServers()); } } diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/IngressAnnotationsProvider.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/IngressAnnotationsProvider.java new file mode 100644 index 0000000000..c2a0d55669 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/IngressAnnotationsProvider.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.server; + +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import java.lang.reflect.Type; +import java.util.Collections; +import java.util.Map; +import javax.inject.Inject; +import javax.inject.Named; +import javax.inject.Provider; +import javax.inject.Singleton; +import org.eclipse.che.commons.annotation.Nullable; + +/** @author Alexander Garagatyi */ +@Singleton +public class IngressAnnotationsProvider implements Provider> { + + private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); + private static final Type type = new TypeToken>() {}.getType(); + + private Map annotations; + + @Inject + public IngressAnnotationsProvider( + @Nullable @Named("che.infra.kubernetes.ingress.annotations_json") String annotationsString) { + + if (annotationsString != null) { + annotations = GSON.fromJson(annotationsString, type); + } else { + annotations = Collections.emptyMap(); + } + } + + @Override + public Map get() { + return annotations; + } +} diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java index 1a7bf0387f..bc0c7f4321 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java @@ -45,6 +45,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import javax.inject.Named; import org.eclipse.che.api.core.model.workspace.config.ServerConfig; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations; @@ -125,13 +126,19 @@ public class KubernetesServerExposer { public static final int SERVER_UNIQUE_PART_SIZE = 8; public static final String SERVER_PREFIX = "server"; + private final Map ingressAnnotations; protected final String machineName; protected final Container container; protected final Pod pod; protected final T kubernetesEnvironment; public KubernetesServerExposer( - String machineName, Pod pod, Container container, T kubernetesEnvironment) { + @Named("infra.kubernetes.ingress.annotations") Map ingressAnnotations, + String machineName, + Pod pod, + Container container, + T kubernetesEnvironment) { + this.ingressAnnotations = ingressAnnotations; this.machineName = machineName; this.pod = pod; this.container = container; @@ -196,6 +203,7 @@ public class KubernetesServerExposer { .withName(serviceName + '-' + servicePort.getName()) .withMachineName(machineName) .withServiceName(serviceName) + .withAnnotations(ingressAnnotations) .withServicePort(servicePort.getName()) .withServers(ingressesServers) .build(); @@ -299,6 +307,7 @@ public class KubernetesServerExposer { private IntOrString servicePort; private Map serversConfigs; private String machineName; + private Map annotations; private IngressBuilder withName(String name) { this.name = name; @@ -310,6 +319,11 @@ public class KubernetesServerExposer { return this; } + private IngressBuilder withAnnotations(Map annotations) { + this.annotations = annotations; + return this; + } + private IngressBuilder withServicePort(String targetPortName) { this.servicePort = new IntOrString(targetPortName); return this; @@ -342,10 +356,7 @@ public class KubernetesServerExposer { IngressRule ingressRule = new IngressRuleBuilder().withHttp(httpIngressRuleValue).build(); IngressSpec ingressSpec = new IngressSpecBuilder().withRules(ingressRule).build(); - Map ingressAnnotations = new HashMap<>(); - ingressAnnotations.put("ingress.kubernetes.io/rewrite-target", "/"); - ingressAnnotations.put("ingress.kubernetes.io/ssl-redirect", "false"); - ingressAnnotations.put("kubernetes.io/ingress.class", "nginx"); + Map ingressAnnotations = new HashMap<>(annotations); ingressAnnotations.putAll( Annotations.newSerializer() .servers(serversConfigs) diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposerTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposerTest.java index fdd847f773..3912962f5d 100644 --- a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposerTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposerTest.java @@ -10,6 +10,7 @@ */ package org.eclipse.che.workspace.infrastructure.kubernetes.server; +import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServerExposer.SERVER_PREFIX; @@ -38,7 +39,9 @@ import org.eclipse.che.api.core.model.workspace.config.ServerConfig; import org.eclipse.che.api.workspace.server.model.impl.ServerConfigImpl; import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations; import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.mockito.testng.MockitoTestNGListener; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Listeners; import org.testng.annotations.Test; /** @@ -46,6 +49,7 @@ import org.testng.annotations.Test; * * @author Sergii Leshchenko */ +@Listeners(MockitoTestNGListener.class) public class KubernetesServerExposerTest { private static final Map ATTRIBUTES_MAP = singletonMap("key", "value"); @@ -76,7 +80,8 @@ public class KubernetesServerExposerTest { kubernetesEnvironment = KubernetesEnvironment.builder().setPods(ImmutableMap.of("pod", pod)).build(); this.serverExposer = - new KubernetesServerExposer<>(MACHINE_NAME, pod, container, kubernetesEnvironment); + new KubernetesServerExposer<>( + emptyMap(), MACHINE_NAME, pod, container, kubernetesEnvironment); } @Test diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerExposer.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerExposer.java index 3bccf04119..da017de0e4 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerExposer.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerExposer.java @@ -19,6 +19,7 @@ import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Service; import io.fabric8.kubernetes.api.model.ServicePort; import io.fabric8.openshift.api.model.Route; +import java.util.Collections; import java.util.Map; import org.eclipse.che.api.core.model.workspace.config.ServerConfig; import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations; @@ -94,7 +95,7 @@ public class OpenShiftServerExposer extends KubernetesServerExposer