CHE-5908 Allow to customize ingress controller specific annotations for ingresses
Signed-off-by: Oleksandr Garagatyi <ogaragat@redhat.com>6.19.x
Oleksandr Garagatyi
Sergii Leshchenko
parent
ac1c5160b8
commit
cb43481e54
|
|
@ -394,6 +394,7 @@ che.infra.kubernetes.pvc.access_mode=ReadWriteOnce
|
|||
# then OpenShift infrastructure will reconfigure installer to use first available from this range
|
||||
che.infra.kubernetes.installer_server_min_port=10000
|
||||
che.infra.kubernetes.installer_server_max_port=20000
|
||||
che.infra.kubernetes.ingress.annotations_json=NULL
|
||||
|
||||
# Defines security context for pods that will be created by Kubernetes Infra
|
||||
#
|
||||
|
|
|
|||
|
|
@ -2,9 +2,29 @@
|
|||
Tested on minikube with vm provider Virtualbox. Note that Che with workspaces requires quite a lot
|
||||
of RAM. Initial tests were done with 10GB, but it is definitely more than it is needed to start Che
|
||||
and couple of workspaces.
|
||||
|
||||
IP of VM is supposed to be `192.168.99.100`. `nip.io` is also used for handling hosts resolution.
|
||||
If you have another IP or DNS replace these values in k8s.yml file.
|
||||
|
||||
Services are exposed using ingress controller approach.
|
||||
We added ingress annotations to customize ingress controller behavior -
|
||||
not to break websocket connections.
|
||||
In particular testing environment was setup with NginX ingress controller 0.9.0-beta.17.
|
||||
So we added annotations specific to this implementation and version:
|
||||
- ingress.kubernetes.io/rewrite-target: /
|
||||
- ingress.kubernetes.io/proxy-read-timeout: "3600"
|
||||
- ingress.kubernetes.io/proxy-connect-timeout: "3600"
|
||||
|
||||
If you use another ingress controller implementation or version you need to customize
|
||||
Che master ingress and value of environment variable `CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON` stored in ConfigMap.
|
||||
Value of the map should be expressed as a stringified JSON. For example most recent NginX controller uses other annotations:
|
||||
- nginx.ingress.kubernetes.io/rewrite-target
|
||||
- nginx.ingress.kubernetes.io/proxy-read-timeout
|
||||
- nginx.ingress.kubernetes.io/proxy-connect-timeout
|
||||
- nginx.ingress.kubernetes.io/ssl-redirect
|
||||
|
||||
And environment variable would be: `'{"ingress.kubernetes.io/rewrite-target": "/","ingress.kubernetes.io/ssl-redirect": "false","ingress.kubernetes.io/proxy-connect-timeout": "3600","ingress.kubernetes.io/proxy-read-timeout": "3600"}'`
|
||||
|
||||
###Prerequisites:
|
||||
- Ingress controller is running. Note: you can start it on minikube with `minikube addons enable ingress`.
|
||||
- Currently Che workspaces work with NginX ingress controller only. Note: it is default ingress controller on minikube.
|
||||
|
|
|
|||
|
|
@ -67,12 +67,15 @@ items:
|
|||
CHE_PREDEFINED_STACKS_RELOAD__ON__START: "false"
|
||||
JAVA_OPTS: "-XX:MaxRAMFraction=2 -XX:+UseParallelGC -XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=20 -XX:GCTimeRatio=4 -XX:AdaptiveSizePolicyWeight=90 -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -Dsun.zip.disableMemoryMapping=true -Xms20m "
|
||||
CHE_WORKSPACE_AUTO_START: "false"
|
||||
CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON: '{"ingress.kubernetes.io/rewrite-target": "/","ingress.kubernetes.io/ssl-redirect": "false","ingress.kubernetes.io/proxy-connect-timeout": "3600","ingress.kubernetes.io/proxy-read-timeout": "3600"}'
|
||||
- apiVersion: extensions/v1beta1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: che-ingress
|
||||
annotations:
|
||||
ingress.kubernetes.io/rewrite-target: /
|
||||
ingress.kubernetes.io/proxy-read-timeout: "3600"
|
||||
ingress.kubernetes.io/proxy-connect-timeout: "3600"
|
||||
spec:
|
||||
rules:
|
||||
- host: 192.168.99.100.nip.io
|
||||
|
|
@ -234,6 +237,11 @@ items:
|
|||
configMapKeyRef:
|
||||
key: CHE_WORKSPACE_AUTO_START
|
||||
name: che
|
||||
- name: CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
key: CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON
|
||||
name: che
|
||||
image: eclipse/che-server:nightly
|
||||
imagePullPolicy: Always
|
||||
livenessProbe:
|
||||
|
|
|
|||
|
|
@ -14,9 +14,11 @@ import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.
|
|||
import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.UniqueWorkspacePVCStrategy.UNIQUE_STRATEGY;
|
||||
|
||||
import com.google.inject.AbstractModule;
|
||||
import com.google.inject.TypeLiteral;
|
||||
import com.google.inject.assistedinject.FactoryModuleBuilder;
|
||||
import com.google.inject.multibindings.MapBinder;
|
||||
import com.google.inject.multibindings.Multibinder;
|
||||
import java.util.Map;
|
||||
import org.eclipse.che.api.workspace.server.spi.RuntimeInfrastructure;
|
||||
import org.eclipse.che.api.workspace.server.spi.environment.InternalEnvironmentFactory;
|
||||
import org.eclipse.che.api.workspace.server.spi.provision.env.CheApiEnvVarProvider;
|
||||
|
|
@ -34,6 +36,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.Workspa
|
|||
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy;
|
||||
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.KubernetesCheApiEnvVarProvider;
|
||||
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.LogsRootEnvVariableProvider;
|
||||
import org.eclipse.che.workspace.infrastructure.kubernetes.server.IngressAnnotationsProvider;
|
||||
|
||||
/** @author Sergii Leshchenko */
|
||||
public class KubernetesInfraModule extends AbstractModule {
|
||||
|
|
@ -65,5 +68,9 @@ public class KubernetesInfraModule extends AbstractModule {
|
|||
Multibinder<EnvVarProvider> envVarProviders =
|
||||
Multibinder.newSetBinder(binder(), EnvVarProvider.class);
|
||||
envVarProviders.addBinding().to(LogsRootEnvVariableProvider.class);
|
||||
|
||||
bind(new TypeLiteral<Map<String, String>>() {})
|
||||
.annotatedWith(com.google.inject.name.Names.named("infra.kubernetes.ingress.annotations"))
|
||||
.toProvider(IngressAnnotationsProvider.class);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ import io.fabric8.kubernetes.api.model.Container;
|
|||
import io.fabric8.kubernetes.api.model.Pod;
|
||||
import io.fabric8.kubernetes.api.model.PodSpec;
|
||||
import java.util.Map;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Named;
|
||||
import javax.inject.Singleton;
|
||||
import org.eclipse.che.api.core.model.workspace.config.ServerConfig;
|
||||
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
|
||||
|
|
@ -36,6 +38,14 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServ
|
|||
@Singleton
|
||||
public class ServersConverter implements ConfigurationProvisioner {
|
||||
|
||||
private final Map<String, String> ingressAnnotations;
|
||||
|
||||
@Inject
|
||||
public ServersConverter(
|
||||
@Named("infra.kubernetes.ingress.annotations") Map<String, String> ingressAnnotations) {
|
||||
this.ingressAnnotations = ingressAnnotations;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity)
|
||||
throws InfrastructureException {
|
||||
|
|
@ -47,7 +57,8 @@ public class ServersConverter implements ConfigurationProvisioner {
|
|||
InternalMachineConfig machineConfig = k8sEnv.getMachines().get(machineName);
|
||||
if (!machineConfig.getServers().isEmpty()) {
|
||||
KubernetesServerExposer kubernetesServerExposer =
|
||||
new KubernetesServerExposer<>(machineName, podConfig, containerConfig, k8sEnv);
|
||||
new KubernetesServerExposer<>(
|
||||
ingressAnnotations, machineName, podConfig, containerConfig, k8sEnv);
|
||||
kubernetesServerExposer.expose(machineConfig.getServers());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2018 Red Hat, Inc.
|
||||
* All rights reserved. This program and the accompanying materials
|
||||
* are made available under the terms of the Eclipse Public License v1.0
|
||||
* which accompanies this distribution, and is available at
|
||||
* http://www.eclipse.org/legal/epl-v10.html
|
||||
*
|
||||
* Contributors:
|
||||
* Red Hat, Inc. - initial API and implementation
|
||||
*/
|
||||
package org.eclipse.che.workspace.infrastructure.kubernetes.server;
|
||||
|
||||
import com.google.common.reflect.TypeToken;
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import java.lang.reflect.Type;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Named;
|
||||
import javax.inject.Provider;
|
||||
import javax.inject.Singleton;
|
||||
import org.eclipse.che.commons.annotation.Nullable;
|
||||
|
||||
/** @author Alexander Garagatyi */
|
||||
@Singleton
|
||||
public class IngressAnnotationsProvider implements Provider<Map<String, String>> {
|
||||
|
||||
private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create();
|
||||
private static final Type type = new TypeToken<Map<String, String>>() {}.getType();
|
||||
|
||||
private Map<String, String> annotations;
|
||||
|
||||
@Inject
|
||||
public IngressAnnotationsProvider(
|
||||
@Nullable @Named("che.infra.kubernetes.ingress.annotations_json") String annotationsString) {
|
||||
|
||||
if (annotationsString != null) {
|
||||
annotations = GSON.fromJson(annotationsString, type);
|
||||
} else {
|
||||
annotations = Collections.emptyMap();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, String> get() {
|
||||
return annotations;
|
||||
}
|
||||
}
|
||||
|
|
@ -45,6 +45,7 @@ import java.util.Map;
|
|||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import javax.inject.Named;
|
||||
import org.eclipse.che.api.core.model.workspace.config.ServerConfig;
|
||||
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
|
||||
import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations;
|
||||
|
|
@ -125,13 +126,19 @@ public class KubernetesServerExposer<T extends KubernetesEnvironment> {
|
|||
public static final int SERVER_UNIQUE_PART_SIZE = 8;
|
||||
public static final String SERVER_PREFIX = "server";
|
||||
|
||||
private final Map<String, String> ingressAnnotations;
|
||||
protected final String machineName;
|
||||
protected final Container container;
|
||||
protected final Pod pod;
|
||||
protected final T kubernetesEnvironment;
|
||||
|
||||
public KubernetesServerExposer(
|
||||
String machineName, Pod pod, Container container, T kubernetesEnvironment) {
|
||||
@Named("infra.kubernetes.ingress.annotations") Map<String, String> ingressAnnotations,
|
||||
String machineName,
|
||||
Pod pod,
|
||||
Container container,
|
||||
T kubernetesEnvironment) {
|
||||
this.ingressAnnotations = ingressAnnotations;
|
||||
this.machineName = machineName;
|
||||
this.pod = pod;
|
||||
this.container = container;
|
||||
|
|
@ -196,6 +203,7 @@ public class KubernetesServerExposer<T extends KubernetesEnvironment> {
|
|||
.withName(serviceName + '-' + servicePort.getName())
|
||||
.withMachineName(machineName)
|
||||
.withServiceName(serviceName)
|
||||
.withAnnotations(ingressAnnotations)
|
||||
.withServicePort(servicePort.getName())
|
||||
.withServers(ingressesServers)
|
||||
.build();
|
||||
|
|
@ -299,6 +307,7 @@ public class KubernetesServerExposer<T extends KubernetesEnvironment> {
|
|||
private IntOrString servicePort;
|
||||
private Map<String, ? extends ServerConfig> serversConfigs;
|
||||
private String machineName;
|
||||
private Map<String, String> annotations;
|
||||
|
||||
private IngressBuilder withName(String name) {
|
||||
this.name = name;
|
||||
|
|
@ -310,6 +319,11 @@ public class KubernetesServerExposer<T extends KubernetesEnvironment> {
|
|||
return this;
|
||||
}
|
||||
|
||||
private IngressBuilder withAnnotations(Map<String, String> annotations) {
|
||||
this.annotations = annotations;
|
||||
return this;
|
||||
}
|
||||
|
||||
private IngressBuilder withServicePort(String targetPortName) {
|
||||
this.servicePort = new IntOrString(targetPortName);
|
||||
return this;
|
||||
|
|
@ -342,10 +356,7 @@ public class KubernetesServerExposer<T extends KubernetesEnvironment> {
|
|||
IngressRule ingressRule = new IngressRuleBuilder().withHttp(httpIngressRuleValue).build();
|
||||
IngressSpec ingressSpec = new IngressSpecBuilder().withRules(ingressRule).build();
|
||||
|
||||
Map<String, String> ingressAnnotations = new HashMap<>();
|
||||
ingressAnnotations.put("ingress.kubernetes.io/rewrite-target", "/");
|
||||
ingressAnnotations.put("ingress.kubernetes.io/ssl-redirect", "false");
|
||||
ingressAnnotations.put("kubernetes.io/ingress.class", "nginx");
|
||||
Map<String, String> ingressAnnotations = new HashMap<>(annotations);
|
||||
ingressAnnotations.putAll(
|
||||
Annotations.newSerializer()
|
||||
.servers(serversConfigs)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
*/
|
||||
package org.eclipse.che.workspace.infrastructure.kubernetes.server;
|
||||
|
||||
import static java.util.Collections.emptyMap;
|
||||
import static java.util.Collections.singletonList;
|
||||
import static java.util.Collections.singletonMap;
|
||||
import static org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServerExposer.SERVER_PREFIX;
|
||||
|
|
@ -38,7 +39,9 @@ import org.eclipse.che.api.core.model.workspace.config.ServerConfig;
|
|||
import org.eclipse.che.api.workspace.server.model.impl.ServerConfigImpl;
|
||||
import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations;
|
||||
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
|
||||
import org.mockito.testng.MockitoTestNGListener;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.Listeners;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
/**
|
||||
|
|
@ -46,6 +49,7 @@ import org.testng.annotations.Test;
|
|||
*
|
||||
* @author Sergii Leshchenko
|
||||
*/
|
||||
@Listeners(MockitoTestNGListener.class)
|
||||
public class KubernetesServerExposerTest {
|
||||
|
||||
private static final Map<String, String> ATTRIBUTES_MAP = singletonMap("key", "value");
|
||||
|
|
@ -76,7 +80,8 @@ public class KubernetesServerExposerTest {
|
|||
kubernetesEnvironment =
|
||||
KubernetesEnvironment.builder().setPods(ImmutableMap.of("pod", pod)).build();
|
||||
this.serverExposer =
|
||||
new KubernetesServerExposer<>(MACHINE_NAME, pod, container, kubernetesEnvironment);
|
||||
new KubernetesServerExposer<>(
|
||||
emptyMap(), MACHINE_NAME, pod, container, kubernetesEnvironment);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import io.fabric8.kubernetes.api.model.Pod;
|
|||
import io.fabric8.kubernetes.api.model.Service;
|
||||
import io.fabric8.kubernetes.api.model.ServicePort;
|
||||
import io.fabric8.openshift.api.model.Route;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import org.eclipse.che.api.core.model.workspace.config.ServerConfig;
|
||||
import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations;
|
||||
|
|
@ -94,7 +95,7 @@ public class OpenShiftServerExposer extends KubernetesServerExposer<OpenShiftEnv
|
|||
|
||||
public OpenShiftServerExposer(
|
||||
String machineName, Pod pod, Container container, OpenShiftEnvironment openShiftEnvironment) {
|
||||
super(machineName, pod, container, openShiftEnvironment);
|
||||
super(Collections.emptyMap(), machineName, pod, container, openShiftEnvironment);
|
||||
this.openShiftEnvironment = openShiftEnvironment;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue