520 lines
16 KiB
Go
520 lines
16 KiB
Go
//
|
|
// Copyright (c) 2019-2021 Red Hat, Inc.
|
|
// This program and the accompanying materials are made
|
|
// available under the terms of the Eclipse Public License 2.0
|
|
// which is available at https://www.eclipse.org/legal/epl-2.0/
|
|
//
|
|
// SPDX-License-Identifier: EPL-2.0
|
|
//
|
|
// Contributors:
|
|
// Red Hat, Inc. - initial API and implementation
|
|
//
|
|
|
|
package solver
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"path"
|
|
"strings"
|
|
|
|
dw "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
|
|
dwo "github.com/devfile/devworkspace-operator/apis/controller/v1alpha1"
|
|
"github.com/devfile/devworkspace-operator/controllers/controller/devworkspacerouting/solvers"
|
|
"github.com/devfile/devworkspace-operator/pkg/common"
|
|
"github.com/devfile/devworkspace-operator/pkg/constants"
|
|
"github.com/devfile/devworkspace-operator/pkg/infrastructure"
|
|
"github.com/eclipse-che/che-operator/api/v2alpha1"
|
|
"github.com/eclipse-che/che-operator/controllers/devworkspace/defaults"
|
|
"github.com/eclipse-che/che-operator/controllers/devworkspace/sync"
|
|
"github.com/google/go-cmp/cmp/cmpopts"
|
|
routeV1 "github.com/openshift/api/route/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/api/extensions/v1beta1"
|
|
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/labels"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/yaml"
|
|
)
|
|
|
|
const (
|
|
uniqueEndpointAttributeName = "unique"
|
|
urlRewriteSupportedEndpointAttributeName = "urlRewriteSupported"
|
|
endpointURLPrefixPattern = "/%s/%s/%d"
|
|
// note - che-theia DEPENDS on this format - we should not change this unless crosschecked with the che-theia impl
|
|
uniqueEndpointURLPrefixPattern = "/%s/%s/%s"
|
|
)
|
|
|
|
var (
|
|
configMapDiffOpts = cmpopts.IgnoreFields(corev1.ConfigMap{}, "TypeMeta", "ObjectMeta")
|
|
)
|
|
|
|
// keys are port numbers, values are maps where keys are endpoint names (in case we need more than 1 endpoint for a single port) and values
|
|
// contain info about the intended endpoint scheme and the order in which the port is defined (used for unique naming)
|
|
type portMapping map[int32]map[string]portMappingValue
|
|
type portMappingValue struct {
|
|
endpointScheme string
|
|
order int
|
|
}
|
|
|
|
func (c *CheRoutingSolver) cheSpecObjects(cheManager *v2alpha1.CheCluster, routing *dwo.DevWorkspaceRouting, workspaceMeta solvers.DevWorkspaceMetadata) (solvers.RoutingObjects, error) {
|
|
objs := solvers.RoutingObjects{}
|
|
|
|
objs.Services = solvers.GetDiscoverableServicesForEndpoints(routing.Spec.Endpoints, workspaceMeta)
|
|
|
|
commonService := solvers.GetServiceForEndpoints(routing.Spec.Endpoints, workspaceMeta, false, dw.PublicEndpointExposure, dw.InternalEndpointExposure)
|
|
if commonService != nil {
|
|
objs.Services = append(objs.Services, *commonService)
|
|
}
|
|
|
|
annos := map[string]string{}
|
|
annos[defaults.ConfigAnnotationCheManagerName] = cheManager.Name
|
|
annos[defaults.ConfigAnnotationCheManagerNamespace] = cheManager.Namespace
|
|
|
|
additionalLabels := defaults.GetLabelsForComponent(cheManager, "exposure")
|
|
|
|
for i := range objs.Services {
|
|
// need to use a ref otherwise s would be a copy
|
|
s := &objs.Services[i]
|
|
|
|
if s.Labels == nil {
|
|
s.Labels = map[string]string{}
|
|
}
|
|
|
|
for k, v := range additionalLabels {
|
|
|
|
if len(s.Labels[k]) == 0 {
|
|
s.Labels[k] = v
|
|
}
|
|
}
|
|
|
|
if s.Annotations == nil {
|
|
s.Annotations = map[string]string{}
|
|
}
|
|
|
|
for k, v := range annos {
|
|
|
|
if len(s.Annotations[k]) == 0 {
|
|
s.Annotations[k] = v
|
|
}
|
|
}
|
|
}
|
|
|
|
// k, now we have to create our own objects for configuring the gateway
|
|
configMaps, err := c.getGatewayConfigsAndFillRoutingObjects(cheManager, workspaceMeta.DevWorkspaceId, routing, &objs)
|
|
if err != nil {
|
|
return solvers.RoutingObjects{}, err
|
|
}
|
|
|
|
syncer := sync.New(c.client, c.scheme)
|
|
|
|
for _, cm := range configMaps {
|
|
_, _, err := syncer.Sync(context.TODO(), nil, &cm, configMapDiffOpts)
|
|
if err != nil {
|
|
return solvers.RoutingObjects{}, err
|
|
}
|
|
|
|
}
|
|
|
|
return objs, nil
|
|
}
|
|
|
|
func (c *CheRoutingSolver) cheExposedEndpoints(manager *v2alpha1.CheCluster, workspaceID string, endpoints map[string]dwo.EndpointList, routingObj solvers.RoutingObjects) (exposedEndpoints map[string]dwo.ExposedEndpointList, ready bool, err error) {
|
|
if manager.Status.GatewayPhase == v2alpha1.GatewayPhaseInitializing {
|
|
return nil, false, nil
|
|
}
|
|
|
|
gatewayHost := manager.Status.GatewayHost
|
|
|
|
exposed := map[string]dwo.ExposedEndpointList{}
|
|
|
|
for machineName, endpoints := range endpoints {
|
|
exposedEndpoints := dwo.ExposedEndpointList{}
|
|
for _, endpoint := range endpoints {
|
|
if endpoint.Exposure != dw.PublicEndpointExposure {
|
|
continue
|
|
}
|
|
|
|
scheme := determineEndpointScheme(manager.Spec.Gateway.IsEnabled(), endpoint)
|
|
|
|
if !isExposableScheme(scheme) {
|
|
// we cannot expose non-http endpoints publicly, because ingresses/routes only support http(s)
|
|
continue
|
|
}
|
|
|
|
// try to find the endpoint in the ingresses/routes first. If it is there, it is exposed on a subdomain
|
|
// otherwise it is exposed through the gateway
|
|
var endpointURL string
|
|
if infrastructure.IsOpenShift() {
|
|
route := findRouteForEndpoint(machineName, endpoint, &routingObj)
|
|
if route != nil {
|
|
endpointURL = path.Join(route.Spec.Host, endpoint.Path)
|
|
}
|
|
} else {
|
|
ingress := findIngressForEndpoint(machineName, endpoint, &routingObj)
|
|
if ingress != nil {
|
|
endpointURL = path.Join(ingress.Spec.Rules[0].Host, endpoint.Path)
|
|
}
|
|
}
|
|
|
|
if endpointURL == "" {
|
|
if !manager.Spec.Gateway.IsEnabled() {
|
|
return map[string]dwo.ExposedEndpointList{}, false, fmt.Errorf("couldn't find an ingress/route for an endpoint `%s` in workspace `%s`, this is a bug", endpoint.Name, workspaceID)
|
|
}
|
|
|
|
if gatewayHost == "" {
|
|
// the gateway has not yet established the host
|
|
return map[string]dwo.ExposedEndpointList{}, false, nil
|
|
}
|
|
|
|
publicURLPrefix := getPublicURLPrefixForEndpoint(workspaceID, machineName, endpoint)
|
|
endpointURL = path.Join(gatewayHost, publicURLPrefix, endpoint.Path)
|
|
}
|
|
|
|
publicURL := scheme + "://" + endpointURL
|
|
|
|
// path.Join() removes the trailing slashes, so make sure to reintroduce that if required.
|
|
if endpoint.Path == "" || strings.HasSuffix(endpoint.Path, "/") {
|
|
publicURL = publicURL + "/"
|
|
}
|
|
|
|
exposedEndpoints = append(exposedEndpoints, dwo.ExposedEndpoint{
|
|
Name: endpoint.Name,
|
|
Url: publicURL,
|
|
Attributes: endpoint.Attributes,
|
|
})
|
|
}
|
|
exposed[machineName] = exposedEndpoints
|
|
}
|
|
|
|
return exposed, true, nil
|
|
}
|
|
|
|
func isExposableScheme(scheme string) bool {
|
|
return strings.HasPrefix(scheme, "http") || strings.HasPrefix(scheme, "ws")
|
|
}
|
|
|
|
func secureScheme(scheme string) string {
|
|
if scheme == "http" {
|
|
return "https"
|
|
} else if scheme == "ws" {
|
|
return "wss"
|
|
} else {
|
|
return scheme
|
|
}
|
|
}
|
|
|
|
func isSecureScheme(scheme string) bool {
|
|
return scheme == "https" || scheme == "wss"
|
|
}
|
|
|
|
func (c *CheRoutingSolver) getGatewayConfigsAndFillRoutingObjects(cheManager *v2alpha1.CheCluster, workspaceID string, routing *dwo.DevWorkspaceRouting, objs *solvers.RoutingObjects) ([]corev1.ConfigMap, error) {
|
|
restrictedAnno, setRestrictedAnno := routing.Annotations[constants.DevWorkspaceRestrictedAccessAnnotation]
|
|
|
|
labels := defaults.AddStandardLabelsForComponent(cheManager, "gateway-config", defaults.GetGatewayWorkspaceConfigMapLabels(cheManager))
|
|
labels[constants.DevWorkspaceIDLabel] = workspaceID
|
|
if setRestrictedAnno {
|
|
labels[constants.DevWorkspaceRestrictedAccessAnnotation] = restrictedAnno
|
|
}
|
|
|
|
configMap := corev1.ConfigMap{
|
|
ObjectMeta: v1.ObjectMeta{
|
|
Name: defaults.GetGatewayWorkpaceConfigMapName(workspaceID),
|
|
Namespace: cheManager.Namespace,
|
|
Labels: labels,
|
|
Annotations: map[string]string{
|
|
defaults.ConfigAnnotationDevWorkspaceRoutingName: routing.Name,
|
|
defaults.ConfigAnnotationDevWorkspaceRoutingNamespace: routing.Namespace,
|
|
},
|
|
},
|
|
Data: map[string]string{},
|
|
}
|
|
|
|
config := traefikConfig{
|
|
HTTP: traefikConfigHTTP{
|
|
Routers: map[string]traefikConfigRouter{},
|
|
Services: map[string]traefikConfigService{},
|
|
Middlewares: map[string]traefikConfigMiddleware{},
|
|
},
|
|
}
|
|
|
|
// we just need something to make the route names unique.. We also need to make the names as short as possible while
|
|
// being relatable to the workspaceID by mere human inspection. So let's just suffix the workspaceID with a "unique"
|
|
// suffix, the easiest of which is the iteration order in the map.
|
|
// Note that this means that the endpoints might get a different route/ingress name on each workspace start because
|
|
// the iteration order is not guaranteed in Go maps. If we want stable ingress/route names for the endpoints, we need
|
|
// to devise a different algorithm to produce them. Some kind of hash of workspaceID, component name, endpoint name and port
|
|
// might work but will not be relatable to the workspace ID just by looking at it anymore.
|
|
order := 0
|
|
if infrastructure.IsOpenShift() {
|
|
exposer := &RouteExposer{}
|
|
if err := exposer.initFrom(context.TODO(), c.client, cheManager, routing); err != nil {
|
|
return []corev1.ConfigMap{}, err
|
|
}
|
|
|
|
exposeAllEndpoints(&order, cheManager, routing, &config, objs, func(info *EndpointInfo) {
|
|
route := exposer.getRouteForService(info)
|
|
objs.Routes = append(objs.Routes, route)
|
|
})
|
|
} else {
|
|
exposer := &IngressExposer{}
|
|
if err := exposer.initFrom(context.TODO(), c.client, cheManager, routing, defaults.GetIngressAnnotations(cheManager)); err != nil {
|
|
return []corev1.ConfigMap{}, err
|
|
}
|
|
|
|
exposeAllEndpoints(&order, cheManager, routing, &config, objs, func(info *EndpointInfo) {
|
|
ingress := exposer.getIngressForService(info)
|
|
objs.Ingresses = append(objs.Ingresses, ingress)
|
|
})
|
|
}
|
|
|
|
if len(config.HTTP.Routers) > 0 {
|
|
contents, err := yaml.Marshal(config)
|
|
if err != nil {
|
|
return []corev1.ConfigMap{}, err
|
|
}
|
|
|
|
configMap.Data[workspaceID+".yml"] = string(contents)
|
|
|
|
return []corev1.ConfigMap{configMap}, nil
|
|
}
|
|
|
|
return []corev1.ConfigMap{}, nil
|
|
}
|
|
|
|
func exposeAllEndpoints(order *int, cheManager *v2alpha1.CheCluster, routing *dwo.DevWorkspaceRouting, config *traefikConfig, objs *solvers.RoutingObjects, ingressExpose func(*EndpointInfo)) {
|
|
info := &EndpointInfo{}
|
|
for componentName, endpoints := range routing.Spec.Endpoints {
|
|
info.componentName = componentName
|
|
singlehostPorts, multihostPorts := classifyEndpoints(cheManager.Spec.Gateway.IsEnabled(), order, &endpoints)
|
|
|
|
addToTraefikConfig(routing.Namespace, routing.Spec.DevWorkspaceId, componentName, singlehostPorts, config)
|
|
|
|
for port, names := range multihostPorts {
|
|
backingService := findServiceForPort(port, objs)
|
|
for endpointName, val := range names {
|
|
info.endpointName = endpointName
|
|
info.order = val.order
|
|
info.port = port
|
|
info.scheme = val.endpointScheme
|
|
info.service = backingService
|
|
|
|
ingressExpose(info)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func getTrackedEndpointName(endpoint *dw.Endpoint) string {
|
|
name := ""
|
|
if endpoint.Attributes.GetString(uniqueEndpointAttributeName, nil) == "true" {
|
|
name = endpoint.Name
|
|
}
|
|
|
|
return name
|
|
}
|
|
|
|
// we need to support unique endpoints - so 1 port can actually be accessible
|
|
// multiple times, each time using a different resulting external URL.
|
|
// non-unique endpoints are all represented using a single external URL
|
|
func classifyEndpoints(gatewayEnabled bool, order *int, endpoints *dwo.EndpointList) (singlehostPorts portMapping, multihostPorts portMapping) {
|
|
singlehostPorts = portMapping{}
|
|
multihostPorts = portMapping{}
|
|
for _, e := range *endpoints {
|
|
if e.Exposure != dw.PublicEndpointExposure {
|
|
continue
|
|
}
|
|
|
|
i := int32(e.TargetPort)
|
|
|
|
name := ""
|
|
if e.Attributes.GetString(uniqueEndpointAttributeName, nil) == "true" {
|
|
name = e.Name
|
|
}
|
|
|
|
ports := multihostPorts
|
|
if gatewayEnabled && e.Attributes.GetString(urlRewriteSupportedEndpointAttributeName, nil) == "true" {
|
|
ports = singlehostPorts
|
|
}
|
|
|
|
if ports[i] == nil {
|
|
ports[i] = map[string]portMappingValue{}
|
|
}
|
|
|
|
if _, ok := ports[i][name]; !ok {
|
|
ports[i][name] = portMappingValue{
|
|
order: *order,
|
|
endpointScheme: determineEndpointScheme(gatewayEnabled, e),
|
|
}
|
|
*order = *order + 1
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func addToTraefikConfig(namespace string, workspaceID string, machineName string, portMapping portMapping, cfg *traefikConfig) {
|
|
rtrs := cfg.HTTP.Routers
|
|
srvcs := cfg.HTTP.Services
|
|
mdls := cfg.HTTP.Middlewares
|
|
|
|
for port, names := range portMapping {
|
|
for endpointName := range names {
|
|
name := getEndpointExposingObjectName(machineName, workspaceID, port, endpointName)
|
|
var prefix string
|
|
var serviceURL string
|
|
|
|
prefix = getPublicURLPrefix(workspaceID, machineName, port, endpointName)
|
|
serviceURL = getServiceURL(port, workspaceID, namespace)
|
|
|
|
rtrs[name] = traefikConfigRouter{
|
|
Rule: fmt.Sprintf("PathPrefix(`%s`)", prefix),
|
|
Service: name,
|
|
Middlewares: []string{name},
|
|
Priority: 100,
|
|
}
|
|
|
|
srvcs[name] = traefikConfigService{
|
|
LoadBalancer: traefikConfigLoadbalancer{
|
|
Servers: []traefikConfigLoadbalancerServer{
|
|
{
|
|
URL: serviceURL,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
mdls[name] = traefikConfigMiddleware{
|
|
StripPrefix: traefikConfigStripPrefix{
|
|
Prefixes: []string{prefix},
|
|
},
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func findServiceForPort(port int32, objs *solvers.RoutingObjects) *corev1.Service {
|
|
for i := range objs.Services {
|
|
svc := &objs.Services[i]
|
|
for j := range svc.Spec.Ports {
|
|
if svc.Spec.Ports[j].Port == port {
|
|
return svc
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func findIngressForEndpoint(machineName string, endpoint dw.Endpoint, objs *solvers.RoutingObjects) *v1beta1.Ingress {
|
|
for i := range objs.Ingresses {
|
|
ingress := &objs.Ingresses[i]
|
|
|
|
if ingress.Annotations[defaults.ConfigAnnotationComponentName] != machineName ||
|
|
ingress.Annotations[defaults.ConfigAnnotationEndpointName] != getTrackedEndpointName(&endpoint) {
|
|
continue
|
|
}
|
|
|
|
for r := range ingress.Spec.Rules {
|
|
rule := ingress.Spec.Rules[r]
|
|
for p := range rule.HTTP.Paths {
|
|
path := rule.HTTP.Paths[p]
|
|
if path.Backend.ServicePort.IntVal == int32(endpoint.TargetPort) {
|
|
return ingress
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func findRouteForEndpoint(machineName string, endpoint dw.Endpoint, objs *solvers.RoutingObjects) *routeV1.Route {
|
|
service := findServiceForPort(int32(endpoint.TargetPort), objs)
|
|
|
|
for r := range objs.Routes {
|
|
route := &objs.Routes[r]
|
|
if route.Annotations[defaults.ConfigAnnotationComponentName] == machineName &&
|
|
route.Annotations[defaults.ConfigAnnotationEndpointName] == getTrackedEndpointName(&endpoint) &&
|
|
route.Spec.To.Kind == "Service" &&
|
|
route.Spec.To.Name == service.Name &&
|
|
route.Spec.Port.TargetPort.IntValue() == endpoint.TargetPort {
|
|
return route
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *CheRoutingSolver) cheRoutingFinalize(cheManager *v2alpha1.CheCluster, routing *dwo.DevWorkspaceRouting) error {
|
|
configs := &corev1.ConfigMapList{}
|
|
|
|
selector, err := labels.Parse(fmt.Sprintf("%s=%s", constants.DevWorkspaceIDLabel, routing.Spec.DevWorkspaceId))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
listOpts := &client.ListOptions{
|
|
Namespace: cheManager.Namespace,
|
|
LabelSelector: selector,
|
|
}
|
|
|
|
err = c.client.List(context.TODO(), configs, listOpts)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, cm := range configs.Items {
|
|
err = c.client.Delete(context.TODO(), &cm)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func getServiceURL(port int32, workspaceID string, workspaceNamespace string) string {
|
|
// the default .cluster.local suffix of the internal domain names seems to be configurable, so let's just
|
|
// not use it so we don't have to know about it...
|
|
return fmt.Sprintf("http://%s.%s.svc:%d", common.ServiceName(workspaceID), workspaceNamespace, port)
|
|
}
|
|
|
|
func getPublicURLPrefixForEndpoint(workspaceID string, machineName string, endpoint dw.Endpoint) string {
|
|
endpointName := ""
|
|
if endpoint.Attributes.GetString(uniqueEndpointAttributeName, nil) == "true" {
|
|
endpointName = endpoint.Name
|
|
}
|
|
|
|
return getPublicURLPrefix(workspaceID, machineName, int32(endpoint.TargetPort), endpointName)
|
|
}
|
|
|
|
func getPublicURLPrefix(workspaceID string, machineName string, port int32, uniqueEndpointName string) string {
|
|
if uniqueEndpointName == "" {
|
|
return fmt.Sprintf(endpointURLPrefixPattern, workspaceID, machineName, port)
|
|
}
|
|
return fmt.Sprintf(uniqueEndpointURLPrefixPattern, workspaceID, machineName, uniqueEndpointName)
|
|
}
|
|
|
|
func determineEndpointScheme(gatewayEnabled bool, e dw.Endpoint) string {
|
|
var scheme string
|
|
if e.Protocol == "" {
|
|
scheme = "http"
|
|
} else {
|
|
scheme = string(e.Protocol)
|
|
}
|
|
|
|
upgradeToSecure := e.Secure
|
|
|
|
// gateway is always on HTTPS, so if the endpoint is served through the gateway, we need to use the TLS'd variant.
|
|
if gatewayEnabled && e.Attributes.GetString(urlRewriteSupportedEndpointAttributeName, nil) == "true" {
|
|
upgradeToSecure = true
|
|
}
|
|
|
|
if upgradeToSecure {
|
|
scheme = secureScheme(scheme)
|
|
}
|
|
|
|
return scheme
|
|
}
|