From 21e00e9b15795415259ed06d65d83fbd1eb47913 Mon Sep 17 00:00:00 2001 From: David Festal Date: Thu, 22 Feb 2018 18:23:10 +0100 Subject: [PATCH] Make CHE 6 multi-tenant compatible for OSIO (#8805) * Copy the boostrapper config file to POD *installer by installer* to avoid executing a huge command, which in turn tries to send a huge GET request (73kb-long URL). This long GET request was previously not supported by the fabric8 oso proxy. See https://github.com/openshiftio/openshift.io/issues/2254 for more details. * Make the Kubernetes / Openshift infrastructures multi-tenant-compatible This includes: 1. Reworking the `KubernetesClientFactory` and `OpenshiftClientFactory` to: - share a single `OkHttpClient` that is the basis of all created Kubernetes or Openshift clients potentially pointing to different.cluster URL with different authorization tokens - provide the workspaceId in the `create` methods whenever it's available (workspace start / stop, idling, etc ... - have distinct methods for creating the Kubernetes client and the Openshift client (required since the Openshift client creates connection leaks on some calls such as exec of POD watch) This is the implementation of issue https://github.com/redhat-developer/rh-che/issues/516 2. Adding the `userId` into the SPI `RuntimeIdentity` object. Currently, only the `userName` is available in this object that gathers information about the identity attached to a workspace runtime. This change is required because the `userId` should be accessible from the `workspaceId` for implementing multi-tenancy and creating workspace resources in a user-specific location. This is the implementation of issue https://github.com/redhat-developer/rh-che/issues/501 * In the Openshift infrastructure, the authentication interceptor should always convert userName/password-based authentication to oauth-token-based authentication, even when using the `KubernetesClient`. Signed-off-by: David Festal --- .../bootstrapper/booter/booter_test.go | 3 +- .../go-agents/bootstrapper/booter/events.go | 7 +- agents/go-agents/bootstrapper/cfg/cfg.go | 11 +- .../workspace/runtime/RuntimeIdentity.java | 4 +- .../infrastructure/docker/Labels.java | 7 +- .../docker/bootstrap/DockerBootstrapper.java | 5 +- .../DockerEnvironmentNormalizer.java | 2 +- .../docker/DockerInternalRuntimeTest.java | 3 +- .../infrastructure/docker/LabelsTest.java | 7 +- .../container/DockerContainersTest.java | 8 +- .../WsAgentServerConfigProvisionerTest.java | 2 +- ...indMountProjectsVolumeProvisionerTest.java | 2 +- .../mapping/SinglePortUrlRewriterTest.java | 11 +- .../kubernetes/KubernetesClientFactory.java | 156 +++++++++++++++-- .../bootstrapper/KubernetesBootstrapper.java | 36 ++-- .../namespace/KubernetesIngresses.java | 11 +- .../namespace/KubernetesNamespace.java | 4 +- .../KubernetesPersistentVolumeClaims.java | 15 +- .../kubernetes/namespace/KubernetesPods.java | 19 ++- .../namespace/KubernetesServices.java | 6 +- .../RemoveNamespaceOnWorkspaceRemove.java | 2 +- .../pvc/UniqueWorkspacePVCStrategy.java | 2 +- .../KubernetesInternalRuntimeTest.java | 2 +- .../namespace/KubernetesNamespaceTest.java | 1 + .../namespace/pvc/CommonPVCStrategyTest.java | 2 +- .../pvc/UniqueWorkspacePVCStrategyTest.java | 4 +- infrastructures/openshift/pom.xml | 4 + .../openshift/OpenShiftClientFactory.java | 160 ++++++++++++++++-- .../openshift/project/OpenShiftProject.java | 15 +- .../openshift/project/OpenShiftRoutes.java | 6 +- .../RemoveProjectOnWorkspaceRemove.java | 2 +- .../OpenShiftInternalRuntimeTest.java | 2 +- .../project/OpenShiftProjectTest.java | 9 +- .../shared/dto/RuntimeIdentityDto.java | 9 +- .../api/workspace/server/DtoConverter.java | 3 +- .../workspace/server/WorkspaceRuntimes.java | 2 +- .../bootstrap/AbstractBootstrapper.java | 2 +- .../model/impl/RuntimeIdentityImpl.java | 19 ++- .../workspace/server/spi/InternalRuntime.java | 2 +- .../spi/RuntimeStartInterruptedException.java | 2 +- .../server/WorkspaceManagerTest.java | 5 +- .../server/WorkspaceRuntimesTest.java | 13 +- .../server/spi/InternalRuntimeTest.java | 2 +- .../env/EnvVarEnvironmentProvisionerTest.java | 2 +- 44 files changed, 466 insertions(+), 125 deletions(-) diff --git a/agents/go-agents/bootstrapper/booter/booter_test.go b/agents/go-agents/bootstrapper/booter/booter_test.go index ed2bd1e452..7a9b6d8624 100644 --- a/agents/go-agents/bootstrapper/booter/booter_test.go +++ b/agents/go-agents/bootstrapper/booter/booter_test.go @@ -26,7 +26,8 @@ var ( testRuntimeID = RuntimeID{ Workspace: "my-workspace", Environment: "my-env", - Owner: "me", + OwnerName: "me", + OwnerId: "id", } testMachineName = "my-machine" diff --git a/agents/go-agents/bootstrapper/booter/events.go b/agents/go-agents/bootstrapper/booter/events.go index ba3e623131..2ce8b4f0b3 100644 --- a/agents/go-agents/bootstrapper/booter/events.go +++ b/agents/go-agents/bootstrapper/booter/events.go @@ -79,8 +79,11 @@ type RuntimeID struct { // Environment is a name of environment e.g. "default". Environment string `json:"envName"` - // Owner is an identifier of user who is runtime owner. - Owner string `json:"owner"` + // OwnerName is the name of user who is runtime owner. + OwnerName string `json:"ownerName"` + + // OwnerId is an identifier of user who is runtime owner. + OwnerId string `json:"ownerId"` } // MachineEvent is a base event for bootstrapper events. diff --git a/agents/go-agents/bootstrapper/cfg/cfg.go b/agents/go-agents/bootstrapper/cfg/cfg.go index 412f80b0b0..041c307def 100644 --- a/agents/go-agents/bootstrapper/cfg/cfg.go +++ b/agents/go-agents/bootstrapper/cfg/cfg.go @@ -97,7 +97,7 @@ func init() { &runtimeIDRaw, "runtime-id", "", - "The identifier of the runtime in format 'workspace:environment:owner'", + "The identifier of the runtime in format 'workspace:environment:ownerName:ownerId'", ) flag.StringVar( &MachineName, @@ -155,10 +155,10 @@ func Parse() { log.Fatal("Runtime ID required(set it with -runtime-id argument)") } parts := strings.Split(runtimeIDRaw, ":") - if len(parts) != 3 { - log.Fatalf("Expected runtime id to be in format 'workspace:env:owner'") + if len(parts) != 4 { + log.Fatalf("Expected runtime id to be in format 'workspace:env:ownerName:ownerId'") } - RuntimeID = booter.RuntimeID{Workspace: parts[0], Environment: parts[1], Owner: parts[2]} + RuntimeID = booter.RuntimeID{Workspace: parts[0], Environment: parts[1], OwnerName: parts[2], OwnerId: parts[3]} // machine-name if len(MachineName) == 0 { @@ -182,7 +182,8 @@ func Print() { log.Print(" Runtime ID:") log.Printf(" Workspace: %s", RuntimeID.Workspace) log.Printf(" Environment: %s", RuntimeID.Environment) - log.Printf(" Owner: %s", RuntimeID.Owner) + log.Printf(" OwnerName: %s", RuntimeID.OwnerName) + log.Printf(" OwnerId: %s", RuntimeID.OwnerId) log.Printf(" Machine name: %s", MachineName) log.Printf(" Installer timeout: %dseconds", InstallerTimeoutSec) log.Printf(" Check servers period: %dseconds", CheckServersPeriodSec) diff --git a/core/che-core-api-model/src/main/java/org/eclipse/che/api/core/model/workspace/runtime/RuntimeIdentity.java b/core/che-core-api-model/src/main/java/org/eclipse/che/api/core/model/workspace/runtime/RuntimeIdentity.java index b3cd1c36e7..0cb0ef2dfc 100644 --- a/core/che-core-api-model/src/main/java/org/eclipse/che/api/core/model/workspace/runtime/RuntimeIdentity.java +++ b/core/che-core-api-model/src/main/java/org/eclipse/che/api/core/model/workspace/runtime/RuntimeIdentity.java @@ -16,5 +16,7 @@ public interface RuntimeIdentity { String getEnvName(); - String getOwner(); + String getOwnerName(); + + String getOwnerId(); } diff --git a/infrastructures/docker/infrastructure/src/main/java/org/eclipse/che/workspace/infrastructure/docker/Labels.java b/infrastructures/docker/infrastructure/src/main/java/org/eclipse/che/workspace/infrastructure/docker/Labels.java index dcfae349af..36a92f09c3 100644 --- a/infrastructures/docker/infrastructure/src/main/java/org/eclipse/che/workspace/infrastructure/docker/Labels.java +++ b/infrastructures/docker/infrastructure/src/main/java/org/eclipse/che/workspace/infrastructure/docker/Labels.java @@ -34,6 +34,7 @@ public final class Labels { public static final String LABEL_WORKSPACE_ID = LABEL_PREFIX + "workspace.id"; public static final String LABEL_WORKSPACE_ENV = LABEL_PREFIX + "workspace.env"; public static final String LABEL_WORKSPACE_OWNER = LABEL_PREFIX + "workspace.owner"; + public static final String LABEL_WORKSPACE_OWNER_ID = LABEL_PREFIX + "workspace.owner.id"; private static final String LABEL_MACHINE_NAME = LABEL_PREFIX + "machine.name"; private static final String LABEL_MACHINE_ATTRIBUTES = LABEL_PREFIX + "machine.attributes"; @@ -88,7 +89,8 @@ public final class Labels { public Serializer runtimeId(RuntimeIdentity runtimeId) { labels.put(LABEL_WORKSPACE_ID, runtimeId.getWorkspaceId()); labels.put(LABEL_WORKSPACE_ENV, runtimeId.getEnvName()); - labels.put(LABEL_WORKSPACE_OWNER, runtimeId.getOwner()); + labels.put(LABEL_WORKSPACE_OWNER, runtimeId.getOwnerName()); + labels.put(LABEL_WORKSPACE_OWNER_ID, runtimeId.getOwnerId()); return this; } @@ -159,7 +161,8 @@ public final class Labels { return new RuntimeIdentityImpl( labels.get(LABEL_WORKSPACE_ID), labels.get(LABEL_WORKSPACE_ENV), - labels.get(LABEL_WORKSPACE_OWNER)); + labels.get(LABEL_WORKSPACE_OWNER), + labels.get(LABEL_WORKSPACE_OWNER_ID)); } /** Retrieves server configuration from docker labels and returns (ref -> server config) map. */ diff --git a/infrastructures/docker/infrastructure/src/main/java/org/eclipse/che/workspace/infrastructure/docker/bootstrap/DockerBootstrapper.java b/infrastructures/docker/infrastructure/src/main/java/org/eclipse/che/workspace/infrastructure/docker/bootstrap/DockerBootstrapper.java index c028421968..ee3f04eb87 100644 --- a/infrastructures/docker/infrastructure/src/main/java/org/eclipse/che/workspace/infrastructure/docker/bootstrap/DockerBootstrapper.java +++ b/infrastructures/docker/infrastructure/src/main/java/org/eclipse/che/workspace/infrastructure/docker/bootstrap/DockerBootstrapper.java @@ -92,10 +92,11 @@ public class DockerBootstrapper extends AbstractBootstrapper { + machineName + " -runtime-id " + String.format( - "%s:%s:%s", + "%s:%s:%s:%s", runtimeIdentity.getWorkspaceId(), runtimeIdentity.getEnvName(), - runtimeIdentity.getOwner()) + runtimeIdentity.getOwnerName(), + runtimeIdentity.getOwnerId()) + " -push-endpoint " + installerWebsocketEndpoint + " -push-logs-endpoint " diff --git a/infrastructures/docker/infrastructure/src/main/java/org/eclipse/che/workspace/infrastructure/docker/environment/DockerEnvironmentNormalizer.java b/infrastructures/docker/infrastructure/src/main/java/org/eclipse/che/workspace/infrastructure/docker/environment/DockerEnvironmentNormalizer.java index 51fd51aa3c..b4d6d4425c 100644 --- a/infrastructures/docker/infrastructure/src/main/java/org/eclipse/che/workspace/infrastructure/docker/environment/DockerEnvironmentNormalizer.java +++ b/infrastructures/docker/infrastructure/src/main/java/org/eclipse/che/workspace/infrastructure/docker/environment/DockerEnvironmentNormalizer.java @@ -44,7 +44,7 @@ public class DockerEnvironmentNormalizer { containerNameGenerator.generateContainerName( identity.getWorkspaceId(), containerConfig.getId(), - identity.getOwner(), + identity.getOwnerName(), containerEntry.getKey())); } normalizeNames(dockerEnvironment); diff --git a/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/DockerInternalRuntimeTest.java b/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/DockerInternalRuntimeTest.java index 429f2a33e8..8c99ce03d8 100644 --- a/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/DockerInternalRuntimeTest.java +++ b/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/DockerInternalRuntimeTest.java @@ -86,7 +86,8 @@ import org.testng.annotations.Test; */ public class DockerInternalRuntimeTest { - private static final RuntimeIdentity IDENTITY = new RuntimeIdentityImpl("ws1", "env1", "usr1"); + private static final RuntimeIdentity IDENTITY = + new RuntimeIdentityImpl("ws1", "env1", "usr1", "id1"); private static final String DEV_MACHINE = "DEV_MACHINE"; private static final String DB_MACHINE = "DB_MACHINE"; private static final String SERVER_1 = "serv1"; diff --git a/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/LabelsTest.java b/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/LabelsTest.java index 0771d8642b..9ab1dd536d 100644 --- a/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/LabelsTest.java +++ b/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/LabelsTest.java @@ -43,7 +43,7 @@ public class LabelsTest { Map serialized = Labels.newSerializer() .machineName("dev-machine") - .runtimeId(new RuntimeIdentityImpl("workspace123", "my-env", "owner")) + .runtimeId(new RuntimeIdentityImpl("workspace123", "my-env", "owner", "id")) .server( "my-server1/http", new ServerConfigImpl("8000/tcp", "http", "/api/info", emptyMap())) @@ -60,6 +60,7 @@ public class LabelsTest { .put("org.eclipse.che.workspace.id", "workspace123") .put("org.eclipse.che.workspace.env", "my-env") .put("org.eclipse.che.workspace.owner", "owner") + .put("org.eclipse.che.workspace.owner.id", "id") .put("org.eclipse.che.server.my-server1/http.port", "8000/tcp") .put("org.eclipse.che.server.my-server1/http.protocol", "http") .put("org.eclipse.che.server.my-server1/http.path", "/api/info") @@ -90,6 +91,7 @@ public class LabelsTest { .put("org.eclipse.che.workspace.id", "workspace123") .put("org.eclipse.che.workspace.env", "my-env") .put("org.eclipse.che.workspace.owner", "owner") + .put("org.eclipse.che.workspace.owner.id", "id") .put("org.eclipse.che.server.my-server1/http.port", "8000/tcp") .put("org.eclipse.che.server.my-server1/http.protocol", "http") .put("org.eclipse.che.server.my-server1/http.path", "/api/info") @@ -121,7 +123,8 @@ public class LabelsTest { RuntimeIdentity runtimeId = deserializer.runtimeId(); assertEquals(runtimeId.getWorkspaceId(), "workspace123", "workspace id"); assertEquals(runtimeId.getEnvName(), "my-env", "workspace environment name"); - assertEquals(runtimeId.getOwner(), "owner", "workspace owner"); + assertEquals(runtimeId.getOwnerName(), "owner", "workspace owner name"); + assertEquals(runtimeId.getOwnerId(), "id", "workspace owner id"); Map servers = deserializer.servers(); assertEquals(servers, expectedServers); diff --git a/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/container/DockerContainersTest.java b/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/container/DockerContainersTest.java index b8e83bfd03..f81a64d796 100644 --- a/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/container/DockerContainersTest.java +++ b/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/container/DockerContainersTest.java @@ -48,8 +48,8 @@ public class DockerContainersTest { @Test public void findsIdentifiers() throws Exception { - RuntimeIdentity id1 = new RuntimeIdentityImpl("workspace123", "default", "test"); - RuntimeIdentity id2 = new RuntimeIdentityImpl("workspace234", "default", "test"); + RuntimeIdentity id1 = new RuntimeIdentityImpl("workspace123", "default", "test", "id"); + RuntimeIdentity id2 = new RuntimeIdentityImpl("workspace234", "default", "test", "id"); List entries = asList(mockContainer(id1, "container1"), mockContainer(id2, "container2")); @@ -81,7 +81,7 @@ public class DockerContainersTest { @Test public void findContainers() throws Exception { - RuntimeIdentity id = new RuntimeIdentityImpl("workspace123", "default", "test"); + RuntimeIdentity id = new RuntimeIdentityImpl("workspace123", "default", "test", "id"); ContainerListEntry entry1 = mockContainer(id, "container1"); ContainerListEntry entry2 = mockContainer(id, "container2"); @@ -113,7 +113,7 @@ public class DockerContainersTest { throws Exception { when(docker.listContainers(anyObject())).thenThrow(new IOException("oops")); - containers.find(new RuntimeIdentityImpl("workspace123", "default", "test")); + containers.find(new RuntimeIdentityImpl("workspace123", "default", "test", "id")); } private ContainerListEntry mockContainer(RuntimeIdentity runtimeId, String containerId) diff --git a/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/local/installer/WsAgentServerConfigProvisionerTest.java b/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/local/installer/WsAgentServerConfigProvisionerTest.java index faa59ef902..7282bd7e72 100644 --- a/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/local/installer/WsAgentServerConfigProvisionerTest.java +++ b/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/local/installer/WsAgentServerConfigProvisionerTest.java @@ -36,7 +36,7 @@ import org.testng.annotations.Test; @Listeners(MockitoTestNGListener.class) public class WsAgentServerConfigProvisionerTest { private static final RuntimeIdentity RUNTIME_IDENTITY = - new RuntimeIdentityImpl("wsId", "env", "owner"); + new RuntimeIdentityImpl("wsId", "env", "owner", "id"); private static final String MACHINE_1_NAME = "machine1"; private static final String MACHINE_2_NAME = "machine2"; diff --git a/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/local/projects/BindMountProjectsVolumeProvisionerTest.java b/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/local/projects/BindMountProjectsVolumeProvisionerTest.java index c46636bf79..55927c5a9d 100644 --- a/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/local/projects/BindMountProjectsVolumeProvisionerTest.java +++ b/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/local/projects/BindMountProjectsVolumeProvisionerTest.java @@ -41,7 +41,7 @@ import org.testng.annotations.Test; public class BindMountProjectsVolumeProvisionerTest { private static final String WORKSPACE_ID = "wsId"; private static final RuntimeIdentity RUNTIME_IDENTITY = - new RuntimeIdentityImpl(WORKSPACE_ID, "env", "owner"); + new RuntimeIdentityImpl(WORKSPACE_ID, "env", "owner", "id"); private static final String MACHINE_1_NAME = "machine1"; private static final String MACHINE_2_NAME = "machine2"; private static final String MACHINE_3_NAME = "machine3"; diff --git a/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/server/mapping/SinglePortUrlRewriterTest.java b/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/server/mapping/SinglePortUrlRewriterTest.java index 86e4220506..125915dcc9 100644 --- a/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/server/mapping/SinglePortUrlRewriterTest.java +++ b/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/server/mapping/SinglePortUrlRewriterTest.java @@ -47,7 +47,7 @@ public class SinglePortUrlRewriterTest { return new Object[][] { // External IP { - new RuntimeIdentityImpl("ws123", null, null), + new RuntimeIdentityImpl("ws123", null, null, null), "172.12.0.2", "127.0.0.1", "machine1", @@ -58,7 +58,7 @@ public class SinglePortUrlRewriterTest { }, // Internal IP, protocol, path param { - new RuntimeIdentityImpl("ws123", null, null), + new RuntimeIdentityImpl("ws123", null, null, null), "127.0.0.1", null, "machine1", @@ -69,7 +69,7 @@ public class SinglePortUrlRewriterTest { }, // Without machine name { - new RuntimeIdentityImpl("ws123", null, null), + new RuntimeIdentityImpl("ws123", null, null, null), "127.0.0.1", null, null, @@ -80,7 +80,7 @@ public class SinglePortUrlRewriterTest { }, // Without server { - new RuntimeIdentityImpl("ws123", null, null), + new RuntimeIdentityImpl("ws123", null, null, null), "127.0.0.1", null, "machine1", @@ -101,6 +101,7 @@ public class SinglePortUrlRewriterTest { Provider provider = () -> new SinglePortHostnameBuilder("172.12.0.2", "127.0.0.1", null); SinglePortUrlRewriter rewriter = new SinglePortUrlRewriter(8080, provider); - rewriter.rewriteURL(new RuntimeIdentityImpl("ws123", null, null), "machine1", "server", ":"); + rewriter.rewriteURL( + new RuntimeIdentityImpl("ws123", null, null, null), "machine1", "server", ":"); } } diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesClientFactory.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesClientFactory.java index 2d2fb63283..865474fbec 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesClientFactory.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesClientFactory.java @@ -11,15 +11,28 @@ package org.eclipse.che.workspace.infrastructure.kubernetes; import static com.google.common.base.Strings.isNullOrEmpty; +import static io.fabric8.kubernetes.client.utils.Utils.isNotNullOrEmpty; import io.fabric8.kubernetes.client.Config; import io.fabric8.kubernetes.client.ConfigBuilder; import io.fabric8.kubernetes.client.DefaultKubernetesClient; import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.utils.HttpClientUtils; +import io.fabric8.kubernetes.client.utils.Utils; +import java.io.IOException; +import java.util.concurrent.ExecutorService; import javax.annotation.PreDestroy; import javax.inject.Inject; import javax.inject.Named; import javax.inject.Singleton; +import okhttp3.Authenticator; +import okhttp3.ConnectionPool; +import okhttp3.Credentials; +import okhttp3.Dispatcher; +import okhttp3.Interceptor; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.commons.annotation.Nullable; import org.slf4j.Logger; @@ -34,7 +47,14 @@ public class KubernetesClientFactory { private static final Logger LOG = LoggerFactory.getLogger(KubernetesClientFactory.class); - private final UnclosableKubernetesClient client; + /** {@link OkHttpClient} instance shared by all Kubernetes clients. */ + private OkHttpClient httpClient; + + /** + * Default Kubernetes {@link Config} that will be the base configuration to create per-workspace + * configurations. + */ + private Config defaultConfig; @Inject public KubernetesClientFactory( @@ -43,6 +63,57 @@ public class KubernetesClientFactory { @Nullable @Named("che.infra.kubernetes.password") String password, @Nullable @Named("che.infra.kubernetes.oauth_token") String oauthToken, @Nullable @Named("che.infra.kubernetes.trust_certs") Boolean doTrustCerts) { + this.defaultConfig = + buildDefaultConfig(masterUrl, username, password, oauthToken, doTrustCerts); + this.httpClient = HttpClientUtils.createHttpClient(defaultConfig); + } + + /** + * Creates an instance of {@link KubernetesClient} that can be used to perform any operation + * related to a given workspace.
For all operations performed in the context of a given + * workspace (workspace start, workspace stop, etc ...), this method should be used to retrieve a + * Kubernetes client. + * + * @param workspaceId Identifier of the workspace on which Kubernetes operations will be performed + * @throws InfrastructureException if any error occurs on client instance creation. + */ + public KubernetesClient create(String workspaceId) throws InfrastructureException { + Config configForWorkspace = buildConfig(defaultConfig, workspaceId); + + return create(configForWorkspace); + } + + /** + * Creates an instance of {@link KubernetesClient} that can be used to perform any operation + * that is not related to a given workspace.
For all operations performed + * in the context of a given workspace (workspace start, workspace stop, etc ...), the {@code + * create(String workspaceId)} method should be used to retrieve a Kubernetes client. + * + * @throws InfrastructureException if any error occurs on client instance creation. + */ + public KubernetesClient create() throws InfrastructureException { + return create(buildConfig(defaultConfig, null)); + } + + /** Retrieves the {@link OkHttpClient} instance shared by all Kubernetes clients. */ + protected OkHttpClient getHttpClient() { + return httpClient; + } + + /** + * Retrieves the default Kubernetes {@link Config} that will be the base configuration to create + * per-workspace configurations. + */ + protected Config getDefaultConfig() { + return defaultConfig; + } + + /** + * Builds the default Kubernetes {@link Config} that will be the base configuration to create + * per-workspace configurations. + */ + protected Config buildDefaultConfig( + String masterUrl, String username, String password, String oauthToken, Boolean doTrustCerts) { ConfigBuilder configBuilder = new ConfigBuilder(); if (!isNullOrEmpty(masterUrl)) { configBuilder.withMasterUrl(masterUrl); @@ -63,22 +134,87 @@ public class KubernetesClientFactory { if (doTrustCerts != null) { configBuilder.withTrustCerts(doTrustCerts); } - this.client = new UnclosableKubernetesClient(configBuilder.build()); + + Config config = configBuilder.build(); + return config; } /** - * Creates instance of {@link KubernetesClient}. + * Builds the Kubernetes {@link Config} object based on a default {@link Config} object and an + * optional workspace Id. + */ + protected Config buildConfig(Config defaultConfig, @Nullable String workspaceId) + throws InfrastructureException { + return defaultConfig; + } + + protected Interceptor buildKubernetesInterceptor(Config config) { + return new Interceptor() { + @Override + public Response intercept(Chain chain) throws IOException { + Request request = chain.request(); + if (isNotNullOrEmpty(config.getUsername()) && isNotNullOrEmpty(config.getPassword())) { + Request authReq = + chain + .request() + .newBuilder() + .addHeader( + "Authorization", + Credentials.basic(config.getUsername(), config.getPassword())) + .build(); + return chain.proceed(authReq); + } else if (isNotNullOrEmpty(config.getOauthToken())) { + Request authReq = + chain + .request() + .newBuilder() + .addHeader("Authorization", "Bearer " + config.getOauthToken()) + .build(); + return chain.proceed(authReq); + } + return chain.proceed(request); + } + }; + } + + protected void doCleanup() { + ConnectionPool connectionPool = httpClient.connectionPool(); + Dispatcher dispatcher = httpClient.dispatcher(); + ExecutorService executorService = + httpClient.dispatcher() != null ? httpClient.dispatcher().executorService() : null; + + if (dispatcher != null) { + dispatcher.cancelAll(); + } + + if (connectionPool != null) { + connectionPool.evictAll(); + } + + Utils.shutdownExecutorService(executorService); + } + + /** + * Creates instance of {@link KubernetesClient} that uses an {@link OkHttpClient} instance derived + * from the shared {@code httpClient} instance in which interceptors are overriden to authenticate + * with the credentials (user/password or Oauth token) contained in the {@code config} parameter. * * @throws InfrastructureException if any error occurs on client instance creation. */ - public KubernetesClient create() throws InfrastructureException { - return client; + private KubernetesClient create(Config config) throws InfrastructureException { + OkHttpClient clientHttpClient = + httpClient.newBuilder().authenticator(Authenticator.NONE).build(); + OkHttpClient.Builder builder = clientHttpClient.newBuilder(); + builder.interceptors().clear(); + clientHttpClient = builder.addInterceptor(buildKubernetesInterceptor(config)).build(); + + return new UnclosableKubernetesClient(clientHttpClient, config); } @PreDestroy private void cleanup() { try { - client.doClose(); + doCleanup(); } catch (RuntimeException ex) { LOG.error(ex.getMessage()); } @@ -89,15 +225,11 @@ public class KubernetesClientFactory { */ private static class UnclosableKubernetesClient extends DefaultKubernetesClient { - public UnclosableKubernetesClient(Config config) { - super(config); + public UnclosableKubernetesClient(OkHttpClient httpClient, Config config) { + super(httpClient, config); } @Override public void close() {} - - void doClose() { - super.close(); - } } } diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/bootstrapper/KubernetesBootstrapper.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/bootstrapper/KubernetesBootstrapper.java index bc8fe23965..4c3111c564 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/bootstrapper/KubernetesBootstrapper.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/bootstrapper/KubernetesBootstrapper.java @@ -13,6 +13,7 @@ package org.eclipse.che.workspace.infrastructure.kubernetes.bootstrapper; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.inject.assistedinject.Assisted; +import java.util.ArrayList; import java.util.List; import javax.inject.Inject; import javax.inject.Named; @@ -93,10 +94,11 @@ public class KubernetesBootstrapper extends AbstractBootstrapper { + kubernetesMachine.getName() + " -runtime-id " + String.format( - "%s:%s:%s", + "%s:%s:%s:%s", runtimeIdentity.getWorkspaceId(), runtimeIdentity.getEnvName(), - runtimeIdentity.getOwner()) + runtimeIdentity.getOwnerName(), + runtimeIdentity.getOwnerId()) + " -push-endpoint " + installerWebsocketEndpoint + " -push-logs-endpoint " @@ -128,14 +130,26 @@ public class KubernetesBootstrapper extends AbstractBootstrapper { kubernetesMachine.exec("chmod", "+x", BOOTSTRAPPER_DIR + BOOTSTRAPPER_FILE); LOG.debug("Bootstrapping {}:{}. Creating config file", runtimeIdentity, machineName); - kubernetesMachine.exec( - "sh", - "-c", - "cat > " - + BOOTSTRAPPER_DIR - + CONFIG_FILE - + " << 'EOF'\n" - + GSON.toJson(installers) - + "\nEOF"); + + kubernetesMachine.exec("sh", "-c", "rm " + BOOTSTRAPPER_DIR + CONFIG_FILE); + + List contentsToContatenate = new ArrayList(); + contentsToContatenate.add("["); + boolean firstOne = true; + for (Installer installer : installers) { + if (firstOne) { + firstOne = false; + } else { + contentsToContatenate.add(","); + } + contentsToContatenate.add(GSON.toJson(installer)); + } + contentsToContatenate.add("]"); + for (String content : contentsToContatenate) { + kubernetesMachine.exec( + "sh", + "-c", + "cat >> " + BOOTSTRAPPER_DIR + CONFIG_FILE + " << 'EOF'\n" + content + "\nEOF"); + } } } diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesIngresses.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesIngresses.java index 6243321da4..ce8c95b08d 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesIngresses.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesIngresses.java @@ -51,7 +51,7 @@ public class KubernetesIngresses { putLabel(ingress, CHE_WORKSPACE_ID_LABEL, workspaceId); try { return clientFactory - .create() + .create(workspaceId) .extensions() .ingresses() .inNamespace(namespace) @@ -68,7 +68,12 @@ public class KubernetesIngresses { Watch watch = null; try { Resource ingressResource = - clientFactory.create().extensions().ingresses().inNamespace(namespace).withName(name); + clientFactory + .create(workspaceId) + .extensions() + .ingresses() + .inNamespace(namespace) + .withName(name); watch = ingressResource.watch( @@ -117,7 +122,7 @@ public class KubernetesIngresses { public void delete() throws InfrastructureException { try { clientFactory - .create() + .create(workspaceId) .extensions() .ingresses() .inNamespace(namespace) diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesNamespace.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesNamespace.java index 50c06de6b4..2c1b58a61d 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesNamespace.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesNamespace.java @@ -86,10 +86,10 @@ public class KubernetesNamespace { this.workspaceId = workspaceId; this.pods = new KubernetesPods(name, workspaceId, clientFactory); this.services = new KubernetesServices(name, workspaceId, clientFactory); - this.pvcs = new KubernetesPersistentVolumeClaims(name, clientFactory); + this.pvcs = new KubernetesPersistentVolumeClaims(name, workspaceId, clientFactory); this.ingresses = new KubernetesIngresses(name, workspaceId, clientFactory); if (doPrepare) { - doPrepare(name, clientFactory.create()); + doPrepare(name, clientFactory.create(workspaceId)); } } diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPersistentVolumeClaims.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPersistentVolumeClaims.java index d9ff89d3db..b85cadca40 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPersistentVolumeClaims.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPersistentVolumeClaims.java @@ -29,11 +29,14 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesInfrastruct */ public class KubernetesPersistentVolumeClaims { private final String namespace; + private final String workspaceId; private final KubernetesClientFactory clientFactory; - KubernetesPersistentVolumeClaims(String namespace, KubernetesClientFactory clientFactory) { + KubernetesPersistentVolumeClaims( + String namespace, String workspaceId, KubernetesClientFactory clientFactory) { this.namespace = namespace; this.clientFactory = clientFactory; + this.workspaceId = workspaceId; } /** @@ -45,7 +48,11 @@ public class KubernetesPersistentVolumeClaims { */ public PersistentVolumeClaim create(PersistentVolumeClaim pvc) throws InfrastructureException { try { - return clientFactory.create().persistentVolumeClaims().inNamespace(namespace).create(pvc); + return clientFactory + .create(workspaceId) + .persistentVolumeClaims() + .inNamespace(namespace) + .create(pvc); } catch (KubernetesClientException e) { throw new KubernetesInfrastructureException(e); } @@ -59,7 +66,7 @@ public class KubernetesPersistentVolumeClaims { public List get() throws InfrastructureException { try { return clientFactory - .create() + .create(workspaceId) .persistentVolumeClaims() .inNamespace(namespace) .list() @@ -81,7 +88,7 @@ public class KubernetesPersistentVolumeClaims { throws InfrastructureException { try { return clientFactory - .create() + .create(workspaceId) .persistentVolumeClaims() .inNamespace(namespace) .withLabel(labelName, labelValue) diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPods.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPods.java index a647bc43a5..b26508ac5c 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPods.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPods.java @@ -96,7 +96,7 @@ public class KubernetesPods { public Pod create(Pod pod) throws InfrastructureException { putLabel(pod, CHE_WORKSPACE_ID_LABEL, workspaceId); try { - return clientFactory.create().pods().inNamespace(namespace).create(pod); + return clientFactory.create(workspaceId).pods().inNamespace(namespace).create(pod); } catch (KubernetesClientException e) { throw new KubernetesInfrastructureException(e); } @@ -110,7 +110,7 @@ public class KubernetesPods { public List get() throws InfrastructureException { try { return clientFactory - .create() + .create(workspaceId) .pods() .inNamespace(namespace) .withLabel(CHE_WORKSPACE_ID_LABEL, workspaceId) @@ -129,7 +129,7 @@ public class KubernetesPods { public Optional get(String name) throws InfrastructureException { try { return Optional.ofNullable( - clientFactory.create().pods().inNamespace(namespace).withName(name).get()); + clientFactory.create(workspaceId).pods().inNamespace(namespace).withName(name).get()); } catch (KubernetesClientException e) { throw new KubernetesInfrastructureException(e); } @@ -153,7 +153,7 @@ public class KubernetesPods { try { PodResource podResource = - clientFactory.create().pods().inNamespace(namespace).withName(name); + clientFactory.create(workspaceId).pods().inNamespace(namespace).withName(name); watch = podResource.watch( @@ -222,7 +222,7 @@ public class KubernetesPods { try { podWatch = clientFactory - .create() + .create(workspaceId) .pods() .inNamespace(namespace) .withLabel(CHE_WORKSPACE_ID_LABEL, workspaceId) @@ -272,7 +272,8 @@ public class KubernetesPods { public void onClose(KubernetesClientException ignored) {} }; try { - containerWatch = clientFactory.create().events().inNamespace(namespace).watch(watcher); + containerWatch = + clientFactory.create(workspaceId).events().inNamespace(namespace).watch(watcher); } catch (KubernetesClientException ex) { throw new KubernetesInfrastructureException(ex); } @@ -321,7 +322,7 @@ public class KubernetesPods { final ExecWatchdog watchdog = new ExecWatchdog(); try (ExecWatch watch = clientFactory - .create() + .create(workspaceId) .pods() .inNamespace(namespace) .withName(podName) @@ -382,7 +383,7 @@ public class KubernetesPods { // pods are removed with some delay related to stopping of containers. It is need to wait them List pods = clientFactory - .create() + .create(workspaceId) .pods() .inNamespace(namespace) .withLabel(CHE_WORKSPACE_ID_LABEL, workspaceId) @@ -414,7 +415,7 @@ public class KubernetesPods { private CompletableFuture doDelete(String name) throws InfrastructureException { try { final PodResource podResource = - clientFactory.create().pods().inNamespace(namespace).withName(name); + clientFactory.create(workspaceId).pods().inNamespace(namespace).withName(name); final CompletableFuture deleteFuture = new CompletableFuture<>(); final Watch watch = podResource.watch(new DeleteWatcher(deleteFuture)); diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesServices.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesServices.java index 8dc98204c4..6b000dc9eb 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesServices.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesServices.java @@ -49,7 +49,7 @@ public class KubernetesServices { putLabel(service, CHE_WORKSPACE_ID_LABEL, workspaceId); putSelector(service, CHE_WORKSPACE_ID_LABEL, workspaceId); try { - return clientFactory.create().services().inNamespace(namespace).create(service); + return clientFactory.create(workspaceId).services().inNamespace(namespace).create(service); } catch (KubernetesClientException e) { throw new KubernetesInfrastructureException(e); } @@ -63,7 +63,7 @@ public class KubernetesServices { public List get() throws InfrastructureException { try { return clientFactory - .create() + .create(workspaceId) .services() .inNamespace(namespace) .withLabel(CHE_WORKSPACE_ID_LABEL, workspaceId) @@ -82,7 +82,7 @@ public class KubernetesServices { public void delete() throws InfrastructureException { try { clientFactory - .create() + .create(workspaceId) .services() .inNamespace(namespace) .withLabel(CHE_WORKSPACE_ID_LABEL, workspaceId) diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/RemoveNamespaceOnWorkspaceRemove.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/RemoveNamespaceOnWorkspaceRemove.java index d3263536c5..2ec2bdf7f1 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/RemoveNamespaceOnWorkspaceRemove.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/RemoveNamespaceOnWorkspaceRemove.java @@ -69,7 +69,7 @@ public class RemoveNamespaceOnWorkspaceRemove implements EventSubscriber machines = new HashMap<>(); InternalMachineConfig machine1 = mock(InternalMachineConfig.class); diff --git a/infrastructures/openshift/pom.xml b/infrastructures/openshift/pom.xml index b0b949f4e8..99c5b9a004 100644 --- a/infrastructures/openshift/pom.xml +++ b/infrastructures/openshift/pom.xml @@ -38,6 +38,10 @@ com.google.inject.extensions guice-multibindings + + com.squareup.okhttp3 + okhttp + io.fabric8 kubernetes-client diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftClientFactory.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftClientFactory.java index 2f5bddf105..822c7c5bd9 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftClientFactory.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftClientFactory.java @@ -11,15 +11,29 @@ package org.eclipse.che.workspace.infrastructure.openshift; import static com.google.common.base.Strings.isNullOrEmpty; +import static io.fabric8.kubernetes.client.utils.Utils.isNotNullOrEmpty; +import io.fabric8.kubernetes.client.Config; +import io.fabric8.kubernetes.client.KubernetesClientException; +import io.fabric8.kubernetes.client.utils.URLUtils; +import io.fabric8.kubernetes.client.utils.Utils; import io.fabric8.openshift.client.DefaultOpenShiftClient; import io.fabric8.openshift.client.OpenShiftClient; import io.fabric8.openshift.client.OpenShiftConfig; import io.fabric8.openshift.client.OpenShiftConfigBuilder; +import io.fabric8.openshift.client.internal.OpenShiftOAuthInterceptor; +import java.io.IOException; +import java.net.URL; import javax.annotation.PreDestroy; import javax.inject.Inject; import javax.inject.Named; import javax.inject.Singleton; +import okhttp3.Authenticator; +import okhttp3.Credentials; +import okhttp3.Interceptor; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.commons.annotation.Nullable; import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory; @@ -33,9 +47,15 @@ import org.slf4j.LoggerFactory; @Singleton public class OpenShiftClientFactory extends KubernetesClientFactory { - private static final Logger LOG = LoggerFactory.getLogger(OpenShiftClientFactory.class); + private static final String AUTHORIZATION = "Authorization"; + private static final String AUTHORIZE_PATH = + "oauth/authorize?response_type=token&client_id=openshift-challenging-client"; + private static final String LOCATION = "Location"; - private final UnclosableOpenShiftClient client; + private static final String BEFORE_TOKEN = "access_token="; + private static final String AFTER_TOKEN = "&expires"; + + private static final Logger LOG = LoggerFactory.getLogger(OpenShiftClientFactory.class); @Inject public OpenShiftClientFactory( @@ -45,6 +65,10 @@ public class OpenShiftClientFactory extends KubernetesClientFactory { @Nullable @Named("che.infra.kubernetes.oauth_token") String oauthToken, @Nullable @Named("che.infra.kubernetes.trust_certs") Boolean doTrustCerts) { super(masterUrl, username, password, oauthToken, doTrustCerts); + } + + protected Config buildDefaultConfig( + String masterUrl, String username, String password, String oauthToken, Boolean doTrustCerts) { OpenShiftConfigBuilder configBuilder = new OpenShiftConfigBuilder(); if (!isNullOrEmpty(masterUrl)) { configBuilder.withMasterUrl(masterUrl); @@ -65,22 +89,132 @@ public class OpenShiftClientFactory extends KubernetesClientFactory { if (doTrustCerts != null) { configBuilder.withTrustCerts(doTrustCerts); } - this.client = new UnclosableOpenShiftClient(configBuilder.build()); + + Config theConfig = configBuilder.build(); + return theConfig; + } + + protected Interceptor buildKubernetesInterceptor(Config config) { + final String oauthToken; + if (Utils.isNotNullOrEmpty(config.getUsername()) + && Utils.isNotNullOrEmpty(config.getPassword())) { + synchronized (getHttpClient()) { + try { + OkHttpClient.Builder builder = getHttpClient().newBuilder(); + builder.interceptors().clear(); + OkHttpClient clone = builder.build(); + + String credential = + Credentials.basic(config.getUsername(), new String(config.getPassword())); + URL url = new URL(URLUtils.join(config.getMasterUrl(), AUTHORIZE_PATH)); + Response response = + clone + .newCall( + new Request.Builder() + .get() + .url(url) + .header(AUTHORIZATION, credential) + .build()) + .execute(); + + response.body().close(); + response = response.priorResponse() != null ? response.priorResponse() : response; + response = response.networkResponse() != null ? response.networkResponse() : response; + String token = response.header(LOCATION); + if (token == null || token.isEmpty()) { + throw new KubernetesClientException( + "Unexpected response (" + + response.code() + + " " + + response.message() + + "), to the authorization request. Missing header:[" + + LOCATION + + "]!"); + } + token = token.substring(token.indexOf(BEFORE_TOKEN) + BEFORE_TOKEN.length()); + token = token.substring(0, token.indexOf(AFTER_TOKEN)); + oauthToken = token; + } catch (Exception e) { + throw KubernetesClientException.launderThrowable(e); + } + } + } else if (Utils.isNotNullOrEmpty(config.getOauthToken())) { + oauthToken = config.getOauthToken(); + } else { + oauthToken = null; + } + + return new Interceptor() { + @Override + public Response intercept(Chain chain) throws IOException { + Request request = chain.request(); + if (isNotNullOrEmpty(oauthToken)) { + Request authReq = + chain + .request() + .newBuilder() + .addHeader("Authorization", "Bearer " + oauthToken) + .build(); + return chain.proceed(authReq); + } + return chain.proceed(request); + } + }; + } + + private OpenShiftClient createOC(Config config) throws InfrastructureException { + OkHttpClient clientHttpClient = + getHttpClient().newBuilder().authenticator(Authenticator.NONE).build(); + OkHttpClient.Builder builder = clientHttpClient.newBuilder(); + builder.interceptors().clear(); + clientHttpClient = + builder + .addInterceptor( + new OpenShiftOAuthInterceptor(clientHttpClient, OpenShiftConfig.wrap(config))) + .build(); + + return new UnclosableOpenShiftClient(clientHttpClient, config); } /** - * Creates instance of {@link OpenShiftClient}. + * Creates an instance of {@link OpenShiftClient} that can be used to perform any operation + * related to a given workspace.
Important note: However, in some + * use-cases involving web sockets, the Openshift client may introduce connection leaks. That's + * why this method should only be used for API calls that are specific to Openshift and thus not + * available in the {@link KubernetesClient} class: mainly route-related calls and project-related + * calls. For all other Kubernetes standard calls, prefer the {@code create(String workspaceId)} + * method that returns a Kubernetes client. + * + * @param workspaceId Identifier of the workspace on which Openshift operations will be performed + * @throws InfrastructureException if any error occurs on client instance creation. + */ + public OpenShiftClient createOC(String workspaceId) throws InfrastructureException { + Config configForWorkspace = buildConfig(getDefaultConfig(), workspaceId); + return createOC(configForWorkspace); + } + + /** + * Creates an instance of {@link OpenShiftClient} that can be used to perform any operation + * that is not related to a given workspace.
For operations performed in + * the context of a given workspace (workspace start, workspace stop, etc ...), the {@code + * createOC(String workspaceId)} method should be used to retrieve an Openshift client.
+ * Important note: However in some use-cases involving web sockets, the + * Openshift client may introduce connection leaks. That's why this method should only be used for + * API calls that are specific to Openshift and thus not available in the {@link KubernetesClient} + * class: mainly route-related calls and project-related calls. For all other Kubernetes standard + * calls, just use the {@code create()} or {@code create(String workspaceId)} methods that return + * a Kubernetes client. * * @throws InfrastructureException if any error occurs on client instance creation. */ - public OpenShiftClient create() throws InfrastructureException { - return client; + public OpenShiftClient createOC() throws InfrastructureException { + return createOC(buildConfig(getDefaultConfig(), null)); } @PreDestroy private void cleanup() { try { - client.doClose(); + doCleanup(); } catch (RuntimeException ex) { LOG.error(ex.getMessage()); } @@ -89,15 +223,15 @@ public class OpenShiftClientFactory extends KubernetesClientFactory { /** Decorates the {@link DefaultOpenShiftClient} so that it can not be closed from the outside. */ private static class UnclosableOpenShiftClient extends DefaultOpenShiftClient { - public UnclosableOpenShiftClient(OpenShiftConfig config) { - super(config); + public UnclosableOpenShiftClient(OkHttpClient httpClient, Config config) { + super( + httpClient, + config instanceof OpenShiftConfig + ? (OpenShiftConfig) config + : new OpenShiftConfig(config)); } @Override public void close() {} - - void doClose() { - super.close(); - } } } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProject.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProject.java index e2fca4fb2e..95b2dd2fd6 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProject.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProject.java @@ -11,6 +11,7 @@ package org.eclipse.che.workspace.infrastructure.openshift.project; import com.google.common.annotations.VisibleForTesting; +import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientException; import io.fabric8.openshift.api.model.Project; import io.fabric8.openshift.api.model.Route; @@ -49,12 +50,13 @@ public class OpenShiftProject extends KubernetesNamespace { throws InfrastructureException { super(clientFactory, name, workspaceId, false); this.routes = new OpenShiftRoutes(name, workspaceId, clientFactory); - doPrepare(name, clientFactory.create()); + doPrepare(name, clientFactory.create(workspaceId), clientFactory.createOC(workspaceId)); } - private void doPrepare(String name, OpenShiftClient osClient) throws InfrastructureException { + private void doPrepare(String name, KubernetesClient kubeClient, OpenShiftClient osClient) + throws InfrastructureException { if (get(name, osClient) == null) { - create(name, osClient); + create(name, kubeClient, osClient); } } @@ -68,16 +70,17 @@ public class OpenShiftProject extends KubernetesNamespace { doRemove(routes::delete, services()::delete, pods()::delete); } - private void create(String projectName, OpenShiftClient client) throws InfrastructureException { + private void create(String projectName, KubernetesClient kubeClient, OpenShiftClient ocClient) + throws InfrastructureException { try { - client + ocClient .projectrequests() .createNew() .withNewMetadata() .withName(projectName) .endMetadata() .done(); - waitDefaultServiceAccount(projectName, client); + waitDefaultServiceAccount(projectName, kubeClient); } catch (KubernetesClientException e) { throw new KubernetesInfrastructureException(e); } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftRoutes.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftRoutes.java index c62692219b..16b8245598 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftRoutes.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftRoutes.java @@ -47,7 +47,7 @@ public class OpenShiftRoutes { public Route create(Route route) throws InfrastructureException { putLabel(route, CHE_WORKSPACE_ID_LABEL, workspaceId); try { - return clientFactory.create().routes().inNamespace(namespace).create(route); + return clientFactory.createOC(workspaceId).routes().inNamespace(namespace).create(route); } catch (KubernetesClientException e) { throw new KubernetesInfrastructureException(e); } @@ -61,7 +61,7 @@ public class OpenShiftRoutes { public List get() throws InfrastructureException { try { return clientFactory - .create() + .createOC(workspaceId) .routes() .inNamespace(namespace) .withLabel(CHE_WORKSPACE_ID_LABEL, workspaceId) @@ -80,7 +80,7 @@ public class OpenShiftRoutes { public void delete() throws InfrastructureException { try { clientFactory - .create() + .createOC(workspaceId) .routes() .inNamespace(namespace) .withLabel(CHE_WORKSPACE_ID_LABEL, workspaceId) diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/RemoveProjectOnWorkspaceRemove.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/RemoveProjectOnWorkspaceRemove.java index 0c30ee9fce..e9afcbd089 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/RemoveProjectOnWorkspaceRemove.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/RemoveProjectOnWorkspaceRemove.java @@ -70,7 +70,7 @@ public class RemoveProjectOnWorkspaceRemove implements EventSubscriber serviceAccountResource; private OpenShiftProject openShiftProject; @BeforeMethod public void setUp() throws Exception { - when(clientFactory.create()).thenReturn(openShiftClient); + when(clientFactory.create()).thenReturn(kubernetesClient); + when(clientFactory.create(anyString())).thenReturn(kubernetesClient); + when(clientFactory.createOC()).thenReturn(openShiftClient); + when(clientFactory.createOC(anyString())).thenReturn(openShiftClient); when(openShiftClient.adapt(OpenShiftClient.class)).thenReturn(openShiftClient); final MixedOperation mixedOperation = mock(MixedOperation.class); final NonNamespaceOperation namespaceOperation = mock(NonNamespaceOperation.class); - doReturn(mixedOperation).when(openShiftClient).serviceAccounts(); + doReturn(mixedOperation).when(kubernetesClient).serviceAccounts(); when(mixedOperation.inNamespace(anyString())).thenReturn(namespaceOperation); when(namespaceOperation.withName(anyString())).thenReturn(serviceAccountResource); when(serviceAccountResource.get()).thenReturn(mock(ServiceAccount.class)); diff --git a/wsmaster/che-core-api-workspace-shared/src/main/java/org/eclipse/che/api/workspace/shared/dto/RuntimeIdentityDto.java b/wsmaster/che-core-api-workspace-shared/src/main/java/org/eclipse/che/api/workspace/shared/dto/RuntimeIdentityDto.java index 4cf272a13a..3fe72af6b3 100644 --- a/wsmaster/che-core-api-workspace-shared/src/main/java/org/eclipse/che/api/workspace/shared/dto/RuntimeIdentityDto.java +++ b/wsmaster/che-core-api-workspace-shared/src/main/java/org/eclipse/che/api/workspace/shared/dto/RuntimeIdentityDto.java @@ -28,7 +28,12 @@ public interface RuntimeIdentityDto extends RuntimeIdentity { RuntimeIdentityDto withEnvName(String envName); @Override - String getOwner(); + String getOwnerName(); - RuntimeIdentityDto withOwner(String owner); + RuntimeIdentityDto withOwnerName(String ownerName); + + @Override + String getOwnerId(); + + RuntimeIdentityDto withOwnerId(String ownerId); } diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/DtoConverter.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/DtoConverter.java index a3e7f4a2a4..34beb3e77a 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/DtoConverter.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/DtoConverter.java @@ -246,7 +246,8 @@ public final class DtoConverter { return newDto(RuntimeIdentityDto.class) .withWorkspaceId(identity.getWorkspaceId()) .withEnvName(identity.getEnvName()) - .withOwner(identity.getOwner()); + .withOwnerName(identity.getOwnerName()) + .withOwnerId(identity.getOwnerId()); } /** Converts {@link Volume} to {@link VolumeDto}. */ diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimes.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimes.java index 91cbf41e84..a4c28175c8 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimes.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimes.java @@ -207,7 +207,7 @@ public class WorkspaceRuntimes { Subject subject = EnvironmentContext.getCurrent().getSubject(); RuntimeIdentity runtimeId = - new RuntimeIdentityImpl(workspaceId, envName, subject.getUserName()); + new RuntimeIdentityImpl(workspaceId, envName, subject.getUserName(), subject.getUserId()); try { InternalEnvironment internalEnv = createInternalEnvironment(environment); RuntimeContext runtimeContext = infrastructure.prepare(runtimeId, internalEnv); diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/bootstrap/AbstractBootstrapper.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/bootstrap/AbstractBootstrapper.java index 7caf9cc310..20a4f80ede 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/bootstrap/AbstractBootstrapper.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/bootstrap/AbstractBootstrapper.java @@ -57,7 +57,7 @@ public abstract class AbstractBootstrapper { RuntimeIdentityDto runtimeId = event.getRuntimeId(); if (event.getMachineName().equals(machineName) && runtimeIdentity.getEnvName().equals(runtimeId.getEnvName()) - && runtimeIdentity.getOwner().equals(runtimeId.getOwner()) + && runtimeIdentity.getOwnerName().equals(runtimeId.getOwnerName()) && runtimeIdentity.getWorkspaceId().equals(runtimeId.getWorkspaceId())) { finishEventFuture.complete(event); diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/model/impl/RuntimeIdentityImpl.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/model/impl/RuntimeIdentityImpl.java index 3e68fcd00d..59beb0567f 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/model/impl/RuntimeIdentityImpl.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/model/impl/RuntimeIdentityImpl.java @@ -17,12 +17,14 @@ public final class RuntimeIdentityImpl implements RuntimeIdentity { private final String workspaceId; private final String envName; - private final String owner; + private final String ownerName; + private final String ownerId; - public RuntimeIdentityImpl(String workspaceId, String envName, String owner) { + public RuntimeIdentityImpl(String workspaceId, String envName, String ownerName, String ownerId) { this.workspaceId = workspaceId; this.envName = envName; - this.owner = owner; + this.ownerName = ownerName; + this.ownerId = ownerId; } @Override @@ -36,8 +38,13 @@ public final class RuntimeIdentityImpl implements RuntimeIdentity { } @Override - public String getOwner() { - return owner; + public String getOwnerName() { + return ownerName; + } + + @Override + public String getOwnerId() { + return ownerId; } @Override @@ -59,7 +66,7 @@ public final class RuntimeIdentityImpl implements RuntimeIdentity { + " environment: " + envName + " owner: " - + owner + + ownerName + " }"; } } diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/spi/InternalRuntime.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/spi/InternalRuntime.java index e6fc8dbdde..5b85862ec4 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/spi/InternalRuntime.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/spi/InternalRuntime.java @@ -60,7 +60,7 @@ public abstract class InternalRuntime implements Runti @Override public String getOwner() { - return context.getIdentity().getOwner(); + return context.getIdentity().getOwnerName(); } @Override diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/spi/RuntimeStartInterruptedException.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/spi/RuntimeStartInterruptedException.java index 11100aa7bd..29fc62af27 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/spi/RuntimeStartInterruptedException.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/spi/RuntimeStartInterruptedException.java @@ -25,6 +25,6 @@ public class RuntimeStartInterruptedException extends InfrastructureException { super( format( "Runtime start for identity 'workspace: %s, environment: %s, owner: %s' is interrupted", - identity.getWorkspaceId(), identity.getEnvName(), identity.getOwner())); + identity.getWorkspaceId(), identity.getEnvName(), identity.getOwnerName())); } } diff --git a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceManagerTest.java b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceManagerTest.java index 2795034986..933c3d9c9e 100644 --- a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceManagerTest.java +++ b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceManagerTest.java @@ -528,7 +528,10 @@ public class WorkspaceManagerTest { throws Exception { RuntimeIdentity identity = new RuntimeIdentityImpl( - workspace.getId(), workspace.getConfig().getDefaultEnv(), workspace.getNamespace()); + workspace.getId(), + workspace.getConfig().getDefaultEnv(), + workspace.getNamespace(), + "id"); // doAnswer(inv -> { // final WorkspaceImpl ws = (WorkspaceImpl)inv.getArguments()[0]; // ws.setStatus(status); diff --git a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimesTest.java b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimesTest.java index 173f8845d2..962676bd59 100644 --- a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimesTest.java +++ b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimesTest.java @@ -101,7 +101,7 @@ public class WorkspaceRuntimesTest { @Test public void runtimeIsRecovered() throws Exception { - RuntimeIdentity identity = new RuntimeIdentityImpl("workspace123", "my-env", "me"); + RuntimeIdentity identity = new RuntimeIdentityImpl("workspace123", "my-env", "me", "myId"); mockWorkspace(identity); RuntimeContext context = mockContext(identity); @@ -119,7 +119,7 @@ public class WorkspaceRuntimesTest { @Test public void runtimeIsNotRecoveredIfNoWorkspaceFound() throws Exception { - RuntimeIdentity identity = new RuntimeIdentityImpl("workspace123", "my-env", "me"); + RuntimeIdentity identity = new RuntimeIdentityImpl("workspace123", "my-env", "me", "myId"); when(workspaceDao.get(identity.getWorkspaceId())).thenThrow(new NotFoundException("no!")); // try recover @@ -130,7 +130,7 @@ public class WorkspaceRuntimesTest { @Test public void runtimeIsNotRecoveredIfNoEnvironmentFound() throws Exception { - RuntimeIdentity identity = new RuntimeIdentityImpl("workspace123", "my-env", "me"); + RuntimeIdentity identity = new RuntimeIdentityImpl("workspace123", "my-env", "me", "myId"); WorkspaceImpl workspace = mockWorkspace(identity); when(workspace.getConfig().getEnvironments()).thenReturn(emptyMap()); @@ -142,7 +142,7 @@ public class WorkspaceRuntimesTest { @Test public void runtimeIsNotRecoveredIfInfraPreparationFailed() throws Exception { - RuntimeIdentity identity = new RuntimeIdentityImpl("workspace123", "my-env", "me"); + RuntimeIdentity identity = new RuntimeIdentityImpl("workspace123", "my-env", "me", "myId"); mockWorkspace(identity); InternalEnvironment internalEnvironment = mock(InternalEnvironment.class); @@ -160,7 +160,7 @@ public class WorkspaceRuntimesTest { @Test public void runtimeIsNotRecoveredIfAnotherRuntimeWithTheSameIdentityAlreadyExists() throws Exception { - RuntimeIdentity identity = new RuntimeIdentityImpl("workspace123", "my-env", "me"); + RuntimeIdentity identity = new RuntimeIdentityImpl("workspace123", "my-env", "me", "myId"); mockWorkspace(identity); RuntimeContext context = mockContext(identity); @@ -203,7 +203,8 @@ public class WorkspaceRuntimesTest { DtoFactory.newDto(RuntimeIdentityDto.class) .withWorkspaceId("workspace123") .withEnvName("my-env") - .withOwner("me"); + .withOwnerName("me") + .withOwnerId("myId"); mockWorkspace(identity); mockContext(identity); RuntimeStatusEvent event = diff --git a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/spi/InternalRuntimeTest.java b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/spi/InternalRuntimeTest.java index 3c6f1536f8..9d184ddc57 100644 --- a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/spi/InternalRuntimeTest.java +++ b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/spi/InternalRuntimeTest.java @@ -630,7 +630,7 @@ public class InternalRuntimeTest { public TestInternalRuntime(URLRewriter urlRewriter, boolean running) throws ValidationException, InfrastructureException { super( - new TestRuntimeContext(null, new RuntimeIdentityImpl("ws", "env", "owner"), null), + new TestRuntimeContext(null, new RuntimeIdentityImpl("ws", "env", "owner", "id"), null), urlRewriter, emptyList(), running); diff --git a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/spi/provision/env/EnvVarEnvironmentProvisionerTest.java b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/spi/provision/env/EnvVarEnvironmentProvisionerTest.java index 06199dc7e3..ba2c20f8ac 100644 --- a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/spi/provision/env/EnvVarEnvironmentProvisionerTest.java +++ b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/spi/provision/env/EnvVarEnvironmentProvisionerTest.java @@ -42,7 +42,7 @@ import org.testng.annotations.Test; public class EnvVarEnvironmentProvisionerTest { private static final RuntimeIdentity RUNTIME_IDENTITY = - new RuntimeIdentityImpl("testWsId", "testEnv", "testOwner"); + new RuntimeIdentityImpl("testWsId", "testEnv", "testOwner", "testOwnerId"); @Mock private EnvVarProvider provider1; @Mock private EnvVarProvider provider2;