diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodInterceptor.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodInterceptor.java index 8c4a4bca00..497642ed9c 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodInterceptor.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodInterceptor.java @@ -18,10 +18,13 @@ import static org.eclipse.che.workspace.infrastructure.kubernetes.provision.Asyn import io.fabric8.kubernetes.api.model.DoneablePod; import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.apps.DoneableDeployment; import io.fabric8.kubernetes.client.KubernetesClientException; import io.fabric8.kubernetes.client.Watch; import io.fabric8.kubernetes.client.Watcher; import io.fabric8.kubernetes.client.dsl.PodResource; +import io.fabric8.kubernetes.client.dsl.RollableScalableResource; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -39,14 +42,15 @@ import org.slf4j.LoggerFactory; /** * This interceptor checks whether the starting workspace is configured with persistent storage and - * makes sure to stop the async storage pod (if any is running) to prevent "Multi-Attach error for - * volume". After the async storage pod is stopped and deleted, the workspace start is resumed. + * makes sure to stop the async storage deployment (if any is running) to prevent "Multi-Attach + * error for volume". After the async storage deployment is stopped and deleted, the workspace start + * is resumed. */ @Singleton public class AsyncStoragePodInterceptor { private static final Logger LOG = LoggerFactory.getLogger(AsyncStoragePodInterceptor.class); - private static final int DELETE_POD_TIMEOUT_IN_MIN = 5; + private static final int DELETE_DEPLOYMENT_TIMEOUT_IN_MIN = 5; private final KubernetesClientFactory kubernetesClientFactory; private final String pvcStrategy; @@ -65,29 +69,32 @@ public class AsyncStoragePodInterceptor { return; } + removeAsyncStoragePodWithoutDeployment(identity); + String namespace = identity.getInfrastructureNamespace(); String workspaceId = identity.getWorkspaceId(); - PodResource asyncStoragePodResource = - getAsyncStoragePodResource(namespace, workspaceId); + RollableScalableResource asyncStorageDeploymentResource = + getAsyncStorageDeploymentResource(namespace, workspaceId); - if (asyncStoragePodResource.get() == null) { // pod doesn't exist + if (asyncStorageDeploymentResource.get() == null) { // deployment doesn't exist return; } try { - deleteAsyncStoragePod(asyncStoragePodResource) - .get(DELETE_POD_TIMEOUT_IN_MIN, TimeUnit.MINUTES); + deleteAsyncStorageDeployment(asyncStorageDeploymentResource) + .get(DELETE_DEPLOYMENT_TIMEOUT_IN_MIN, TimeUnit.MINUTES); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); throw new InfrastructureException( format( - "Interrupted while waiting for pod '%s' removal. " + ex.getMessage(), ASYNC_STORAGE), + "Interrupted while waiting for deployment '%s' removal. " + ex.getMessage(), + ASYNC_STORAGE), ex); } catch (ExecutionException ex) { throw new InfrastructureException( format( - "Error occurred while waiting for pod '%s' removal. " + ex.getMessage(), + "Error occurred while waiting for deployment '%s' removal. " + ex.getMessage(), ASYNC_STORAGE), ex); } catch (TimeoutException ex) { @@ -96,15 +103,72 @@ public class AsyncStoragePodInterceptor { } } - private PodResource getAsyncStoragePodResource( - String namespace, String workspaceId) throws InfrastructureException { + private RollableScalableResource + getAsyncStorageDeploymentResource(String namespace, String workspaceId) + throws InfrastructureException { return kubernetesClientFactory .create(workspaceId) - .pods() + .apps() + .deployments() .inNamespace(namespace) .withName(ASYNC_STORAGE); } + private CompletableFuture deleteAsyncStorageDeployment( + RollableScalableResource resource) + throws InfrastructureException { + Watch toCloseOnException = null; + try { + final CompletableFuture deleteFuture = new CompletableFuture<>(); + final Watch watch = resource.watch(new DeleteWatcher<>(deleteFuture)); + toCloseOnException = watch; + + Boolean deleteSucceeded = resource.withPropagationPolicy("Background").delete(); + if (deleteSucceeded == null || !deleteSucceeded) { + deleteFuture.complete(null); + } + return deleteFuture.whenComplete( + (v, e) -> { + if (e != null) { + LOG.warn("Failed to remove deployment {} cause {}", ASYNC_STORAGE, e.getMessage()); + } + watch.close(); + }); + } catch (KubernetesClientException e) { + if (toCloseOnException != null) { + toCloseOnException.close(); + } + throw new KubernetesInfrastructureException(e); + } catch (Exception e) { + if (toCloseOnException != null) { + toCloseOnException.close(); + } + throw e; + } + } + + /** + * Cleanup existed Async Storage pods running without Deployment see + * https://github.com/eclipse/che/issues/17616. Method can be removed in 7.20.x + */ + private void removeAsyncStoragePodWithoutDeployment(RuntimeIdentity identity) + throws InfrastructureException { + String namespace = identity.getInfrastructureNamespace(); + String workspaceId = identity.getWorkspaceId(); + + PodResource asyncStoragePodResource = + kubernetesClientFactory + .create(workspaceId) + .pods() + .inNamespace(namespace) + .withName(ASYNC_STORAGE); + + if (asyncStoragePodResource.get() + != null) { // remove existed pod to replace it with deployment on provision step + deleteAsyncStoragePod(asyncStoragePodResource); + } + } + private CompletableFuture deleteAsyncStoragePod(PodResource podResource) throws InfrastructureException { Watch toCloseOnException = null; diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodWatcher.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodWatcher.java index 260c2ce438..c081de90c9 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodWatcher.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodWatcher.java @@ -25,7 +25,10 @@ import static org.eclipse.che.workspace.infrastructure.kubernetes.provision.Asyn import io.fabric8.kubernetes.api.model.DoneablePod; import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.apps.DoneableDeployment; import io.fabric8.kubernetes.client.dsl.PodResource; +import io.fabric8.kubernetes.client.dsl.RollableScalableResource; import java.time.Instant; import java.util.Map; import javax.inject.Inject; @@ -137,14 +140,16 @@ public class AsyncStoragePodWatcher { Instant expectedShutdownAfter = ofEpochSecond(lastTimeAccessSec).plusSeconds(shutdownTimeoutSec); if (now().isAfter(expectedShutdownAfter)) { - PodResource doneablePodResource = + removeAsyncStoragePodWithoutDeployment(namespace); + RollableScalableResource doneableResource = kubernetesClientFactory .create() - .pods() + .apps() + .deployments() .inNamespace(namespace) .withName(ASYNC_STORAGE); - if (doneablePodResource.get() != null) { - doneablePodResource.delete(); + if (doneableResource.get() != null) { + doneableResource.delete(); } } } catch (InfrastructureException | ServerException e) { @@ -153,4 +158,20 @@ public class AsyncStoragePodWatcher { } } } + + /** + * Cleanup existed Async Storage pods running without Deployment see + * https://github.com/eclipse/che/issues/17616. Method can be removed in 7.20.x + * + * @param namespace + * @throws InfrastructureException + */ + private void removeAsyncStoragePodWithoutDeployment(String namespace) + throws InfrastructureException { + PodResource doneablePodResource = + kubernetesClientFactory.create().pods().inNamespace(namespace).withName(ASYNC_STORAGE); + if (doneablePodResource.get() != null) { + doneablePodResource.delete(); + } + } } diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStorageProvisioner.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStorageProvisioner.java index 4a9b72af7d..5c983c265f 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStorageProvisioner.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStorageProvisioner.java @@ -17,6 +17,7 @@ import static java.lang.String.format; import static java.lang.String.valueOf; import static java.util.Collections.singletonList; import static org.eclipse.che.api.workspace.shared.Constants.ASYNC_PERSIST_ATTRIBUTE; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_DEPLOYMENT_NAME_LABEL; import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_USER_ID_LABEL; import static org.eclipse.che.workspace.infrastructure.kubernetes.Warnings.NOT_ABLE_TO_PROVISION_SSH_KEYS; import static org.eclipse.che.workspace.infrastructure.kubernetes.Warnings.NOT_ABLE_TO_PROVISION_SSH_KEYS_MESSAGE; @@ -31,15 +32,13 @@ import io.fabric8.kubernetes.api.model.ContainerBuilder; import io.fabric8.kubernetes.api.model.ContainerPortBuilder; import io.fabric8.kubernetes.api.model.DoneableConfigMap; import io.fabric8.kubernetes.api.model.DoneablePersistentVolumeClaim; -import io.fabric8.kubernetes.api.model.DoneablePod; import io.fabric8.kubernetes.api.model.DoneableService; import io.fabric8.kubernetes.api.model.IntOrString; import io.fabric8.kubernetes.api.model.IntOrStringBuilder; import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; import io.fabric8.kubernetes.api.model.PersistentVolumeClaim; import io.fabric8.kubernetes.api.model.PersistentVolumeClaimVolumeSourceBuilder; -import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.api.model.PodBuilder; import io.fabric8.kubernetes.api.model.PodSpec; import io.fabric8.kubernetes.api.model.PodSpecBuilder; import io.fabric8.kubernetes.api.model.Quantity; @@ -51,9 +50,12 @@ import io.fabric8.kubernetes.api.model.Volume; import io.fabric8.kubernetes.api.model.VolumeBuilder; import io.fabric8.kubernetes.api.model.VolumeMount; import io.fabric8.kubernetes.api.model.VolumeMountBuilder; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder; +import io.fabric8.kubernetes.api.model.apps.DoneableDeployment; import io.fabric8.kubernetes.client.KubernetesClient; -import io.fabric8.kubernetes.client.dsl.PodResource; import io.fabric8.kubernetes.client.dsl.Resource; +import io.fabric8.kubernetes.client.dsl.RollableScalableResource; import io.fabric8.kubernetes.client.dsl.ServiceResource; import java.util.List; import java.util.Map; @@ -269,10 +271,11 @@ public class AsyncStorageProvisioner { */ private void createAsyncStoragePodIfNotExist( KubernetesClient k8sClient, String namespace, String configMap, String userId) { - PodResource podResource = - k8sClient.pods().inNamespace(namespace).withName(ASYNC_STORAGE); - if (podResource.get() != null) { - return; // pod already exist + + RollableScalableResource resource = + k8sClient.apps().deployments().inNamespace(namespace).withName(ASYNC_STORAGE); + if (resource.get() != null) { + return; // deployment already exist } String containerName = Names.generateName(ASYNC_STORAGE); @@ -333,19 +336,37 @@ public class AsyncStorageProvisioner { PodSpec podSpec = podSpecBuilder.withContainers(container).withVolumes(storageVolume, sshKeyVolume).build(); - Pod pod = - new PodBuilder() - .withApiVersion("v1") - .withKind("Pod") - .withNewMetadata() - .withName(ASYNC_STORAGE) + ObjectMetaBuilder metaBuilder = new ObjectMetaBuilder(); + ObjectMeta meta = + metaBuilder + .withLabels( + of( + "app", + "che", + CHE_USER_ID_LABEL, + userId, + CHE_DEPLOYMENT_NAME_LABEL, + ASYNC_STORAGE)) .withNamespace(namespace) - .withLabels(of("app", ASYNC_STORAGE, CHE_USER_ID_LABEL, userId)) - .endMetadata() - .withSpec(podSpec) + .withName(ASYNC_STORAGE) .build(); - k8sClient.pods().inNamespace(namespace).create(pod); + Deployment deployment = + new DeploymentBuilder() + .withMetadata(meta) + .withNewSpec() + .withNewSelector() + .withMatchLabels(meta.getLabels()) + .endSelector() + .withReplicas(1) + .withNewTemplate() + .withMetadata(meta) + .withSpec(podSpec) + .endTemplate() + .endSpec() + .build(); + + k8sClient.apps().deployments().inNamespace(namespace).create(deployment); } /** Create service for serving rsync connection */ diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodInterceptorTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodInterceptorTest.java index ba611acb0f..002b829d06 100644 --- a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodInterceptorTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodInterceptorTest.java @@ -28,13 +28,17 @@ import io.fabric8.kubernetes.api.model.DoneablePod; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodList; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.apps.DoneableDeployment; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.Watch; import io.fabric8.kubernetes.client.Watcher; +import io.fabric8.kubernetes.client.dsl.AppsAPIGroupDSL; import io.fabric8.kubernetes.client.dsl.FilterWatchListDeletable; import io.fabric8.kubernetes.client.dsl.MixedOperation; import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation; import io.fabric8.kubernetes.client.dsl.PodResource; +import io.fabric8.kubernetes.client.dsl.RollableScalableResource; import java.util.UUID; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; @@ -56,10 +60,14 @@ public class AsyncStoragePodInterceptorTest { @Mock private RuntimeIdentity identity; @Mock private KubernetesClientFactory clientFactory; @Mock private KubernetesClient kubernetesClient; + @Mock private RollableScalableResource deploymentResource; @Mock private PodResource podResource; + @Mock private MixedOperation mixedOperation; @Mock private MixedOperation mixedOperationPod; + @Mock private NonNamespaceOperation namespaceOperation; @Mock private NonNamespaceOperation namespacePodOperation; @Mock private FilterWatchListDeletable> deletable; + @Mock private AppsAPIGroupDSL apps; private AsyncStoragePodInterceptor asyncStoragePodInterceptor; @@ -70,6 +78,11 @@ public class AsyncStoragePodInterceptorTest { @Test public void shouldDoNothingIfNotCommonStrategy() throws Exception { + when(kubernetesClient.pods()).thenReturn(mixedOperationPod); + when(mixedOperationPod.inNamespace(NAMESPACE)).thenReturn(namespacePodOperation); + when(namespacePodOperation.withName(ASYNC_STORAGE)).thenReturn(podResource); + when(podResource.get()).thenReturn(null); + AsyncStoragePodInterceptor asyncStoragePodInterceptor = new AsyncStoragePodInterceptor(randomUUID().toString(), clientFactory); asyncStoragePodInterceptor.intercept(kubernetesEnvironment, identity); @@ -107,6 +120,12 @@ public class AsyncStoragePodInterceptorTest { when(namespacePodOperation.withName(ASYNC_STORAGE)).thenReturn(podResource); when(podResource.get()).thenReturn(null); + when(kubernetesClient.apps()).thenReturn(apps); + when(apps.deployments()).thenReturn(mixedOperation); + when(mixedOperation.inNamespace(NAMESPACE)).thenReturn(namespaceOperation); + when(namespaceOperation.withName(ASYNC_STORAGE)).thenReturn(deploymentResource); + when(deploymentResource.get()).thenReturn(null); + asyncStoragePodInterceptor.intercept(kubernetesEnvironment, identity); verifyNoMoreInteractions(clientFactory); verifyNoMoreInteractions(identity); @@ -124,17 +143,23 @@ public class AsyncStoragePodInterceptorTest { when(kubernetesClient.pods()).thenReturn(mixedOperationPod); when(mixedOperationPod.inNamespace(NAMESPACE)).thenReturn(namespacePodOperation); when(namespacePodOperation.withName(ASYNC_STORAGE)).thenReturn(podResource); + when(podResource.get()).thenReturn(null); + + when(kubernetesClient.apps()).thenReturn(apps); + when(apps.deployments()).thenReturn(mixedOperation); + when(mixedOperation.inNamespace(NAMESPACE)).thenReturn(namespaceOperation); + when(namespaceOperation.withName(ASYNC_STORAGE)).thenReturn(deploymentResource); ObjectMeta meta = new ObjectMeta(); meta.setName(ASYNC_STORAGE); - Pod pod = new Pod(); - pod.setMetadata(meta); + Deployment deployment = new Deployment(); + deployment.setMetadata(meta); - when(podResource.get()).thenReturn(pod); - when(podResource.withPropagationPolicy("Background")).thenReturn(deletable); + when(deploymentResource.get()).thenReturn(deployment); + when(deploymentResource.withPropagationPolicy("Background")).thenReturn(deletable); Watch watch = mock(Watch.class); - when(podResource.watch(any())).thenReturn(watch); + when(deploymentResource.watch(any())).thenReturn(watch); asyncStoragePodInterceptor.intercept(kubernetesEnvironment, identity); verify(deletable).delete(); @@ -154,17 +179,23 @@ public class AsyncStoragePodInterceptorTest { when(kubernetesClient.pods()).thenReturn(mixedOperationPod); when(mixedOperationPod.inNamespace(NAMESPACE)).thenReturn(namespacePodOperation); when(namespacePodOperation.withName(ASYNC_STORAGE)).thenReturn(podResource); + when(podResource.get()).thenReturn(null); + + when(kubernetesClient.apps()).thenReturn(apps); + when(apps.deployments()).thenReturn(mixedOperation); + when(mixedOperation.inNamespace(NAMESPACE)).thenReturn(namespaceOperation); + when(namespaceOperation.withName(ASYNC_STORAGE)).thenReturn(deploymentResource); ObjectMeta meta = new ObjectMeta(); meta.setName(ASYNC_STORAGE); - Pod pod = new Pod(); - pod.setMetadata(meta); + Deployment deployment = new Deployment(); + deployment.setMetadata(meta); - when(podResource.get()).thenReturn(pod); - when(podResource.withPropagationPolicy("Background")).thenReturn(deletable); + when(deploymentResource.get()).thenReturn(deployment); + when(deploymentResource.withPropagationPolicy("Background")).thenReturn(deletable); Watch watch = mock(Watch.class); - when(podResource.watch(any())).thenReturn(watch); + when(deploymentResource.watch(any())).thenReturn(watch); asyncStoragePodInterceptor.intercept(kubernetesEnvironment, identity); verify(deletable).delete(); diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodWatcherTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodWatcherTest.java index 3c90e84fb1..1b3b377b1c 100644 --- a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodWatcherTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodWatcherTest.java @@ -28,10 +28,14 @@ import static org.mockito.Mockito.when; import io.fabric8.kubernetes.api.model.DoneablePod; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.apps.DoneableDeployment; import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.dsl.AppsAPIGroupDSL; import io.fabric8.kubernetes.client.dsl.MixedOperation; import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation; import io.fabric8.kubernetes.client.dsl.PodResource; +import io.fabric8.kubernetes.client.dsl.RollableScalableResource; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -63,10 +67,14 @@ public class AsyncStoragePodWatcherTest { @Mock private PreferenceManager preferenceManager; @Mock private WorkspaceRuntimes runtimes; @Mock private KubernetesClient kubernetesClient; + @Mock private RollableScalableResource deploymentResource; + @Mock private MixedOperation mixedOperation; + @Mock private NonNamespaceOperation namespaceOperation; @Mock private PodResource podResource; @Mock private MixedOperation mixedOperationPod; @Mock private NonNamespaceOperation namespacePodOperation; @Mock private UserImpl user; + @Mock private AppsAPIGroupDSL apps; @BeforeMethod public void setUp() throws Exception { @@ -82,13 +90,19 @@ public class AsyncStoragePodWatcherTest { when(userManager.getAll(anyInt(), anyLong())).thenReturn(userPage); when(kubernetesClientFactory.create()).thenReturn(kubernetesClient); + when(kubernetesClient.apps()).thenReturn(apps); + when(apps.deployments()).thenReturn(mixedOperation); + when(mixedOperation.inNamespace(NAMESPACE)).thenReturn(namespaceOperation); + when(namespaceOperation.withName(ASYNC_STORAGE)).thenReturn(deploymentResource); + when(kubernetesClient.pods()).thenReturn(mixedOperationPod); when(mixedOperationPod.inNamespace(NAMESPACE)).thenReturn(namespacePodOperation); when(namespacePodOperation.withName(ASYNC_STORAGE)).thenReturn(podResource); + when(podResource.get()).thenReturn(null); } @Test - public void shouldDeleteAsyncStoragePod() throws Exception { + public void shouldDeleteAsyncStorageDeployment() throws Exception { AsyncStoragePodWatcher watcher = new AsyncStoragePodWatcher( kubernetesClientFactory, @@ -105,14 +119,14 @@ public class AsyncStoragePodWatcherTest { ObjectMeta meta = new ObjectMeta(); meta.setName(ASYNC_STORAGE); - Pod pod = new Pod(); - pod.setMetadata(meta); - when(podResource.get()).thenReturn(pod); + Deployment deployment = new Deployment(); + deployment.setMetadata(meta); + when(deploymentResource.get()).thenReturn(deployment); watcher.check(); verify(preferenceManager).find(USER_ID); - verify(podResource).delete(); + verify(deploymentResource).delete(); } @Test @@ -135,7 +149,7 @@ public class AsyncStoragePodWatcherTest { verify(preferenceManager).find(USER_ID); verifyNoMoreInteractions(kubernetesClientFactory); - verifyNoMoreInteractions(podResource); + verifyNoMoreInteractions(deploymentResource); } @Test @@ -165,7 +179,7 @@ public class AsyncStoragePodWatcherTest { verify(preferenceManager).find(USER_ID); verifyNoMoreInteractions(kubernetesClientFactory); - verifyNoMoreInteractions(podResource); + verifyNoMoreInteractions(deploymentResource); } @Test @@ -187,7 +201,7 @@ public class AsyncStoragePodWatcherTest { verify(preferenceManager).find(USER_ID); verifyNoMoreInteractions(kubernetesClientFactory); - verifyNoMoreInteractions(podResource); + verifyNoMoreInteractions(deploymentResource); } @Test @@ -209,7 +223,7 @@ public class AsyncStoragePodWatcherTest { verifyNoMoreInteractions(preferenceManager); verifyNoMoreInteractions(kubernetesClientFactory); - verifyNoMoreInteractions(podResource); + verifyNoMoreInteractions(deploymentResource); } @Test @@ -231,7 +245,7 @@ public class AsyncStoragePodWatcherTest { verifyNoMoreInteractions(preferenceManager); verifyNoMoreInteractions(kubernetesClientFactory); - verifyNoMoreInteractions(podResource); + verifyNoMoreInteractions(deploymentResource); } @Test @@ -253,7 +267,7 @@ public class AsyncStoragePodWatcherTest { verifyNoMoreInteractions(preferenceManager); verifyNoMoreInteractions(kubernetesClientFactory); - verifyNoMoreInteractions(podResource); + verifyNoMoreInteractions(deploymentResource); } @Test @@ -275,7 +289,7 @@ public class AsyncStoragePodWatcherTest { verifyNoMoreInteractions(preferenceManager); verifyNoMoreInteractions(kubernetesClientFactory); - verifyNoMoreInteractions(podResource); + verifyNoMoreInteractions(deploymentResource); } @Test @@ -297,6 +311,6 @@ public class AsyncStoragePodWatcherTest { verifyNoMoreInteractions(preferenceManager); verifyNoMoreInteractions(kubernetesClientFactory); - verifyNoMoreInteractions(podResource); + verifyNoMoreInteractions(deploymentResource); } } diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStorageProvisionerTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStorageProvisionerTest.java index bc31df4d2d..e5dfbe8cc9 100644 --- a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStorageProvisionerTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStorageProvisionerTest.java @@ -37,11 +37,16 @@ import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.PersistentVolumeClaim; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.apps.DoneableDeployment; import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.Watcher; +import io.fabric8.kubernetes.client.dsl.AppsAPIGroupDSL; import io.fabric8.kubernetes.client.dsl.MixedOperation; import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation; import io.fabric8.kubernetes.client.dsl.PodResource; import io.fabric8.kubernetes.client.dsl.Resource; +import io.fabric8.kubernetes.client.dsl.RollableScalableResource; import io.fabric8.kubernetes.client.dsl.ServiceResource; import java.util.HashMap; import java.util.Map; @@ -54,6 +59,8 @@ import org.eclipse.che.api.ssh.server.model.impl.SshPairImpl; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory; import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.Mock; import org.mockito.testng.MockitoTestNGListener; import org.testng.annotations.BeforeMethod; @@ -77,15 +84,20 @@ public class AsyncStorageProvisionerTest { @Mock private Resource pvcResource; @Mock private Resource mapResource; @Mock private PodResource podResource; + @Mock private RollableScalableResource deploymentResource; @Mock private ServiceResource serviceResource; @Mock private MixedOperation mixedOperationPvc; @Mock private MixedOperation mixedOperationConfigMap; @Mock private MixedOperation mixedOperationPod; + @Mock private MixedOperation mixedOperationDeployment; @Mock private MixedOperation mixedOperationService; @Mock private NonNamespaceOperation namespacePvcOperation; @Mock private NonNamespaceOperation namespaceConfigMapOperation; @Mock private NonNamespaceOperation namespacePodOperation; + @Mock private NonNamespaceOperation namespaceDeploymentOperation; @Mock private NonNamespaceOperation namespaceServiceOperation; + @Mock private AppsAPIGroupDSL apps; + @Captor private ArgumentCaptor> watcherCaptor; private Map attributes; private AsyncStorageProvisioner asyncStorageProvisioner; @@ -180,10 +192,11 @@ public class AsyncStorageProvisionerTest { when(namespaceConfigMapOperation.withName(anyString())).thenReturn(mapResource); when(mapResource.get()).thenReturn(null); - when(kubernetesClient.pods()).thenReturn(mixedOperationPod); - when(mixedOperationPod.inNamespace(NAMESPACE)).thenReturn(namespacePodOperation); - when(namespacePodOperation.withName(ASYNC_STORAGE)).thenReturn(podResource); - when(podResource.get()).thenReturn(null); + when(kubernetesClient.apps()).thenReturn(apps); + when(apps.deployments()).thenReturn(mixedOperationDeployment); + when(mixedOperationDeployment.inNamespace(NAMESPACE)).thenReturn(namespaceDeploymentOperation); + when(namespaceDeploymentOperation.withName(ASYNC_STORAGE)).thenReturn(deploymentResource); + when(deploymentResource.get()).thenReturn(null); when(kubernetesClient.services()).thenReturn(mixedOperationService); when(mixedOperationService.inNamespace(NAMESPACE)).thenReturn(namespaceServiceOperation); @@ -191,6 +204,7 @@ public class AsyncStorageProvisionerTest { when(serviceResource.get()).thenReturn(null); asyncStorageProvisioner.provision(kubernetesEnvironment, identity); + verify(identity, times(1)).getInfrastructureNamespace(); verify(identity, times(1)).getOwnerId(); verify(sshManager, times(1)).getPairs(USER, "internal"); @@ -198,7 +212,8 @@ public class AsyncStorageProvisionerTest { verify(kubernetesClient.services().inNamespace(NAMESPACE), times(1)).create(any(Service.class)); verify(kubernetesClient.configMaps().inNamespace(NAMESPACE), times(1)) .create(any(ConfigMap.class)); - verify(kubernetesClient.pods().inNamespace(NAMESPACE), times(1)).create(any(Pod.class)); + verify(kubernetesClient.apps().deployments().inNamespace(NAMESPACE), times(1)) + .create(any(Deployment.class)); verify(kubernetesClient.persistentVolumeClaims().inNamespace(NAMESPACE), times(1)) .create(any(PersistentVolumeClaim.class)); } @@ -231,6 +246,12 @@ public class AsyncStorageProvisionerTest { when(namespacePodOperation.withName(ASYNC_STORAGE)).thenReturn(podResource); when(podResource.get()).thenReturn(null); + when(kubernetesClient.apps()).thenReturn(apps); + when(apps.deployments()).thenReturn(mixedOperationDeployment); + when(mixedOperationDeployment.inNamespace(NAMESPACE)).thenReturn(namespaceDeploymentOperation); + when(namespaceDeploymentOperation.withName(ASYNC_STORAGE)).thenReturn(deploymentResource); + when(deploymentResource.get()).thenReturn(null); + when(kubernetesClient.services()).thenReturn(mixedOperationService); when(mixedOperationService.inNamespace(NAMESPACE)).thenReturn(namespaceServiceOperation); when(namespaceServiceOperation.withName(ASYNC_STORAGE)).thenReturn(serviceResource); @@ -245,7 +266,8 @@ public class AsyncStorageProvisionerTest { verify(kubernetesClient.services().inNamespace(NAMESPACE), times(1)).create(any(Service.class)); verify(kubernetesClient.configMaps().inNamespace(NAMESPACE), never()) .create(any(ConfigMap.class)); - verify(kubernetesClient.pods().inNamespace(NAMESPACE), times(1)).create(any(Pod.class)); + verify(kubernetesClient.apps().deployments().inNamespace(NAMESPACE), times(1)) + .create(any(Deployment.class)); verify(kubernetesClient.persistentVolumeClaims().inNamespace(NAMESPACE), times(1)) .create(any(PersistentVolumeClaim.class)); } @@ -273,11 +295,16 @@ public class AsyncStorageProvisionerTest { when(kubernetesClient.pods()).thenReturn(mixedOperationPod); when(mixedOperationPod.inNamespace(NAMESPACE)).thenReturn(namespacePodOperation); when(namespacePodOperation.withName(ASYNC_STORAGE)).thenReturn(podResource); + + when(kubernetesClient.apps()).thenReturn(apps); + when(apps.deployments()).thenReturn(mixedOperationDeployment); + when(mixedOperationDeployment.inNamespace(NAMESPACE)).thenReturn(namespaceDeploymentOperation); + when(namespaceDeploymentOperation.withName(ASYNC_STORAGE)).thenReturn(deploymentResource); ObjectMeta meta = new ObjectMeta(); meta.setName(ASYNC_STORAGE); - Pod pod = new Pod(); - pod.setMetadata(meta); - when(podResource.get()).thenReturn(pod); + Deployment deployment = new Deployment(); + deployment.setMetadata(meta); + when(deploymentResource.get()).thenReturn(deployment); when(kubernetesClient.services()).thenReturn(mixedOperationService); when(mixedOperationService.inNamespace(NAMESPACE)).thenReturn(namespaceServiceOperation); @@ -322,6 +349,12 @@ public class AsyncStorageProvisionerTest { when(namespacePodOperation.withName(ASYNC_STORAGE)).thenReturn(podResource); when(podResource.get()).thenReturn(null); + when(kubernetesClient.apps()).thenReturn(apps); + when(apps.deployments()).thenReturn(mixedOperationDeployment); + when(mixedOperationDeployment.inNamespace(NAMESPACE)).thenReturn(namespaceDeploymentOperation); + when(namespaceDeploymentOperation.withName(ASYNC_STORAGE)).thenReturn(deploymentResource); + when(deploymentResource.get()).thenReturn(null); + when(kubernetesClient.services()).thenReturn(mixedOperationService); when(mixedOperationService.inNamespace(NAMESPACE)).thenReturn(namespaceServiceOperation); when(namespaceServiceOperation.withName(ASYNC_STORAGE)).thenReturn(serviceResource); @@ -339,7 +372,8 @@ public class AsyncStorageProvisionerTest { verify(kubernetesClient.services().inNamespace(NAMESPACE), never()).create(any(Service.class)); verify(kubernetesClient.configMaps().inNamespace(NAMESPACE), times(1)) .create(any(ConfigMap.class)); - verify(kubernetesClient.pods().inNamespace(NAMESPACE), times(1)).create(any(Pod.class)); + verify(kubernetesClient.apps().deployments().inNamespace(NAMESPACE), times(1)) + .create(any(Deployment.class)); verify(kubernetesClient.persistentVolumeClaims().inNamespace(NAMESPACE), times(1)) .create(any(PersistentVolumeClaim.class)); }