From fa72f4480665aa32f4f06f4ce9806b4ee7ba2eb3 Mon Sep 17 00:00:00 2001 From: Sergii Leshchenko Date: Tue, 3 Jul 2018 14:31:26 +0300 Subject: [PATCH] Add cascade removing of Kubernetes runtime state when workspace is removed (#10244) --- .../kubernetes/KubernetesInfraModule.java | 8 +- ...oreKubernetesRuntimeStateRemovedEvent.java | 32 ++++++++ .../cache/jpa/JpaKubernetesMachineCache.java | 23 ++++++ .../jpa/JpaKubernetesRuntimeCacheModule.java | 28 +++++++ .../jpa/JpaKubernetesRuntimeStateCache.java | 78 ++++++++++++++++++- .../model/KubernetesRuntimeState.java | 20 +++-- .../openshift/OpenShiftInfraModule.java | 8 +- .../integration-tests/cascade-removal/pom.xml | 10 +++ .../che/core/db/jpa/CascadeRemovalTest.java | 34 +++++++- .../che/core/db/jpa/TestObjectsFactory.java | 31 ++++++++ .../src/test/java/PostgreSqlTckModule.java | 4 + 11 files changed, 250 insertions(+), 26 deletions(-) create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/cache/BeforeKubernetesRuntimeStateRemovedEvent.java create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/cache/jpa/JpaKubernetesRuntimeCacheModule.java diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java index b7cae46f35..4c81b9e20f 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java @@ -32,10 +32,7 @@ import org.eclipse.che.api.workspace.server.wsnext.WorkspaceNextApplier; import org.eclipse.che.workspace.infrastructure.docker.environment.dockerimage.DockerImageEnvironment; import org.eclipse.che.workspace.infrastructure.docker.environment.dockerimage.DockerImageEnvironmentFactory; import org.eclipse.che.workspace.infrastructure.kubernetes.bootstrapper.KubernetesBootstrapperFactory; -import org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesMachineCache; -import org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesRuntimeStateCache; -import org.eclipse.che.workspace.infrastructure.kubernetes.cache.jpa.JpaKubernetesMachineCache; -import org.eclipse.che.workspace.infrastructure.kubernetes.cache.jpa.JpaKubernetesRuntimeStateCache; +import org.eclipse.che.workspace.infrastructure.kubernetes.cache.jpa.JpaKubernetesRuntimeCacheModule; import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironmentFactory; import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.RemoveNamespaceOnWorkspaceRemove; @@ -117,8 +114,7 @@ public class KubernetesInfraModule extends AbstractModule { .annotatedWith(com.google.inject.name.Names.named("infra.kubernetes.ingress.annotations")) .toProvider(IngressAnnotationsProvider.class); - bind(KubernetesRuntimeStateCache.class).to(JpaKubernetesRuntimeStateCache.class); - bind(KubernetesMachineCache.class).to(JpaKubernetesMachineCache.class); + install(new JpaKubernetesRuntimeCacheModule()); MapBinder wsNext = MapBinder.newMapBinder(binder(), String.class, WorkspaceNextApplier.class); diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/cache/BeforeKubernetesRuntimeStateRemovedEvent.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/cache/BeforeKubernetesRuntimeStateRemovedEvent.java new file mode 100644 index 0000000000..c527f193b0 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/cache/BeforeKubernetesRuntimeStateRemovedEvent.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.cache; + +import org.eclipse.che.core.db.cascade.event.RemoveEvent; +import org.eclipse.che.workspace.infrastructure.kubernetes.model.KubernetesRuntimeState; + +/** + * Published before {@link KubernetesRuntimeState kubernetes runtime state} removed. + * + * @author Sergii Leshchenko + */ +public class BeforeKubernetesRuntimeStateRemovedEvent extends RemoveEvent { + + private final KubernetesRuntimeState k8sRuntimeState; + + public BeforeKubernetesRuntimeStateRemovedEvent(KubernetesRuntimeState k8sRuntimeState) { + this.k8sRuntimeState = k8sRuntimeState; + } + + public KubernetesRuntimeState getRuntimeState() { + return k8sRuntimeState; + } +} diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/cache/jpa/JpaKubernetesMachineCache.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/cache/jpa/JpaKubernetesMachineCache.java index 4f28328523..645a73ad8e 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/cache/jpa/JpaKubernetesMachineCache.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/cache/jpa/JpaKubernetesMachineCache.java @@ -17,14 +17,19 @@ import com.google.inject.persist.Transactional; import java.util.Collection; import java.util.Map; import java.util.function.Function; +import javax.annotation.PostConstruct; import javax.inject.Inject; import javax.inject.Provider; +import javax.inject.Singleton; import javax.persistence.EntityManager; import org.eclipse.che.api.core.model.workspace.runtime.MachineStatus; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.api.core.model.workspace.runtime.ServerStatus; +import org.eclipse.che.api.core.notification.EventService; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.core.db.cascade.CascadeEventSubscriber; import org.eclipse.che.core.db.jpa.DuplicateKeyException; +import org.eclipse.che.workspace.infrastructure.kubernetes.cache.BeforeKubernetesRuntimeStateRemovedEvent; import org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesMachineCache; import org.eclipse.che.workspace.infrastructure.kubernetes.model.KubernetesMachineImpl; import org.eclipse.che.workspace.infrastructure.kubernetes.model.KubernetesMachineImpl.MachineId; @@ -182,4 +187,22 @@ public class JpaKubernetesMachineCache implements KubernetesMachineCache { } return false; } + + @Singleton + public static class RemoveKubernetesMachinesBeforeRuntimesRemoved + extends CascadeEventSubscriber { + + @Inject private EventService eventService; + @Inject private JpaKubernetesMachineCache k8sMachines; + + @PostConstruct + public void subscribe() { + eventService.subscribe(this, BeforeKubernetesRuntimeStateRemovedEvent.class); + } + + @Override + public void onCascadeEvent(BeforeKubernetesRuntimeStateRemovedEvent event) throws Exception { + k8sMachines.remove(event.getRuntimeState().getRuntimeId()); + } + } } diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/cache/jpa/JpaKubernetesRuntimeCacheModule.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/cache/jpa/JpaKubernetesRuntimeCacheModule.java new file mode 100644 index 0000000000..093dbc58ba --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/cache/jpa/JpaKubernetesRuntimeCacheModule.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.cache.jpa; + +import com.google.inject.AbstractModule; +import org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesMachineCache; +import org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesRuntimeStateCache; + +/** @author Sergii Leshchenko */ +public class JpaKubernetesRuntimeCacheModule extends AbstractModule { + @Override + protected void configure() { + bind(KubernetesRuntimeStateCache.class).to(JpaKubernetesRuntimeStateCache.class); + bind(KubernetesMachineCache.class).to(JpaKubernetesMachineCache.class); + bind(JpaKubernetesRuntimeStateCache.RemoveKubernetesRuntimeBeforeWorkspaceRemoved.class) + .asEagerSingleton(); + bind(JpaKubernetesMachineCache.RemoveKubernetesMachinesBeforeRuntimesRemoved.class) + .asEagerSingleton(); + } +} diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/cache/jpa/JpaKubernetesRuntimeStateCache.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/cache/jpa/JpaKubernetesRuntimeStateCache.java index d04af3c7d4..e48042797d 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/cache/jpa/JpaKubernetesRuntimeStateCache.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/cache/jpa/JpaKubernetesRuntimeStateCache.java @@ -10,22 +10,34 @@ */ package org.eclipse.che.workspace.infrastructure.kubernetes.cache.jpa; +import static java.lang.String.format; + import com.google.inject.persist.Transactional; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; +import javax.annotation.PostConstruct; import javax.inject.Inject; import javax.inject.Provider; +import javax.inject.Singleton; import javax.persistence.EntityExistsException; import javax.persistence.EntityManager; +import javax.persistence.NoResultException; +import org.eclipse.che.api.core.ServerException; import org.eclipse.che.api.core.model.workspace.WorkspaceStatus; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.api.core.notification.EventService; +import org.eclipse.che.api.workspace.server.event.BeforeWorkspaceRemovedEvent; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.core.db.cascade.CascadeEventSubscriber; import org.eclipse.che.core.db.jpa.DuplicateKeyException; +import org.eclipse.che.workspace.infrastructure.kubernetes.cache.BeforeKubernetesRuntimeStateRemovedEvent; import org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesRuntimeStateCache; import org.eclipse.che.workspace.infrastructure.kubernetes.model.KubernetesRuntimeState; import org.eclipse.che.workspace.infrastructure.kubernetes.model.KubernetesRuntimeState.RuntimeId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * JPA based implementation of {@link KubernetesRuntimeStateCache}. @@ -33,12 +45,16 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.model.KubernetesRunti * @author Sergii Leshchenko */ public class JpaKubernetesRuntimeStateCache implements KubernetesRuntimeStateCache { + private static final Logger LOG = LoggerFactory.getLogger(JpaKubernetesRuntimeStateCache.class); private final Provider managerProvider; + private final EventService eventService; @Inject - public JpaKubernetesRuntimeStateCache(Provider managerProvider) { + public JpaKubernetesRuntimeStateCache( + Provider managerProvider, EventService eventService) { this.managerProvider = managerProvider; + this.eventService = eventService; } @Override @@ -126,19 +142,42 @@ public class JpaKubernetesRuntimeStateCache implements KubernetesRuntimeStateCac public void remove(RuntimeIdentity runtimeId) throws InfrastructureException { try { doRemove(runtimeId); + } catch (ServerException | RuntimeException x) { + throw new InfrastructureException(x.getMessage(), x); + } + } + + @Transactional(rollbackOn = InfrastructureException.class) + protected Optional find(String workspaceId) + throws InfrastructureException { + try { + KubernetesRuntimeState queried = + managerProvider + .get() + .createNamedQuery("KubernetesRuntime.getByWorkspaceId", KubernetesRuntimeState.class) + .setParameter("workspaceId", workspaceId) + .getSingleResult(); + return Optional.of(queried); + } catch (NoResultException e) { + return Optional.empty(); } catch (RuntimeException x) { throw new InfrastructureException(x.getMessage(), x); } } @Transactional - protected void doRemove(RuntimeIdentity runtimeIdentity) { + protected void doRemove(RuntimeIdentity runtimeIdentity) throws ServerException { EntityManager em = managerProvider.get(); KubernetesRuntimeState runtime = em.find(KubernetesRuntimeState.class, new RuntimeId(runtimeIdentity)); if (runtime != null) { + eventService + .publish( + new BeforeKubernetesRuntimeStateRemovedEvent(new KubernetesRuntimeState(runtime))) + .propagateException(); + em.remove(runtime); } } @@ -186,4 +225,39 @@ public class JpaKubernetesRuntimeStateCache implements KubernetesRuntimeStateCac em.persist(runtimeState); em.flush(); } + + @Singleton + public static class RemoveKubernetesRuntimeBeforeWorkspaceRemoved + extends CascadeEventSubscriber { + + @Inject private EventService eventService; + @Inject private JpaKubernetesRuntimeStateCache k8sRuntimes; + + @PostConstruct + public void subscribe() { + eventService.subscribe(this, BeforeWorkspaceRemovedEvent.class); + } + + @Override + public void onCascadeEvent(BeforeWorkspaceRemovedEvent event) throws Exception { + Optional k8sRuntimeStateOpt = + k8sRuntimes.find(event.getWorkspace().getId()); + if (k8sRuntimeStateOpt.isPresent()) { + KubernetesRuntimeState existingK8sRuntimeState = k8sRuntimeStateOpt.get(); + RuntimeId runtimeId = existingK8sRuntimeState.getRuntimeId(); + + // It is not normal case when non STOPPED workspace is going to be removed. + // Need to log error to investigate why it may happen + // and clean up existing runtime not to lock removing of workspace. + LOG.error( + format( + "Workspace is being removed while Kubernetes runtime state '%s:%s:%s' exists. " + + "This situation indicates a bug that needs to be reported. Runtime state " + + "will be removed from DB, but Kubernetes resources (pods, pvcs, etc.) " + + "won't be cleaned up.", + runtimeId.getWorkspaceId(), runtimeId.getEnvName(), runtimeId.getOwnerId())); + k8sRuntimes.remove(runtimeId); + } + } + } } diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/model/KubernetesRuntimeState.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/model/KubernetesRuntimeState.java index e378ce9254..0edd6b1e32 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/model/KubernetesRuntimeState.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/model/KubernetesRuntimeState.java @@ -25,10 +25,14 @@ import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; @Entity(name = "KubernetesRuntime") @Table(name = "che_k8s_runtime") @NamedQueries({ - @NamedQuery(name = "KubernetesRuntime.getAll", query = "SELECT r FROM KubernetesRuntime r") + @NamedQuery(name = "KubernetesRuntime.getAll", query = "SELECT r FROM KubernetesRuntime r"), + @NamedQuery( + name = "KubernetesRuntime.getByWorkspaceId", + query = "SELECT r FROM KubernetesRuntime r WHERE r.runtimeId.workspaceId = :workspaceId" + ) }) public class KubernetesRuntimeState { - @EmbeddedId private RuntimeId runtimeRuntimeId; + @EmbeddedId private RuntimeId runtimeId; @Column(name = "namespace") private String namespace; @@ -40,7 +44,7 @@ public class KubernetesRuntimeState { public KubernetesRuntimeState( RuntimeIdentity runtimeIdentity, String namespace, WorkspaceStatus status) { - this.runtimeRuntimeId = + this.runtimeId = new RuntimeId( runtimeIdentity.getWorkspaceId(), runtimeIdentity.getEnvName(), @@ -58,7 +62,7 @@ public class KubernetesRuntimeState { } public RuntimeId getRuntimeId() { - return runtimeRuntimeId; + return runtimeId; } public WorkspaceStatus getStatus() { @@ -83,7 +87,7 @@ public class KubernetesRuntimeState { return false; } final KubernetesRuntimeState that = (KubernetesRuntimeState) obj; - return Objects.equals(runtimeRuntimeId, that.runtimeRuntimeId) + return Objects.equals(runtimeId, that.runtimeId) && Objects.equals(namespace, that.namespace) && Objects.equals(status, that.status); } @@ -91,7 +95,7 @@ public class KubernetesRuntimeState { @Override public int hashCode() { int hash = 7; - hash = 31 * hash + Objects.hashCode(runtimeRuntimeId); + hash = 31 * hash + Objects.hashCode(runtimeId); hash = 31 * hash + Objects.hashCode(namespace); hash = 31 * hash + Objects.hashCode(status); return hash; @@ -100,8 +104,8 @@ public class KubernetesRuntimeState { @Override public String toString() { return "KubernetesRuntimeState{" - + "runtimeRuntimeId=" - + runtimeRuntimeId + + "runtimeId=" + + runtimeId + ", namespace='" + namespace + '\'' diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInfraModule.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInfraModule.java index f501c2cc3d..5c53d73a00 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInfraModule.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInfraModule.java @@ -30,10 +30,7 @@ import org.eclipse.che.workspace.infrastructure.docker.environment.dockerimage.D import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientTermination; import org.eclipse.che.workspace.infrastructure.kubernetes.StartSynchronizerFactory; import org.eclipse.che.workspace.infrastructure.kubernetes.bootstrapper.KubernetesBootstrapperFactory; -import org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesMachineCache; -import org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesRuntimeStateCache; -import org.eclipse.che.workspace.infrastructure.kubernetes.cache.jpa.JpaKubernetesMachineCache; -import org.eclipse.che.workspace.infrastructure.kubernetes.cache.jpa.JpaKubernetesRuntimeStateCache; +import org.eclipse.che.workspace.infrastructure.kubernetes.cache.jpa.JpaKubernetesRuntimeCacheModule; import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory; import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.CommonPVCStrategy; import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.UniqueWorkspacePVCStrategy; @@ -91,8 +88,7 @@ public class OpenShiftInfraModule extends AbstractModule { Multibinder.newSetBinder(binder(), EnvVarProvider.class); envVarProviders.addBinding().to(LogsRootEnvVariableProvider.class); - bind(KubernetesRuntimeStateCache.class).to(JpaKubernetesRuntimeStateCache.class); - bind(KubernetesMachineCache.class).to(JpaKubernetesMachineCache.class); + install(new JpaKubernetesRuntimeCacheModule()); Multibinder.newSetBinder(binder(), ServiceTermination.class) .addBinding() diff --git a/wsmaster/integration-tests/cascade-removal/pom.xml b/wsmaster/integration-tests/cascade-removal/pom.xml index 71b6c98643..1984c2b942 100644 --- a/wsmaster/integration-tests/cascade-removal/pom.xml +++ b/wsmaster/integration-tests/cascade-removal/pom.xml @@ -111,6 +111,11 @@ che-core-commons-inject test + + org.eclipse.che.core + che-core-commons-lang + test + org.eclipse.che.core che-core-commons-test @@ -131,6 +136,11 @@ che-core-sql-schema test + + org.eclipse.che.infrastructure + infrastructure-kubernetes + test + org.eclipse.persistence javax.persistence diff --git a/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java b/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java index ab3041931f..5ebac4f893 100644 --- a/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java +++ b/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java @@ -13,6 +13,7 @@ package org.eclipse.che.core.db.jpa; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; import static org.eclipse.che.core.db.jpa.TestObjectsFactory.createAccount; +import static org.eclipse.che.core.db.jpa.TestObjectsFactory.createK8sRuntimeState; import static org.eclipse.che.core.db.jpa.TestObjectsFactory.createPreferences; import static org.eclipse.che.core.db.jpa.TestObjectsFactory.createProfile; import static org.eclipse.che.core.db.jpa.TestObjectsFactory.createSshPair; @@ -44,9 +45,7 @@ import org.eclipse.che.account.api.AccountModule; import org.eclipse.che.account.event.BeforeAccountRemovedEvent; import org.eclipse.che.account.spi.AccountDao; import org.eclipse.che.account.spi.AccountImpl; -import org.eclipse.che.api.core.ConflictException; import org.eclipse.che.api.core.NotFoundException; -import org.eclipse.che.api.core.ServerException; import org.eclipse.che.api.core.notification.EventService; import org.eclipse.che.api.ssh.server.jpa.JpaSshDao.RemoveSshKeysBeforeUserRemovedEventSubscriber; import org.eclipse.che.api.ssh.server.jpa.SshJpaModule; @@ -94,6 +93,12 @@ import org.eclipse.che.core.db.h2.jpa.eclipselink.H2ExceptionHandler; import org.eclipse.che.core.db.schema.SchemaInitializer; import org.eclipse.che.core.db.schema.impl.flyway.FlywaySchemaInitializer; import org.eclipse.che.inject.lifecycle.InitModule; +import org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesMachineCache; +import org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesRuntimeStateCache; +import org.eclipse.che.workspace.infrastructure.kubernetes.cache.jpa.JpaKubernetesRuntimeCacheModule; +import org.eclipse.che.workspace.infrastructure.kubernetes.model.KubernetesMachineImpl; +import org.eclipse.che.workspace.infrastructure.kubernetes.model.KubernetesRuntimeState; +import org.eclipse.che.workspace.infrastructure.kubernetes.model.KubernetesServerImpl; import org.h2.Driver; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -116,6 +121,8 @@ public class CascadeRemovalTest { private WorkspaceDao workspaceDao; private SshDao sshDao; private WorkspaceActivityDao workspaceActivityDao; + private KubernetesRuntimeStateCache k8sRuntimes; + private KubernetesMachineCache k8sMachines; /** Account and User are a root of dependency tree. */ private AccountImpl account; @@ -141,6 +148,8 @@ public class CascadeRemovalTest { private SshPairImpl sshPair2; + private KubernetesRuntimeState k8sRuntimeState; + private H2DBTestServer server; @BeforeMethod @@ -173,7 +182,13 @@ public class CascadeRemovalTest { CommandImpl.class, RecipeImpl.class, SshPairImpl.class, - VolumeImpl.class) + VolumeImpl.class, + KubernetesRuntimeState.class, + KubernetesRuntimeState.RuntimeId.class, + KubernetesMachineImpl.class, + KubernetesMachineImpl.MachineId.class, + KubernetesServerImpl.class, + KubernetesServerImpl.ServerId.class) .addEntityClass( "org.eclipse.che.api.workspace.server.model.impl.ProjectConfigImpl$Attribute") .setExceptionHandler(H2ExceptionHandler.class) @@ -199,6 +214,7 @@ public class CascadeRemovalTest { install(new SshJpaModule()); install(new WorkspaceJpaModule()); install(new WorkspaceActivityModule()); + install(new JpaKubernetesRuntimeCacheModule()); bind(WorkspaceManager.class); RuntimeInfrastructure infra = mock(RuntimeInfrastructure.class); @@ -237,6 +253,8 @@ public class CascadeRemovalTest { sshDao = injector.getInstance(SshDao.class); workspaceDao = injector.getInstance(WorkspaceDao.class); workspaceActivityDao = injector.getInstance(WorkspaceActivityDao.class); + k8sRuntimes = injector.getInstance(KubernetesRuntimeStateCache.class); + k8sMachines = injector.getInstance(KubernetesMachineCache.class); } @AfterMethod @@ -316,7 +334,7 @@ public class CascadeRemovalTest { }; } - private void createTestData() throws ConflictException, ServerException { + private void createTestData() throws Exception { accountDao.create(account = createAccount("bobby")); userDao.create(user = createUser("bobby")); @@ -335,6 +353,11 @@ public class CascadeRemovalTest { sshDao.create(sshPair1 = createSshPair(user.getId(), "service", "name1")); sshDao.create(sshPair2 = createSshPair(user.getId(), "service", "name2")); + + k8sRuntimes.putIfAbsent(k8sRuntimeState = createK8sRuntimeState(workspace1.getId())); + + k8sMachines.put( + k8sRuntimeState.getRuntimeId(), TestObjectsFactory.createK8sMachine(k8sRuntimeState)); } private void wipeTestData() throws Exception { @@ -344,6 +367,9 @@ public class CascadeRemovalTest { workspaceActivityDao.removeExpiration(workspace1.getId()); workspaceActivityDao.removeExpiration(workspace2.getId()); + k8sMachines.remove(k8sRuntimeState.getRuntimeId()); + k8sRuntimes.remove(k8sRuntimeState.getRuntimeId()); + workspaceDao.remove(workspace1.getId()); workspaceDao.remove(workspace2.getId()); diff --git a/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/TestObjectsFactory.java b/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/TestObjectsFactory.java index 97cd598f6b..871be12b64 100644 --- a/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/TestObjectsFactory.java +++ b/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/TestObjectsFactory.java @@ -20,6 +20,9 @@ import java.util.HashMap; import java.util.Map; import org.eclipse.che.account.shared.model.Account; import org.eclipse.che.account.spi.AccountImpl; +import org.eclipse.che.api.core.model.workspace.WorkspaceStatus; +import org.eclipse.che.api.core.model.workspace.runtime.MachineStatus; +import org.eclipse.che.api.core.model.workspace.runtime.ServerStatus; import org.eclipse.che.api.ssh.server.model.impl.SshPairImpl; import org.eclipse.che.api.user.server.model.impl.ProfileImpl; import org.eclipse.che.api.user.server.model.impl.UserImpl; @@ -28,13 +31,18 @@ import org.eclipse.che.api.workspace.server.model.impl.EnvironmentImpl; import org.eclipse.che.api.workspace.server.model.impl.MachineConfigImpl; import org.eclipse.che.api.workspace.server.model.impl.ProjectConfigImpl; import org.eclipse.che.api.workspace.server.model.impl.RecipeImpl; +import org.eclipse.che.api.workspace.server.model.impl.RuntimeIdentityImpl; import org.eclipse.che.api.workspace.server.model.impl.ServerConfigImpl; +import org.eclipse.che.api.workspace.server.model.impl.ServerImpl; import org.eclipse.che.api.workspace.server.model.impl.SourceStorageImpl; import org.eclipse.che.api.workspace.server.model.impl.WorkspaceConfigImpl; import org.eclipse.che.api.workspace.server.model.impl.WorkspaceImpl; import org.eclipse.che.api.workspace.server.model.impl.stack.StackComponentImpl; import org.eclipse.che.api.workspace.server.model.impl.stack.StackImpl; import org.eclipse.che.api.workspace.server.stack.image.StackIcon; +import org.eclipse.che.commons.lang.NameGenerator; +import org.eclipse.che.workspace.infrastructure.kubernetes.model.KubernetesMachineImpl; +import org.eclipse.che.workspace.infrastructure.kubernetes.model.KubernetesRuntimeState; /** * Defines method for creating tests object instances. @@ -153,5 +161,28 @@ public final class TestObjectsFactory { .build(); } + public static KubernetesRuntimeState createK8sRuntimeState(String workspaceId) { + return new KubernetesRuntimeState( + new RuntimeIdentityImpl(workspaceId, "envName", "ownerId"), + "test-namespace", + WorkspaceStatus.RUNNING); + } + + public static KubernetesMachineImpl createK8sMachine(KubernetesRuntimeState k8sRuntimeState) { + return new KubernetesMachineImpl( + k8sRuntimeState.getRuntimeId().getWorkspaceId(), + NameGenerator.generate("machine-", 5), + NameGenerator.generate("pod-", 5), + NameGenerator.generate("container-", 5), + MachineStatus.RUNNING, + ImmutableMap.of("test", "true"), + ImmutableMap.of( + "server", + new ServerImpl( + "http://localhost:8080/api", + ServerStatus.RUNNING, + ImmutableMap.of("key", "value")))); + } + private TestObjectsFactory() {} } diff --git a/wsmaster/integration-tests/postgresql-tck/src/test/java/PostgreSqlTckModule.java b/wsmaster/integration-tests/postgresql-tck/src/test/java/PostgreSqlTckModule.java index f50a5a4816..66b61b2d7a 100644 --- a/wsmaster/integration-tests/postgresql-tck/src/test/java/PostgreSqlTckModule.java +++ b/wsmaster/integration-tests/postgresql-tck/src/test/java/PostgreSqlTckModule.java @@ -207,6 +207,10 @@ public class PostgreSqlTckModule extends TckModule { bind(KubernetesRuntimeStateCache.class).to(JpaKubernetesRuntimeStateCache.class); bind(KubernetesMachineCache.class).to(JpaKubernetesMachineCache.class); + bind(JpaKubernetesRuntimeStateCache.RemoveKubernetesRuntimeBeforeWorkspaceRemoved.class) + .asEagerSingleton(); + bind(JpaKubernetesMachineCache.RemoveKubernetesMachinesBeforeRuntimesRemoved.class) + .asEagerSingleton(); } private static void waitConnectionIsEstablished(String dbUrl, String dbUser, String dbPassword) {