idsToStop;
try (@SuppressWarnings("unused") Unlocker u = locks.writeAllLock()) {
idsToStop = states.entrySet()
.stream()
- .filter(e -> e.getValue().status != STOPPING)
+ .filter(e -> e.getValue().status != WorkspaceStatus.STOPPING)
.map(Map.Entry::getKey)
.collect(Collectors.toList());
states.clear();
}
- // nothing to stop
- if (idsToStop.isEmpty()) {
- return;
- }
-
- LOG.info("Shutdown running states, states to shutdown '{}'", idsToStop.size());
- ExecutorService executor =
- Executors.newFixedThreadPool(2 * Runtime.getRuntime().availableProcessors(),
- new ThreadFactoryBuilder().setNameFormat("StopEnvironmentsPool-%d")
- .setDaemon(false)
- .build());
- for (String id : idsToStop) {
- executor.execute(() -> {
- try {
- envEngine.stop(id);
- } catch (EnvironmentNotRunningException ignored) {
- // could be stopped during workspace pool shutdown
- } catch (Exception x) {
- LOG.error(x.getMessage(), x);
- }
- });
- }
-
- executor.shutdown();
- try {
- if (!executor.awaitTermination(30, TimeUnit.SECONDS)) {
- executor.shutdownNow();
- if (!executor.awaitTermination(60, TimeUnit.SECONDS)) {
- LOG.error("Unable terminate machines pool");
- }
+ if (!idsToStop.isEmpty()) {
+ LOG.info("Shutdown running environments, environments to stop: '{}'", idsToStop.size());
+ ExecutorService executor =
+ Executors.newFixedThreadPool(2 * Runtime.getRuntime().availableProcessors(),
+ new ThreadFactoryBuilder().setNameFormat("StopEnvironmentsPool-%d")
+ .setDaemon(false)
+ .build());
+ for (String id : idsToStop) {
+ executor.execute(() -> {
+ try {
+ envEngine.stop(id);
+ } catch (EnvironmentNotRunningException ignored) {
+ // might be already stopped
+ } catch (Exception x) {
+ LOG.error(x.getMessage(), x);
+ }
+ });
+ }
+
+ executor.shutdown();
+ try {
+ if (!executor.awaitTermination(30, TimeUnit.SECONDS)) {
+ executor.shutdownNow();
+ if (!executor.awaitTermination(60, TimeUnit.SECONDS)) {
+ LOG.error("Unable to stop runtimes termination pool");
+ }
+ }
+ } catch (InterruptedException e) {
+ executor.shutdownNow();
+ Thread.currentThread().interrupt();
}
- } catch (InterruptedException e) {
- executor.shutdownNow();
- Thread.currentThread().interrupt();
}
}
- private void ensurePreDestroyIsNotExecuted() {
- if (isPreDestroyInvoked) {
- throw new IllegalStateException("Could not perform operation because application server is stopping");
+ private void checkIsNotTerminated(String operation) throws ServerException {
+ if (isShutdown.get()) {
+ throw new ServerException("Could not " + operation + " because workspaces service is being terminated");
}
}
@@ -711,7 +737,7 @@ public class WorkspaceRuntimes {
// disallow direct start cancellation, STARTING -> RUNNING
WorkspaceStatus prevStatus;
try (@SuppressWarnings("unused") Unlocker u = locks.writeLock(workspaceId)) {
- ensurePreDestroyIsNotExecuted();
+ checkIsNotTerminated("finish workspace start");
RuntimeState state = states.get(workspaceId);
prevStatus = state.status;
if (state.status == WorkspaceStatus.STARTING) {
@@ -799,9 +825,9 @@ public class WorkspaceRuntimes {
* with {@code from} and if they are equal sets the status to {@code to}.
* Returns true if the status of workspace was updated with {@code to} value.
*/
- private boolean compareAndSetStatus(String id, WorkspaceStatus from, WorkspaceStatus to) {
+ private boolean compareAndSetStatus(String id, WorkspaceStatus from, WorkspaceStatus to) throws ServerException {
try (@SuppressWarnings("unused") Unlocker u = locks.writeLock(id)) {
- ensurePreDestroyIsNotExecuted();
+ checkIsNotTerminated(format("change status from '%s' to '%s' for the workspace '%s'", from, to, id));
RuntimeState state = states.get(id);
if (state != null && state.status == from) {
state.status = to;
@@ -814,7 +840,6 @@ public class WorkspaceRuntimes {
/** Removes state from in-memory storage in write lock. */
private void removeState(String workspaceId) {
try (@SuppressWarnings("unused") Unlocker u = locks.writeLock(workspaceId)) {
- ensurePreDestroyIsNotExecuted();
states.remove(workspaceId);
}
}
@@ -954,7 +979,7 @@ public class WorkspaceRuntimes {
cmpFuture.complete(runtime);
return runtime;
} catch (IllegalStateException illegalStateEx) {
- if (isPreDestroyInvoked) {
+ if (isShutdown.get()) {
exception = new EnvironmentStartInterruptedException(workspaceId, envName);
} else {
exception = new ServerException(illegalStateEx.getMessage(), illegalStateEx);
@@ -1029,6 +1054,7 @@ public class WorkspaceRuntimes {
launchAgents(machine, extMachine.getAgents());
}
}
+
}
private static EnvironmentImpl copyEnv(Workspace workspace, String envName) {
diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceSharedPool.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceSharedPool.java
index fecee50a59..cb4fce6ac3 100644
--- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceSharedPool.java
+++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceSharedPool.java
@@ -10,7 +10,6 @@
*******************************************************************************/
package org.eclipse.che.api.workspace.server;
-import com.google.common.base.MoreObjects;
import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.inject.Inject;
@@ -21,10 +20,10 @@ import org.eclipse.che.commons.lang.concurrent.ThreadLocalPropagateContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.annotation.PostConstruct;
import javax.inject.Named;
import javax.inject.Singleton;
import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@@ -94,38 +93,37 @@ public class WorkspaceSharedPool {
}
/**
- * Terminates this pool, may be called multiple times,
- * waits until pool is terminated or timeout is reached.
+ * Asynchronously runs the given task wrapping it with {@link ThreadLocalPropagateContext#wrap(Runnable)}
*
- * Note that the method is not designed to be used from
- * different threads, but the other components may use it in their
- * post construct methods to ensure that all the tasks finished their execution.
- *
- * @return true if executor successfully terminated and false if not
- * terminated(either await termination timeout is reached or thread was interrupted)
+ * @param runnable
+ * task to run
+ * @return completable future bounded to the task
*/
- @PostConstruct
- public boolean terminateAndWait() {
- if (executor.isShutdown()) {
- return true;
- }
- Logger logger = LoggerFactory.getLogger(getClass());
- executor.shutdown();
- try {
- logger.info("Shutdown workspace threads pool, wait 30s to stop normally");
- if (!executor.awaitTermination(30, TimeUnit.SECONDS)) {
- executor.shutdownNow();
- logger.info("Interrupt workspace threads pool, wait 60s to stop");
- if (!executor.awaitTermination(60, TimeUnit.SECONDS)) {
- logger.error("Couldn't terminate workspace threads pool");
- return false;
+ public CompletableFuture runAsync(Runnable runnable) {
+ return CompletableFuture.runAsync(ThreadLocalPropagateContext.wrap(runnable), executor);
+ }
+
+ /**
+ * Terminates this pool if it's not terminated yet.
+ */
+ void shutdown() {
+ if (!executor.isShutdown()) {
+ Logger logger = LoggerFactory.getLogger(getClass());
+ executor.shutdown();
+ try {
+ logger.info("Shutdown workspace threads pool, wait 30s to stop normally");
+ if (!executor.awaitTermination(30, TimeUnit.SECONDS)) {
+ executor.shutdownNow();
+ logger.info("Interrupt workspace threads pool, wait 60s to stop");
+ if (!executor.awaitTermination(60, TimeUnit.SECONDS)) {
+ logger.error("Couldn't shutdown workspace threads pool");
+ }
}
+ } catch (InterruptedException x) {
+ executor.shutdownNow();
+ Thread.currentThread().interrupt();
}
- } catch (InterruptedException x) {
- executor.shutdownNow();
- Thread.currentThread().interrupt();
- return false;
+ logger.info("Workspace threads pool is terminated");
}
- return true;
}
}
diff --git a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceManagerTest.java b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceManagerTest.java
index 129cd4d4f2..3fccd73395 100644
--- a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceManagerTest.java
+++ b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceManagerTest.java
@@ -43,12 +43,14 @@ import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.InOrder;
import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
import org.mockito.testng.MockitoTestNGListener;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -71,6 +73,7 @@ import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.inOrder;
@@ -117,6 +120,7 @@ public class WorkspaceManagerTest {
@BeforeMethod
public void setUp() throws Exception {
+ MockitoAnnotations.initMocks(this);
workspaceManager = new WorkspaceManager(workspaceDao,
runtimes,
eventService,
@@ -463,7 +467,7 @@ public class WorkspaceManagerTest {
workspaceManager.stopWorkspace(workspace.getId());
// then
- captureAsyncTaskAndExecuteSynchronously();
+ captureRunAsyncCallsAndRunSynchronously();
verify(runtimes).stop(workspace.getId());
@@ -479,7 +483,7 @@ public class WorkspaceManagerTest {
workspaceManager.stopWorkspace(workspace.getId(), true);
- captureAsyncTaskAndExecuteSynchronously();
+ captureRunAsyncCallsAndRunSynchronously();
verify(runtimes).snapshot(workspace.getId());
}
@@ -501,7 +505,7 @@ public class WorkspaceManagerTest {
workspaceManager.stopWorkspace(workspace.getId(), true);
- captureAsyncTaskAndExecuteSynchronously();
+ captureRunAsyncCallsAndRunSynchronously();
verify(runtimes).stop(any());
}
@@ -513,7 +517,7 @@ public class WorkspaceManagerTest {
workspaceManager.stopWorkspace(workspace.getId());
- captureAsyncTaskAndExecuteSynchronously();
+ captureRunAsyncCallsAndRunSynchronously();
verify(workspaceDao).remove(workspace.getId());
}
@@ -526,7 +530,7 @@ public class WorkspaceManagerTest {
workspaceManager.stopWorkspace(workspace.getId());
- captureAsyncTaskAndExecuteSynchronously();
+ captureRunAsyncCallsAndRunSynchronously();
verify(workspaceDao).remove(workspace.getId());
}
@@ -562,7 +566,7 @@ public class WorkspaceManagerTest {
workspaceManager.stopWorkspace(workspace.getId());
- captureAsyncTaskAndExecuteSynchronously();
+ captureRunAsyncCallsAndRunSynchronously();
verify(runtimes, never()).snapshot(workspace.getId());
verify(runtimes).stop(workspace.getId());
}
@@ -582,7 +586,7 @@ public class WorkspaceManagerTest {
workspaceManager.stopWorkspace(workspace.getId());
- captureAsyncTaskAndExecuteSynchronously();
+ captureRunAsyncCallsAndRunSynchronously();
verify(runtimes, never()).snapshot(workspace.getId());
verify(runtimes).stop(workspace.getId());
}
@@ -612,7 +616,7 @@ public class WorkspaceManagerTest {
workspaceManager.stopWorkspace(workspace.getId());
// then
- captureAsyncTaskAndExecuteSynchronously();
+ captureRunAsyncCallsAndRunSynchronously();
verify(runtimes).snapshot(workspace.getId());
verify(runtimes).stop(workspace.getId());
}
@@ -678,7 +682,7 @@ public class WorkspaceManagerTest {
workspaceManager.removeSnapshots(testWsId);
// then
- captureAsyncTaskAndExecuteSynchronously();
+ captureExecuteCallsAndRunSynchronously();
verify(runtimes).removeBinaries(asList(snapshot1, snapshot2));
InOrder snapshotDaoInOrder = inOrder(snapshotDao);
snapshotDaoInOrder.verify(snapshotDao).removeSnapshot(snapshot1.getId());
@@ -713,7 +717,7 @@ public class WorkspaceManagerTest {
workspaceManager.removeSnapshots(testWsId);
// then
- captureAsyncTaskAndExecuteSynchronously();
+ captureExecuteCallsAndRunSynchronously();
verify(runtimes).removeBinaries(singletonList(snapshot2));
verify(snapshotDao).removeSnapshot(snapshot1.getId());
verify(snapshotDao).removeSnapshot(snapshot2.getId());
@@ -732,7 +736,7 @@ public class WorkspaceManagerTest {
workspaceManager.startMachine(machineConfig, workspace.getId());
// then
- captureAsyncTaskAndExecuteSynchronously();
+ captureExecuteCallsAndRunSynchronously();
verify(runtimes).startMachine(workspace.getId(), machineConfig);
}
@@ -824,7 +828,7 @@ public class WorkspaceManagerTest {
workspaceManager.stopWorkspace(workspace.getId(), false);
- captureAsyncTaskAndExecuteSynchronously();
+ captureRunAsyncCallsAndRunSynchronously();
verify(runtimes, never()).snapshot(workspace.getId());
}
@@ -836,7 +840,7 @@ public class WorkspaceManagerTest {
workspaceManager.stopWorkspace(workspace.getId(), null);
- captureAsyncTaskAndExecuteSynchronously();
+ captureRunAsyncCallsAndRunSynchronously();
verify(runtimes).snapshot(workspace.getId());
}
@@ -848,13 +852,48 @@ public class WorkspaceManagerTest {
workspaceManager.stopWorkspace(workspace.getId(), false);
- captureAsyncTaskAndExecuteSynchronously();
+ captureRunAsyncCallsAndRunSynchronously();
verify(runtimes, never()).snapshot(workspace.getId());
}
- private void captureAsyncTaskAndExecuteSynchronously() {
- verify(sharedPool).execute(taskCaptor.capture());
- taskCaptor.getValue().run();
+ @Test
+ public void stopsRunningWorkspacesOnShutdown() throws Exception {
+ when(runtimes.refuseWorkspacesStart()).thenReturn(true);
+
+ WorkspaceImpl stopped = createAndMockWorkspace();
+ mockRuntime(stopped, STOPPED);
+
+ WorkspaceImpl starting = createAndMockWorkspace();
+ mockRuntime(starting, STARTING);
+
+ WorkspaceImpl running = createAndMockWorkspace();
+ mockRuntime(running, RUNNING);
+
+ when(runtimes.getRuntimesIds()).thenReturn(new HashSet<>(asList(running.getId(), starting.getId())));
+
+ // action
+ workspaceManager.shutdown();
+
+ captureRunAsyncCallsAndRunSynchronously();
+ verify(runtimes).stop(running.getId());
+ verify(runtimes).stop(starting.getId());
+ verify(runtimes, never()).stop(stopped.getId());
+ verify(runtimes).shutdown();
+ verify(sharedPool).shutdown();
+ }
+
+ private void captureRunAsyncCallsAndRunSynchronously() {
+ verify(sharedPool, atLeastOnce()).runAsync(taskCaptor.capture());
+ for (Runnable runnable : taskCaptor.getAllValues()) {
+ runnable.run();
+ }
+ }
+
+ private void captureExecuteCallsAndRunSynchronously() {
+ verify(sharedPool, atLeastOnce()).execute(taskCaptor.capture());
+ for (Runnable runnable : taskCaptor.getAllValues()) {
+ runnable.run();
+ }
}
private WorkspaceRuntimeImpl mockRuntime(WorkspaceImpl workspace, WorkspaceStatus status) {
@@ -870,6 +909,7 @@ public class WorkspaceManagerTest {
workspace.setRuntime(runtime);
return null;
}).when(runtimes).injectRuntime(workspace);
+ when(runtimes.isAnyRunning()).thenReturn(true);
return runtime;
}
diff --git a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimesTest.java b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimesTest.java
index 9a922eb155..56c61b7513 100644
--- a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimesTest.java
+++ b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimesTest.java
@@ -572,15 +572,22 @@ public class WorkspaceRuntimesTest {
}
@Test
- public void cleanup() throws Exception {
+ public void shutdown() throws Exception {
setRuntime("workspace", WorkspaceStatus.RUNNING, "env-name");
- runtimes.cleanup();
+ runtimes.shutdown();
assertFalse(runtimes.hasRuntime("workspace"));
verify(envEngine).stop("workspace");
}
+ @Test(expectedExceptions = IllegalStateException.class,
+ expectedExceptionsMessageRegExp = "Workspace runtimes service shutdown has been already called")
+ public void throwsExceptionWhenShutdownCalledTwice() throws Exception {
+ runtimes.shutdown();
+ runtimes.shutdown();
+ }
+
@Test
public void startedRuntimeAndReturnedFromGetMethodAreTheSame() throws Exception {
WorkspaceImpl workspace = newWorkspace("workspace", "env-name");
@@ -795,12 +802,32 @@ public class WorkspaceRuntimesTest {
"workspace4"));
}
+ @Test
+ public void isAnyRunningReturnsFalseIfThereIsNoSingleRuntime() {
+ assertFalse(runtimes.isAnyRunning());
+ }
+
+ @Test
+ public void isAnyRunningReturnsTrueIfThereIsAtLeastOneRunningWorkspace() {
+ setRuntime("workspace1", WorkspaceStatus.STARTING);
+
+ assertTrue(runtimes.isAnyRunning());
+ }
+
+ @Test(expectedExceptions = ConflictException.class,
+ expectedExceptionsMessageRegExp = "Start of the workspace 'test-workspace' is rejected by the system, " +
+ "no more workspaces are allowed to start")
+ public void doesNotAllowToStartWorkspaceIfStartIsRefused() throws Exception {
+ runtimes.refuseWorkspacesStart();
+
+ runtimes.startAsync(newWorkspace("workspace1", "env-name"), "env-name", false);
+ }
+
private void captureAsyncTaskAndExecuteSynchronously() throws Exception {
verify(sharedPool).submit(taskCaptor.capture());
taskCaptor.getValue().call();
}
-
private void captureAndVerifyRuntimeStateAfterInterruption(Workspace workspace,
CompletableFuture cmpFuture) throws Exception {
try {
diff --git a/wsmaster/pom.xml b/wsmaster/pom.xml
index 6a38ff5264..1ceb10d4ff 100644
--- a/wsmaster/pom.xml
+++ b/wsmaster/pom.xml
@@ -41,5 +41,7 @@
wsmaster-local
che-core-sql-schema
integration-tests
+ che-core-api-system
+ che-core-api-system-shared