diff --git a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/multiuser.properties b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/multiuser.properties index a976e6e080..aa64d180f9 100644 --- a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/multiuser.properties +++ b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/multiuser.properties @@ -40,6 +40,10 @@ che.limits.workspace.env.ram=16gb # counts toward idleness. che.limits.workspace.idle.timeout=1800000 +# The length of time that a workspace will run, regardless of activity, before +# the system will suspend it. +che.limits.workspace.run.timeout=0 + ### Users workspace limits # The total amount of RAM that a single user is allowed to allocate to running diff --git a/multiuser/api/che-multiuser-api-workspace-activity/src/main/java/org/eclipse/che/multiuser/api/workspace/activity/MultiUserWorkspaceActivityManager.java b/multiuser/api/che-multiuser-api-workspace-activity/src/main/java/org/eclipse/che/multiuser/api/workspace/activity/MultiUserWorkspaceActivityManager.java index d98fc45b9f..d60d526e1b 100644 --- a/multiuser/api/che-multiuser-api-workspace-activity/src/main/java/org/eclipse/che/multiuser/api/workspace/activity/MultiUserWorkspaceActivityManager.java +++ b/multiuser/api/che-multiuser-api-workspace-activity/src/main/java/org/eclipse/che/multiuser/api/workspace/activity/MultiUserWorkspaceActivityManager.java @@ -46,6 +46,7 @@ public class MultiUserWorkspaceActivityManager extends WorkspaceActivityManager private final AccountManager accountManager; private final ResourceManager resourceManager; private final long defaultTimeout; + private final long runTimeout; @Inject public MultiUserWorkspaceActivityManager( @@ -54,11 +55,13 @@ public class MultiUserWorkspaceActivityManager extends WorkspaceActivityManager EventService eventService, AccountManager accountManager, ResourceManager resourceManager, - @Named("che.limits.workspace.idle.timeout") long defaultTimeout) { - super(workspaceManager, activityDao, eventService, defaultTimeout); + @Named("che.limits.workspace.idle.timeout") long defaultTimeout, + @Named("che.limits.workspace.run.timeout") long runTimeout) { + super(workspaceManager, activityDao, eventService, defaultTimeout, runTimeout); this.accountManager = accountManager; this.resourceManager = resourceManager; this.defaultTimeout = defaultTimeout; + this.runTimeout = runTimeout; } @Override diff --git a/multiuser/api/che-multiuser-api-workspace-activity/src/test/java/org/eclipse/che/multiuser/api/workspace/activity/MultiUserWorkspaceActivityManagerTest.java b/multiuser/api/che-multiuser-api-workspace-activity/src/test/java/org/eclipse/che/multiuser/api/workspace/activity/MultiUserWorkspaceActivityManagerTest.java index 511c0915e0..1b39b95586 100644 --- a/multiuser/api/che-multiuser-api-workspace-activity/src/test/java/org/eclipse/che/multiuser/api/workspace/activity/MultiUserWorkspaceActivityManagerTest.java +++ b/multiuser/api/che-multiuser-api-workspace-activity/src/test/java/org/eclipse/che/multiuser/api/workspace/activity/MultiUserWorkspaceActivityManagerTest.java @@ -35,6 +35,7 @@ import org.testng.annotations.Test; public class MultiUserWorkspaceActivityManagerTest { private static final long DEFAULT_TIMEOUT = 60_000L; // 1 minute private static final long USER_LIMIT_TIMEOUT = 120_000L; // 2 minutes + private static final long DEFAULT_HARD_EXPIRATION_TIMEOUT = 60000 * 60 * 3; // 3 hours @Mock private AccountManager accountManager; @Mock private ResourceManager resourceManager; @@ -58,7 +59,8 @@ public class MultiUserWorkspaceActivityManagerTest { eventService, accountManager, resourceManager, - DEFAULT_TIMEOUT); + DEFAULT_TIMEOUT, + DEFAULT_HARD_EXPIRATION_TIMEOUT); when(account.getId()).thenReturn("account123"); when(accountManager.getByName(anyString())).thenReturn(account); diff --git a/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/InmemoryWorkspaceActivityDao.java b/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/InmemoryWorkspaceActivityDao.java index 22ff994daa..bf5fca19bd 100644 --- a/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/InmemoryWorkspaceActivityDao.java +++ b/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/InmemoryWorkspaceActivityDao.java @@ -43,11 +43,14 @@ public class InmemoryWorkspaceActivityDao implements WorkspaceActivityDao { } @Override - public List findExpired(long timestamp) { + public List findExpired(long timestamp, long runTimeout) { return workspaceActivities .values() .stream() - .filter(a -> a.getExpiration() != null && a.getExpiration() < timestamp) + .filter( + a -> + (a.getExpiration() != null && a.getExpiration() < timestamp) + || (runTimeout > 0 && a.getLastRunning() - a.getLastStarting() > runTimeout)) .map(WorkspaceActivity::getWorkspaceId) .collect(toList()); } diff --git a/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/JpaWorkspaceActivityDao.java b/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/JpaWorkspaceActivityDao.java index 7d8b827327..d02c2f0b1a 100644 --- a/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/JpaWorkspaceActivityDao.java +++ b/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/JpaWorkspaceActivityDao.java @@ -52,12 +52,13 @@ public class JpaWorkspaceActivityDao implements WorkspaceActivityDao { @Override @Transactional(rollbackOn = ServerException.class) - public List findExpired(long timestamp) throws ServerException { + public List findExpired(long timestamp, long runTimeout) throws ServerException { try { return managerProvider .get() .createNamedQuery("WorkspaceActivity.getExpired", WorkspaceActivity.class) .setParameter("expiration", timestamp) + .setParameter("runTimeout", runTimeout) .getResultList() .stream() .map(WorkspaceActivity::getWorkspaceId) diff --git a/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivity.java b/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivity.java index d83b3101f1..a9240bb94e 100644 --- a/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivity.java +++ b/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivity.java @@ -27,7 +27,11 @@ import org.eclipse.che.api.core.model.workspace.WorkspaceStatus; @NamedQueries({ @NamedQuery( name = "WorkspaceActivity.getExpired", - query = "SELECT a FROM WorkspaceActivity a WHERE a.expiration < :expiration"), + query = + "SELECT a FROM WorkspaceActivity a WHERE a.expiration < :expiration OR " + + "(a.status = org.eclipse.che.api.core.model.workspace.WorkspaceStatus.RUNNING AND " + + ":runTimeout > 0 AND " + + ":expiration - a.lastRunning > :runTimeout)"), @NamedQuery( name = "WorkspaceActivity.getStoppedSince", query = @@ -165,6 +169,10 @@ public class WorkspaceActivity { this.expiration = expiration; } + public Long getRunTimeout() { + return this.lastRunning - this.lastStarting; + } + public WorkspaceStatus getStatus() { return status; } @@ -181,6 +189,7 @@ public class WorkspaceActivity { if (o == null || getClass() != o.getClass()) { return false; } + WorkspaceActivity activity = (WorkspaceActivity) o; return Objects.equals(workspaceId, activity.workspaceId) && Objects.equals(created, activity.created) diff --git a/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityChecker.java b/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityChecker.java index 80923e2d67..0df9604921 100644 --- a/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityChecker.java +++ b/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityChecker.java @@ -101,7 +101,9 @@ public class WorkspaceActivityChecker { private void stopAllExpired() { try { - activityDao.findExpired(clock.millis()).forEach(this::stopExpiredQuietly); + activityDao + .findExpired(clock.millis(), workspaceActivityManager.getRunTimeout()) + .forEach(this::stopExpiredQuietly); } catch (ServerException e) { LOG.error("Failed to list all expired to perform stop. Cause: {}", e.getMessage(), e); } diff --git a/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityDao.java b/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityDao.java index d7cda269a1..0c1d3e297b 100644 --- a/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityDao.java +++ b/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityDao.java @@ -46,10 +46,11 @@ public interface WorkspaceActivityDao { * Finds workspaces which are passed given expiration time and must be stopped. * * @param timestamp expiration time + * @param runTimeout time after which the workspace will be stopped regardless of activity * @return list of workspaces which expiration time is older than given timestamp * @throws ServerException when operation failed */ - List findExpired(long timestamp) throws ServerException; + List findExpired(long timestamp, long runTimeout) throws ServerException; /** * Removes the activity record of the provided workspace. diff --git a/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityManager.java b/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityManager.java index 7a12b328ad..aa48140d6f 100644 --- a/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityManager.java +++ b/wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityManager.java @@ -54,6 +54,7 @@ public class WorkspaceActivityManager { private static final Logger LOG = LoggerFactory.getLogger(WorkspaceActivityManager.class); private final long defaultTimeout; + private final long runTimeout; private final WorkspaceActivityDao activityDao; private final EventService eventService; private final EventSubscriber updateStatusChangedTimestampSubscriber; @@ -69,9 +70,16 @@ public class WorkspaceActivityManager { WorkspaceManager workspaceManager, WorkspaceActivityDao activityDao, EventService eventService, - @Named("che.limits.workspace.idle.timeout") long timeout) { + @Named("che.limits.workspace.idle.timeout") long timeout, + @Named("che.limits.workspace.run.timeout") long runTimeout) { - this(workspaceManager, activityDao, eventService, timeout, Clock.systemDefaultZone()); + this( + workspaceManager, + activityDao, + eventService, + timeout, + runTimeout, + Clock.systemDefaultZone()); } @VisibleForTesting @@ -80,11 +88,13 @@ public class WorkspaceActivityManager { WorkspaceActivityDao activityDao, EventService eventService, long timeout, + long runTimeout, Clock clock) { this.workspaceManager = workspaceManager; this.eventService = eventService; this.activityDao = activityDao; this.defaultTimeout = timeout; + this.runTimeout = runTimeout; this.clock = clock; if (timeout > 0 && timeout < MINIMAL_TIMEOUT) { LOG.warn( @@ -116,7 +126,6 @@ public class WorkspaceActivityManager { activityDao.removeActivity(event.getWorkspace().getId()); } }; - this.updateStatusChangedTimestampSubscriber = new UpdateStatusChangedTimestampSubscriber(); } @@ -170,6 +179,10 @@ public class WorkspaceActivityManager { return defaultTimeout; } + protected long getRunTimeout() { + return runTimeout; + } + private class UpdateStatusChangedTimestampSubscriber implements EventSubscriber { @Override diff --git a/wsmaster/che-core-api-workspace-activity/src/test/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityCheckerTest.java b/wsmaster/che-core-api-workspace-activity/src/test/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityCheckerTest.java index fce62a3fd2..a5d51a1fe1 100644 --- a/wsmaster/che-core-api-workspace-activity/src/test/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityCheckerTest.java +++ b/wsmaster/che-core-api-workspace-activity/src/test/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityCheckerTest.java @@ -55,6 +55,8 @@ import org.testng.annotations.Test; @Listeners(value = MockitoTestNGListener.class) public class WorkspaceActivityCheckerTest { private static final long DEFAULT_TIMEOUT = 60_000L; // 1 minute + private static final long DEFAULT_RUN_TIMEOUT = 0; // No default run timeout + private static final long ACTIVE_RUN_TIMEOUT = 60000 * 60 * 3; // 3 hours private ManualClock clock; private WorkspaceActivityChecker checker; @@ -69,7 +71,12 @@ public class WorkspaceActivityCheckerTest { WorkspaceActivityManager activityManager = new WorkspaceActivityManager( - workspaceManager, workspaceActivityDao, eventService, DEFAULT_TIMEOUT, clock); + workspaceManager, + workspaceActivityDao, + eventService, + DEFAULT_TIMEOUT, + DEFAULT_RUN_TIMEOUT, + clock); lenient() .when(workspaceActivityDao.getAll(anyInt(), anyLong())) @@ -88,7 +95,7 @@ public class WorkspaceActivityCheckerTest { @Test public void shouldStopAllExpiredWorkspaces() throws Exception { - when(workspaceActivityDao.findExpired(anyLong())).thenReturn(asList("1", "2", "3")); + when(workspaceActivityDao.findExpired(anyLong(), anyLong())).thenReturn(asList("1", "2", "3")); checker.expire(); diff --git a/wsmaster/che-core-api-workspace-activity/src/test/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityManagerTest.java b/wsmaster/che-core-api-workspace-activity/src/test/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityManagerTest.java index 21298b42e5..f9b8461177 100644 --- a/wsmaster/che-core-api-workspace-activity/src/test/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityManagerTest.java +++ b/wsmaster/che-core-api-workspace-activity/src/test/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityManagerTest.java @@ -49,6 +49,8 @@ import org.testng.annotations.Test; public class WorkspaceActivityManagerTest { private static final long DEFAULT_TIMEOUT = 60_000L; // 1 minute + private static final long DEFAULT_RUN_TIMEOUT = 0; // No run timeout + private static final long ACTIVE_RUN_TIMEOUT = 60000 * 60 * 3; // 3 hours @Mock private WorkspaceManager workspaceManager; @@ -68,7 +70,11 @@ public class WorkspaceActivityManagerTest { private void setUp() throws Exception { activityManager = new WorkspaceActivityManager( - workspaceManager, workspaceActivityDao, eventService, DEFAULT_TIMEOUT); + workspaceManager, + workspaceActivityDao, + eventService, + DEFAULT_TIMEOUT, + DEFAULT_RUN_TIMEOUT); lenient().when(account.getName()).thenReturn("accountName"); lenient().when(account.getId()).thenReturn("account123"); @@ -102,6 +108,19 @@ public class WorkspaceActivityManagerTest { assertEquals(wsIdCaptor.getValue(), wsId); } + @Test + public void shouldRemoveRunTimeoutWhenWorkspaceStopped() throws Exception { + final String wsId = "testWsId"; + final EventSubscriber subscriber = subscribeAndGetStatusEventSubscriber(); + + subscriber.onEvent( + DtoFactory.newDto(WorkspaceStatusEvent.class) + .withStatus(WorkspaceStatus.STOPPED) + .withWorkspaceId(wsId)); + ArgumentCaptor wsIdCaptor = ArgumentCaptor.forClass(String.class); + verify(workspaceActivityDao, times(1)).removeExpiration(wsIdCaptor.capture()); + } + @Test public void shouldCeaseToTrackTheWorkspaceActivityAfterStopping() throws Exception { final String wsId = "testWsId"; diff --git a/wsmaster/che-core-api-workspace-activity/src/test/java/org/eclipse/che/api/workspace/activity/spi/tck/WorkspaceActivityDaoTest.java b/wsmaster/che-core-api-workspace-activity/src/test/java/org/eclipse/che/api/workspace/activity/spi/tck/WorkspaceActivityDaoTest.java index ac2b6cfad1..4b7b47c7f9 100644 --- a/wsmaster/che-core-api-workspace-activity/src/test/java/org/eclipse/che/api/workspace/activity/spi/tck/WorkspaceActivityDaoTest.java +++ b/wsmaster/che-core-api-workspace-activity/src/test/java/org/eclipse/che/api/workspace/activity/spi/tck/WorkspaceActivityDaoTest.java @@ -12,8 +12,7 @@ package org.eclipse.che.api.workspace.activity.spi.tck; import static java.util.Arrays.asList; -import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; +import static java.util.Collections.*; import static org.eclipse.che.api.core.model.workspace.WorkspaceStatus.STARTING; import static org.eclipse.che.api.core.model.workspace.WorkspaceStatus.STOPPED; import static org.testng.Assert.assertEquals; @@ -65,6 +64,8 @@ public class WorkspaceActivityDaoTest { private static final int COUNT = 3; + private static final long DEFAULT_RUN_TIMEOUT = 0L; + @Inject private WorkspaceActivityDao workspaceActivityDao; private AccountImpl[] accounts = new AccountImpl[COUNT]; @@ -95,6 +96,7 @@ public class WorkspaceActivityDaoTest { a.setExpiration(base * 1_000_000); activities[i] = a; + System.out.println("activity is " + a); } accountTckRepository.createAll(asList(accounts)); @@ -112,7 +114,7 @@ public class WorkspaceActivityDaoTest { @Test public void shouldFindExpirationsByTimestamp() throws Exception { List expected = asList(activities[0].getWorkspaceId(), activities[1].getWorkspaceId()); - List found = workspaceActivityDao.findExpired(2_500_000); + List found = workspaceActivityDao.findExpired(2_500_000, DEFAULT_RUN_TIMEOUT); assertEquals(found, expected); } @@ -123,7 +125,22 @@ public class WorkspaceActivityDaoTest { workspaceActivityDao.removeExpiration(activities[0].getWorkspaceId()); - List found = workspaceActivityDao.findExpired(2_500_000); + List found = workspaceActivityDao.findExpired(2_500_000, DEFAULT_RUN_TIMEOUT); + assertEquals(found, expected); + } + + @Test(dependsOnMethods = "shouldFindExpirationsByTimestamp") + public void shouldExpireWorkspaceThatExceedsRunTimeout() throws Exception { + List expected = singletonList(activities[0].getWorkspaceId()); + + // Need more accurate activities for this test + workspaceActivityDao.removeActivity("ws0"); + workspaceActivityDao.removeActivity("ws1"); + workspaceActivityDao.removeActivity("ws2"); + + activityTckRepository.createAll(createWorkspaceActivitiesWithStatuses()); + + List found = workspaceActivityDao.findExpired(8_000_000, 1_000_000); assertEquals(found, expected); } @@ -137,7 +154,7 @@ public class WorkspaceActivityDaoTest { workspaceActivityDao.setExpirationTime(activities[2].getWorkspaceId(), 1_750_000); - List found = workspaceActivityDao.findExpired(2_500_000); + List found = workspaceActivityDao.findExpired(2_500_000, DEFAULT_RUN_TIMEOUT); assertEquals(found, expected); } @@ -160,7 +177,7 @@ public class WorkspaceActivityDaoTest { // create new again workspaceActivityDao.setExpirationTime(activities[1].getWorkspaceId(), 1_250_000); - List found = workspaceActivityDao.findExpired(1_500_000); + List found = workspaceActivityDao.findExpired(1_500_000, DEFAULT_RUN_TIMEOUT); assertEquals(found, expected); } @@ -286,6 +303,45 @@ public class WorkspaceActivityDaoTest { .toArray(Object[][]::new); } + /** + * Helper function that creates workspaces that are in the RUNNING and STOPPED state for + * shouldExpireWorkspaceThatExceedsRunTimeout + * + * @return A list of WorkspaceActivity objects + */ + private List createWorkspaceActivitiesWithStatuses() { + WorkspaceActivity[] a = new WorkspaceActivity[3]; + a[0] = new WorkspaceActivity(); + a[0].setWorkspaceId("ws0"); + a[0].setStatus(WorkspaceStatus.RUNNING); + a[0].setCreated(1_000_000); + a[0].setLastStarting(1_000_000); + a[0].setLastRunning(1_000_100); + a[0].setLastStopped(0); + a[0].setLastStopping(0); + a[0].setExpiration(1_100_000L); + + a[1] = new WorkspaceActivity(); + a[1].setWorkspaceId("ws1"); + a[1].setStatus(WorkspaceStatus.RUNNING); + a[1].setCreated(7_000_000); + a[1].setLastStarting(7_000_000); + a[1].setLastRunning(7_100_000); + a[1].setLastStopped(0); + a[1].setLastStopping(0); + a[1].setExpiration(8_000_000L); + + a[2] = new WorkspaceActivity(); + a[2].setWorkspaceId("ws2"); + a[2].setStatus(WorkspaceStatus.STOPPED); + a[2].setCreated(1_000_200); + a[2].setLastStarting(1_000_200); + a[2].setLastRunning(1_000_300); + a[2].setLastStopped(1_000_400); + a[2].setLastStopping(1_000_350); + return asList(a); + } + private static WorkspaceConfigImpl createWorkspaceConfig(String name) { // Project Sources configuration final SourceStorageImpl source1 = new SourceStorageImpl(); diff --git a/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java b/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java index bb9faa46ee..a542b617fc 100644 --- a/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java +++ b/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java @@ -239,6 +239,11 @@ public class CascadeRemovalTest { bind(Long.class) .annotatedWith(Names.named("che.limits.workspace.idle.timeout")) .toInstance(100000L); + + bind(Long.class) + .annotatedWith(Names.named("che.limits.workspace.run.timeout")) + .toInstance(0L); + bind(UserManager.class); bind(AccountManager.class);