seg_yinzy
/
che-server
mirror of https://github.com/eclipse-che/che-server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #17323 from tomgeorge/che-17231 

Add workspace maximum time
7.20.x
Tom George 2020-07-14 10:20:33 -05:00 committed by GitHub
parent 77f11644a4 c32cb855b5
commit cfb2cf4a9b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 199 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/multiuser.properties
View File

@ -40,6 +40,12 @@ che.limits.workspace.env.ram=16gb
# counts toward idleness.
che.limits.workspace.idle.timeout=1800000
# The length of time in milliseconds that a workspace will run, regardless of activity, before
# the system will suspend it. Set this property if you want to automatically stop
# workspaces after a period of time. The default is zero, meaning that there is no
# run timeout.
che.limits.workspace.run.timeout=0
### Users workspace limits
# The total amount of RAM that a single user is allowed to allocate to running

7
multiuser/api/che-multiuser-api-workspace-activity/src/main/java/org/eclipse/che/multiuser/api/workspace/activity/MultiUserWorkspaceActivityManager.java
View File

@ -46,6 +46,7 @@ public class MultiUserWorkspaceActivityManager extends WorkspaceActivityManager
private final AccountManager accountManager;
private final ResourceManager resourceManager;
private final long defaultTimeout;
private final long runTimeout;
@Inject
public MultiUserWorkspaceActivityManager(
@ -54,11 +55,13 @@ public class MultiUserWorkspaceActivityManager extends WorkspaceActivityManager
EventService eventService,
AccountManager accountManager,
ResourceManager resourceManager,
@Named("che.limits.workspace.idle.timeout") long defaultTimeout) {
super(workspaceManager, activityDao, eventService, defaultTimeout);
@Named("che.limits.workspace.idle.timeout") long defaultTimeout,
@Named("che.limits.workspace.run.timeout") long runTimeout) {
super(workspaceManager, activityDao, eventService, defaultTimeout, runTimeout);
this.accountManager = accountManager;
this.resourceManager = resourceManager;
this.defaultTimeout = defaultTimeout;
this.runTimeout = runTimeout;
}
@Override

4
multiuser/api/che-multiuser-api-workspace-activity/src/test/java/org/eclipse/che/multiuser/api/workspace/activity/MultiUserWorkspaceActivityManagerTest.java
View File

@ -35,6 +35,7 @@ import org.testng.annotations.Test;
public class MultiUserWorkspaceActivityManagerTest {
private static final long DEFAULT_TIMEOUT = 60_000L; // 1 minute
private static final long USER_LIMIT_TIMEOUT = 120_000L; // 2 minutes
private static final long DEFAULT_RUN_TIMEOUT = 0; // No default run timeout
@Mock private AccountManager accountManager;
@Mock private ResourceManager resourceManager;
@ -58,7 +59,8 @@ public class MultiUserWorkspaceActivityManagerTest {
eventService,
accountManager,
resourceManager,
DEFAULT_TIMEOUT);
DEFAULT_TIMEOUT,
DEFAULT_RUN_TIMEOUT);
when(account.getId()).thenReturn("account123");
when(accountManager.getByName(anyString())).thenReturn(account);

27
wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/InmemoryWorkspaceActivityDao.java
View File

@ -42,8 +42,33 @@ public class InmemoryWorkspaceActivityDao implements WorkspaceActivityDao {
findActivity(workspaceId).setExpiration(null);
}
/**
* Finds any workspaces that have expired.
*
* <p>A workspace is expired when the expiration value is less than the current time or when the
* difference between the current time and the last running time is greater than the run timeout
*
* @param timestamp expiration time
* @param runTimeout time after which the workspace will be stopped regardless of activity
* @return
*/
@Override
public List<String> findExpired(long timestamp) {
public List<String> findExpiredRunTimeout(long timestamp, long runTimeout) {
return workspaceActivities
.values()
.stream()
.filter(
a ->
(a.getExpiration() != null && a.getExpiration() < timestamp)
|| (runTimeout > 0
&& a.getStatus() == WorkspaceStatus.RUNNING
&& timestamp - a.getLastRunning() > runTimeout))
.map(WorkspaceActivity::getWorkspaceId)
.collect(toList());
}
@Override
public List<String> findExpiredIdle(long timestamp) {
return workspaceActivities
.values()
.stream()

32
wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/JpaWorkspaceActivityDao.java
View File

@ -50,17 +50,43 @@ public class JpaWorkspaceActivityDao implements WorkspaceActivityDao {
doUpdateOptionally(workspaceId, a -> a.setExpiration(null));
}
/**
* Finds any workspaces that have expired.
*
* <p>A workspace is expired when the expiration value is less than the current time or when the
* difference between the current time and the last running time is greater than the run timeout
*
* @param timestamp expiration time
* @param runTimeout time after which the workspace will be stopped regardless of activity
* @return
*/
@Override
@Transactional(rollbackOn = ServerException.class)
public List<String> findExpired(long timestamp) throws ServerException {
public List<String> findExpiredRunTimeout(long timestamp, long runTimeout)
throws ServerException {
try {
return managerProvider
.get()
.createNamedQuery("WorkspaceActivity.getExpired", WorkspaceActivity.class)
.createNamedQuery("WorkspaceActivity.getExpiredRunTimeout", String.class)
.setParameter("timeDifference", timestamp - runTimeout)
.getResultList()
.stream()
.collect(Collectors.toList());
} catch (RuntimeException x) {
throw new ServerException(x.getLocalizedMessage(), x);
}
}
@Override
@Transactional(rollbackOn = ServerException.class)
public List<String> findExpiredIdle(long timestamp) throws ServerException {
try {
return managerProvider
.get()
.createNamedQuery("WorkspaceActivity.getExpiredIdle", String.class)
.setParameter("expiration", timestamp)
.getResultList()
.stream()
.map(WorkspaceActivity::getWorkspaceId)
.collect(Collectors.toList());
} catch (RuntimeException x) {
throw new ServerException(x.getLocalizedMessage(), x);

10
wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivity.java
View File

@ -26,8 +26,14 @@ import org.eclipse.che.api.core.model.workspace.WorkspaceStatus;
@Table(name = "che_workspace_activity")
@NamedQueries({
@NamedQuery(
name = "WorkspaceActivity.getExpired",
query = "SELECT a FROM WorkspaceActivity a WHERE a.expiration < :expiration"),
name = "WorkspaceActivity.getExpiredIdle",
query = "SELECT a.workspaceId FROM WorkspaceActivity a WHERE a.expiration < :expiration"),
@NamedQuery(
name = "WorkspaceActivity.getExpiredRunTimeout",
query =
"SELECT a.workspaceId FROM WorkspaceActivity a WHERE "
+ "(a.status = org.eclipse.che.api.core.model.workspace.WorkspaceStatus.RUNNING AND "
+ "a.lastRunning < :timeDifference)"),
@NamedQuery(
name = "WorkspaceActivity.getStoppedSince",
query =

21
wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityChecker.java
View File

@ -46,6 +46,9 @@ public class WorkspaceActivityChecker {
private static final Logger LOG = LoggerFactory.getLogger(WorkspaceActivityChecker.class);
private static final String ACTIVITY_CHECKER = "activity-checker";
private final String WORKSPACE_IDLE_TIMEOUT_EXCEEDED = "Workspace idle timeout exceeded";
private final String WORKSPACE_RUN_TIMEOUT_EXCEEDED = "Workspace run timeout exceeded";
private final WorkspaceActivityDao activityDao;
private final WorkspaceManager workspaceManager;
private final WorkspaceRuntimes workspaceRuntimes;
@ -101,19 +104,29 @@ public class WorkspaceActivityChecker {
private void stopAllExpired() {
try {
activityDao.findExpired(clock.millis()).forEach(this::stopExpiredQuietly);
activityDao
.findExpiredIdle(clock.millis())
.forEach(wsId -> stopExpiredQuietly(wsId, WORKSPACE_IDLE_TIMEOUT_EXCEEDED));
if (workspaceActivityManager.getRunTimeout() > 0) {
activityDao
.findExpiredRunTimeout(clock.millis(), workspaceActivityManager.getRunTimeout())
.forEach(
wsId -> {
LOG.info("{} for workspace {}", WORKSPACE_RUN_TIMEOUT_EXCEEDED, wsId);
stopExpiredQuietly(wsId, WORKSPACE_RUN_TIMEOUT_EXCEEDED);
});
}
} catch (ServerException e) {
LOG.error("Failed to list all expired to perform stop. Cause: {}", e.getMessage(), e);
}
}
private void stopExpiredQuietly(String workspaceId) {
private void stopExpiredQuietly(String workspaceId, String reason) {
try {
Workspace workspace = workspaceManager.getWorkspace(workspaceId);
workspace.getAttributes().put(WORKSPACE_STOPPED_BY, ACTIVITY_CHECKER);
workspaceManager.updateWorkspace(workspaceId, workspace);
workspaceManager.stopWorkspace(
workspaceId, singletonMap(WORKSPACE_STOP_REASON, "Workspace idle timeout exceeded"));
workspaceManager.stopWorkspace(workspaceId, singletonMap(WORKSPACE_STOP_REASON, reason));
} catch (NotFoundException ignored) {
// workspace no longer exists, no need to do anything
} catch (ConflictException e) {

11
wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityDao.java
View File

@ -42,6 +42,15 @@ public interface WorkspaceActivityDao {
*/
void removeExpiration(String workspaceId) throws ServerException;
/**
* Finds workspaces which are passed given run timeout and must be stopped.
*
* @param runTimeout time after which the workspace will be stopped regardless of activity
* @return list of workspaces which expiration time is older than given timestamp
* @throws ServerException when operation failed
*/
List<String> findExpiredRunTimeout(long timestamp, long runTimeout) throws ServerException;
/**
* Finds workspaces which are passed given expiration time and must be stopped.
*
@ -49,7 +58,7 @@ public interface WorkspaceActivityDao {
* @return list of workspaces which expiration time is older than given timestamp
* @throws ServerException when operation failed
*/
List<String> findExpired(long timestamp) throws ServerException;
List<String> findExpiredIdle(long timestamp) throws ServerException;
/**
* Removes the activity record of the provided workspace.

19
wsmaster/che-core-api-workspace-activity/src/main/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityManager.java
View File

@ -54,6 +54,7 @@ public class WorkspaceActivityManager {
private static final Logger LOG = LoggerFactory.getLogger(WorkspaceActivityManager.class);
private final long defaultTimeout;
private final long runTimeout;
private final WorkspaceActivityDao activityDao;
private final EventService eventService;
private final EventSubscriber<WorkspaceStatusEvent> updateStatusChangedTimestampSubscriber;
@ -69,9 +70,16 @@ public class WorkspaceActivityManager {
WorkspaceManager workspaceManager,
WorkspaceActivityDao activityDao,
EventService eventService,
@Named("che.limits.workspace.idle.timeout") long timeout) {
@Named("che.limits.workspace.idle.timeout") long timeout,
@Named("che.limits.workspace.run.timeout") long runTimeout) {
this(workspaceManager, activityDao, eventService, timeout, Clock.systemDefaultZone());
this(
workspaceManager,
activityDao,
eventService,
timeout,
runTimeout,
Clock.systemDefaultZone());
}
@VisibleForTesting
@ -80,11 +88,13 @@ public class WorkspaceActivityManager {
WorkspaceActivityDao activityDao,
EventService eventService,
long timeout,
long runTimeout,
Clock clock) {
this.workspaceManager = workspaceManager;
this.eventService = eventService;
this.activityDao = activityDao;
this.defaultTimeout = timeout;
this.runTimeout = runTimeout;
this.clock = clock;
if (timeout > 0 && timeout < MINIMAL_TIMEOUT) {
LOG.warn(
@ -116,7 +126,6 @@ public class WorkspaceActivityManager {
activityDao.removeActivity(event.getWorkspace().getId());
}
};
this.updateStatusChangedTimestampSubscriber = new UpdateStatusChangedTimestampSubscriber();
}
@ -170,6 +179,10 @@ public class WorkspaceActivityManager {
return defaultTimeout;
}
protected long getRunTimeout() {
return runTimeout;
}
private class UpdateStatusChangedTimestampSubscriber
implements EventSubscriber<WorkspaceStatusEvent> {
@Override

10
wsmaster/che-core-api-workspace-activity/src/test/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityCheckerTest.java
View File

@ -55,6 +55,7 @@ import org.testng.annotations.Test;
@Listeners(value = MockitoTestNGListener.class)
public class WorkspaceActivityCheckerTest {
private static final long DEFAULT_TIMEOUT = 60_000L; // 1 minute
private static final long DEFAULT_RUN_TIMEOUT = 0; // No default run timeout
private ManualClock clock;
private WorkspaceActivityChecker checker;
@ -69,7 +70,12 @@ public class WorkspaceActivityCheckerTest {
WorkspaceActivityManager activityManager =
new WorkspaceActivityManager(
workspaceManager, workspaceActivityDao, eventService, DEFAULT_TIMEOUT, clock);
workspaceManager,
workspaceActivityDao,
eventService,
DEFAULT_TIMEOUT,
DEFAULT_RUN_TIMEOUT,
clock);
lenient()
.when(workspaceActivityDao.getAll(anyInt(), anyLong()))
@ -88,7 +94,7 @@ public class WorkspaceActivityCheckerTest {
@Test
public void shouldStopAllExpiredWorkspaces() throws Exception {
when(workspaceActivityDao.findExpired(anyLong())).thenReturn(asList("1", "2", "3"));
when(workspaceActivityDao.findExpiredIdle(anyLong())).thenReturn(asList("1", "2", "3"));
checker.expire();

7
wsmaster/che-core-api-workspace-activity/src/test/java/org/eclipse/che/api/workspace/activity/WorkspaceActivityManagerTest.java
View File

@ -49,6 +49,7 @@ import org.testng.annotations.Test;
public class WorkspaceActivityManagerTest {
private static final long DEFAULT_TIMEOUT = 60_000L; // 1 minute
private static final long DEFAULT_RUN_TIMEOUT = 0; // No run timeout
@Mock private WorkspaceManager workspaceManager;
@ -68,7 +69,11 @@ public class WorkspaceActivityManagerTest {
private void setUp() throws Exception {
activityManager =
new WorkspaceActivityManager(
workspaceManager, workspaceActivityDao, eventService, DEFAULT_TIMEOUT);
workspaceManager,
workspaceActivityDao,
eventService,
DEFAULT_TIMEOUT,
DEFAULT_RUN_TIMEOUT);
lenient().when(account.getName()).thenReturn("accountName");
lenient().when(account.getId()).thenReturn("account123");

64
wsmaster/che-core-api-workspace-activity/src/test/java/org/eclipse/che/api/workspace/activity/spi/tck/WorkspaceActivityDaoTest.java
View File

@ -65,6 +65,8 @@ public class WorkspaceActivityDaoTest {
private static final int COUNT = 3;
private static final long DEFAULT_RUN_TIMEOUT = 0L;
@Inject private WorkspaceActivityDao workspaceActivityDao;
private AccountImpl[] accounts = new AccountImpl[COUNT];
@ -112,7 +114,7 @@ public class WorkspaceActivityDaoTest {
@Test
public void shouldFindExpirationsByTimestamp() throws Exception {
List<String> expected = asList(activities[0].getWorkspaceId(), activities[1].getWorkspaceId());
List<String> found = workspaceActivityDao.findExpired(2_500_000);
List<String> found = workspaceActivityDao.findExpiredIdle(2_500_000);
assertEquals(found, expected);
}
@ -123,7 +125,22 @@ public class WorkspaceActivityDaoTest {
workspaceActivityDao.removeExpiration(activities[0].getWorkspaceId());
List<String> found = workspaceActivityDao.findExpired(2_500_000);
List<String> found = workspaceActivityDao.findExpiredIdle(2_500_000);
assertEquals(found, expected);
}
@Test(dependsOnMethods = "shouldFindExpirationsByTimestamp")
public void shouldExpireWorkspaceThatExceedsRunTimeout() throws Exception {
List<String> expected = singletonList(activities[0].getWorkspaceId());
// Need more accurate activities for this test
workspaceActivityDao.removeActivity("ws0");
workspaceActivityDao.removeActivity("ws1");
workspaceActivityDao.removeActivity("ws2");
activityTckRepository.createAll(createWorkspaceActivitiesWithStatuses());
List<String> found = workspaceActivityDao.findExpiredIdle(8_000_000);
assertEquals(found, expected);
}
@ -137,7 +154,7 @@ public class WorkspaceActivityDaoTest {
workspaceActivityDao.setExpirationTime(activities[2].getWorkspaceId(), 1_750_000);
List<String> found = workspaceActivityDao.findExpired(2_500_000);
List<String> found = workspaceActivityDao.findExpiredIdle(2_500_000);
assertEquals(found, expected);
}
@ -160,7 +177,7 @@ public class WorkspaceActivityDaoTest {
// create new again
workspaceActivityDao.setExpirationTime(activities[1].getWorkspaceId(), 1_250_000);
List<String> found = workspaceActivityDao.findExpired(1_500_000);
List<String> found = workspaceActivityDao.findExpiredIdle(1_500_000);
assertEquals(found, expected);
}
@ -286,6 +303,45 @@ public class WorkspaceActivityDaoTest {
.toArray(Object[][]::new);
}
/**
* Helper function that creates workspaces that are in the RUNNING and STOPPED state for
* shouldExpireWorkspaceThatExceedsRunTimeout
*
* @return A list of WorkspaceActivity objects
*/
private List<WorkspaceActivity> createWorkspaceActivitiesWithStatuses() {
WorkspaceActivity[] a = new WorkspaceActivity[3];
a[0] = new WorkspaceActivity();
a[0].setWorkspaceId("ws0");
a[0].setStatus(WorkspaceStatus.RUNNING);
a[0].setCreated(1_000_000);
a[0].setLastStarting(1_000_000);
a[0].setLastRunning(1_000_100);
a[0].setLastStopped(0);
a[0].setLastStopping(0);
a[0].setExpiration(1_100_000L);
a[1] = new WorkspaceActivity();
a[1].setWorkspaceId("ws1");
a[1].setStatus(WorkspaceStatus.RUNNING);
a[1].setCreated(7_000_000);
a[1].setLastStarting(7_000_000);
a[1].setLastRunning(7_100_000);
a[1].setLastStopped(0);
a[1].setLastStopping(0);
a[1].setExpiration(8_000_000L);
a[2] = new WorkspaceActivity();
a[2].setWorkspaceId("ws2");
a[2].setStatus(WorkspaceStatus.STOPPED);
a[2].setCreated(1_000_200);
a[2].setLastStarting(1_000_200);
a[2].setLastRunning(1_000_300);
a[2].setLastStopped(1_000_400);
a[2].setLastStopping(1_000_350);
return asList(a);
}
private static WorkspaceConfigImpl createWorkspaceConfig(String name) {
// Project Sources configuration
final SourceStorageImpl source1 = new SourceStorageImpl();

5
wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java
View File

@ -239,6 +239,11 @@ public class CascadeRemovalTest {
bind(Long.class)
.annotatedWith(Names.named("che.limits.workspace.idle.timeout"))
.toInstance(100000L);
bind(Long.class)
.annotatedWith(Names.named("che.limits.workspace.run.timeout"))
.toInstance(0L);
bind(UserManager.class);
bind(AccountManager.class);