seg_yinzy
/
che-server
mirror of https://github.com/eclipse-che/che-server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add workspace failure count metric. 

Signed-off-by: Lukas Krejci <lkrejci@redhat.com>
6.19.x
Lukas Krejci 2019-01-09 10:40:57 +01:00
parent 43da30e0b6
commit 551ff2b0e1
6 changed files with 302 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
wsmaster/che-core-api-metrics/pom.xml
View File

@ -39,6 +39,10 @@
<groupId>org.eclipse.che.core</groupId>
<artifactId>che-core-api-core</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.che.core</groupId>
<artifactId>che-core-api-dto</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.che.core</groupId>
<artifactId>che-core-api-model</artifactId>
@ -47,5 +51,24 @@
<groupId>org.eclipse.che.core</groupId>
<artifactId>che-core-api-workspace-activity</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.che.core</groupId>
<artifactId>che-core-api-workspace-shared</artifactId>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-testng</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

8
wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceActivityMeterBinder.java
View File

@ -11,6 +11,9 @@
*/
package org.eclipse.che.api.metrics;
import static org.eclipse.che.api.metrics.WorkspaceBinders.withStandardTags;
import static org.eclipse.che.api.metrics.WorkspaceBinders.workspaceMetric;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.binder.MeterBinder;
@ -34,9 +37,8 @@ public class WorkspaceActivityMeterBinder implements MeterBinder {
@Override
public void bindTo(MeterRegistry registry) {
for (WorkspaceStatus s : WorkspaceStatus.values()) {
Gauge.builder("che.workspace.status", () -> count(s))
.tag("status", s.name())
.tag("area", "workspace")
Gauge.builder(workspaceMetric("status"), () -> count(s))
.tags(withStandardTags("status", s.name()))
.description("The number of workspaces in a given status")
.register(registry);
}

45
wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceBinders.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.api.metrics;
import java.util.Arrays;
/** Utility methods for workspace meter binders. */
final class WorkspaceBinders {
private static final String METRIC_NAME_PREFIX = "che.workspace.";
private static final String[] STANDARD_TAGS = new String[] {"area", "workspace"};
private WorkspaceBinders() {}
/** Produces a name for the workspace metric with the standard prefix. */
static String workspaceMetric(String name) {
return METRIC_NAME_PREFIX + name;
}
/**
* Produces a list of tags to add to the metric. The returned array always contains standard tags
* common to all workspace metrics.
*
* @param tags the additional tags to add in addition to the standard tags
* @return an array representing the tags
*/
static String[] withStandardTags(String... tags) {
if (tags.length == 0) {
return STANDARD_TAGS;
}
String[] ret = Arrays.copyOf(STANDARD_TAGS, STANDARD_TAGS.length + tags.length);
System.arraycopy(tags, 0, ret, STANDARD_TAGS.length, tags.length);
return ret;
}
}

85
wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceFailureMeterBinder.java Normal file
View File

@ -0,0 +1,85 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.api.metrics;
import static org.eclipse.che.api.metrics.WorkspaceBinders.withStandardTags;
import static org.eclipse.che.api.metrics.WorkspaceBinders.workspaceMetric;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.binder.MeterBinder;
import org.eclipse.che.api.core.model.workspace.WorkspaceStatus;
import org.eclipse.che.api.core.notification.EventService;
import org.eclipse.che.api.workspace.shared.dto.event.WorkspaceStatusEvent;
/**
* Counts errors in workspaces while in different statuses. I.e. the errors while starting, running
* or stopping are counted separately. The counter IDs only differ in the "while" tag which
* specifies the workspace status in which the failure occurred.
*/
@Singleton
public class WorkspaceFailureMeterBinder implements MeterBinder {
private final EventService eventService;
private Counter startingStoppedFailureCounter;
private Counter stoppingStoppedFailureCounter;
private Counter runningStoppedFailureCounter;
@Inject
public WorkspaceFailureMeterBinder(EventService eventService) {
this.eventService = eventService;
}
@Override
public void bindTo(MeterRegistry registry) {
startingStoppedFailureCounter = bindFailureFrom(WorkspaceStatus.STARTING, registry);
runningStoppedFailureCounter = bindFailureFrom(WorkspaceStatus.RUNNING, registry);
stoppingStoppedFailureCounter = bindFailureFrom(WorkspaceStatus.STOPPING, registry);
// only subscribe to the event once we have the counters ready
eventService.subscribe(
event -> {
if (event.getError() == null || event.getStatus() != WorkspaceStatus.STOPPED) {
return;
}
Counter counter;
switch (event.getPrevStatus()) {
case STARTING:
counter = startingStoppedFailureCounter;
break;
case RUNNING:
counter = runningStoppedFailureCounter;
break;
case STOPPING:
counter = stoppingStoppedFailureCounter;
break;
default:
return;
}
counter.increment();
},
WorkspaceStatusEvent.class);
}
private Counter bindFailureFrom(WorkspaceStatus previousState, MeterRegistry registry) {
// there's apparently a convention to suffix the counters with "_total" (which is what the name
// will end up looking like).
return Counter.builder(workspaceMetric("failure.total"))
.tags(withStandardTags("while", previousState.name()))
.description("The count of failed workspaces")
.register(registry);
}
}

1
wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WsMasterMetricsModule.java
View File

@ -27,5 +27,6 @@ public class WsMasterMetricsModule extends AbstractModule {
Multibinder.newSetBinder(binder(), MeterBinder.class);
meterMultibinder.addBinding().to(WorkspaceActivityMeterBinder.class);
meterMultibinder.addBinding().to(WorkspaceFailureMeterBinder.class);
}
}

143
wsmaster/che-core-api-metrics/src/test/java/org/eclipse/che/api/metrics/WorkspaceFailureMeterBinderTest.java Normal file
View File

@ -0,0 +1,143 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.api.metrics;
import static java.util.Arrays.asList;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.testng.Assert.assertEquals;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.eclipse.che.api.core.model.workspace.WorkspaceStatus;
import org.eclipse.che.api.core.notification.EventService;
import org.eclipse.che.api.core.notification.EventSubscriber;
import org.eclipse.che.api.workspace.shared.dto.event.WorkspaceStatusEvent;
import org.eclipse.che.dto.server.DtoFactory;
import org.mockito.ArgumentCaptor;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
public class WorkspaceFailureMeterBinderTest {
private Collection<Counter> failureCounters;
private EventSubscriber<WorkspaceStatusEvent> events;
@BeforeMethod
public void setup() {
MeterRegistry registry = new SimpleMeterRegistry();
EventService eventService = mock(EventService.class);
WorkspaceFailureMeterBinder meterBinder = new WorkspaceFailureMeterBinder(eventService);
meterBinder.bindTo(registry);
@SuppressWarnings("unchecked")
ArgumentCaptor<EventSubscriber<WorkspaceStatusEvent>> statusChangeEventCaptor =
ArgumentCaptor.forClass(EventSubscriber.class);
failureCounters = registry.find("che.workspace.failure.total").counters();
verify(eventService)
.subscribe(statusChangeEventCaptor.capture(), eq(WorkspaceStatusEvent.class));
events = statusChangeEventCaptor.getValue();
}
@Test(dataProvider = "failureWhileInStatus")
public void shouldCollectFailureCountsPerStatus(WorkspaceStatus failureStatus) {
events.onEvent(
DtoFactory.newDto(WorkspaceStatusEvent.class)
.withPrevStatus(failureStatus)
.withStatus(WorkspaceStatus.STOPPED)
.withError("D'oh!")
.withWorkspaceId("1"));
List<Counter> restOfCounters = new ArrayList<>(failureCounters);
Counter counter =
failureCounters
.stream()
.filter(c -> failureStatus.name().equals(c.getId().getTag("while")))
.findAny()
.orElseThrow(
() ->
new AssertionError(
"Could not find a counter for failure status " + failureStatus));
restOfCounters.remove(counter);
assertEquals(counter.count(), 1d);
restOfCounters.forEach(c -> assertEquals(c.count(), 0d));
}
@Test(dataProvider = "failureWhileInStatus")
public void shouldNotCollectFailureWhenNoErrorInEvent(WorkspaceStatus prevStatus) {
events.onEvent(
DtoFactory.newDto(WorkspaceStatusEvent.class)
.withPrevStatus(prevStatus)
.withStatus(WorkspaceStatus.STOPPED)
.withWorkspaceId("1"));
failureCounters.forEach(c -> assertEquals(c.count(), 0d));
}
@Test(dataProvider = "allStatusTransitionsWithoutToStopped")
public void shouldNotCollectFailureWhenNotTransitioningToStopped(
WorkspaceStatus from, WorkspaceStatus to) {
// This really doesn't make much sense because the codebase always transitions the workspace
// to STOPPED on any kind of failure. This is just a precaution that a potential bug in the
// rest of the codebase doesn't affect the metric collection ;)
events.onEvent(
DtoFactory.newDto(WorkspaceStatusEvent.class)
.withPrevStatus(from)
.withStatus(to)
.withError("D'oh!")
.withWorkspaceId("1"));
failureCounters.forEach(c -> assertEquals(c.count(), 0d));
}
@DataProvider
public Object[][] failureWhileInStatus() {
return new Object[][] {
new Object[] {WorkspaceStatus.STARTING},
new Object[] {WorkspaceStatus.RUNNING},
new Object[] {WorkspaceStatus.STOPPING},
};
}
@DataProvider
public Object[][] allStatusTransitionsWithoutToStopped() {
List<List<WorkspaceStatus>> transitions = new ArrayList<>(9);
for (WorkspaceStatus from : WorkspaceStatus.values()) {
for (WorkspaceStatus to : WorkspaceStatus.values()) {
if (from == to || to == WorkspaceStatus.STOPPED) {
continue;
}
transitions.add(asList(from, to));
}
}
return transitions.stream().map(List::toArray).toArray(Object[][]::new);
}
}