From 551ff2b0e14a6aef6bd526d129ce4f3ec8f94c17 Mon Sep 17 00:00:00 2001 From: Lukas Krejci Date: Wed, 9 Jan 2019 10:40:57 +0100 Subject: [PATCH] Add workspace failure count metric. Signed-off-by: Lukas Krejci --- wsmaster/che-core-api-metrics/pom.xml | 23 +++ .../metrics/WorkspaceActivityMeterBinder.java | 8 +- .../che/api/metrics/WorkspaceBinders.java | 45 ++++++ .../metrics/WorkspaceFailureMeterBinder.java | 85 +++++++++++ .../api/metrics/WsMasterMetricsModule.java | 1 + .../WorkspaceFailureMeterBinderTest.java | 143 ++++++++++++++++++ 6 files changed, 302 insertions(+), 3 deletions(-) create mode 100644 wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceBinders.java create mode 100644 wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceFailureMeterBinder.java create mode 100644 wsmaster/che-core-api-metrics/src/test/java/org/eclipse/che/api/metrics/WorkspaceFailureMeterBinderTest.java diff --git a/wsmaster/che-core-api-metrics/pom.xml b/wsmaster/che-core-api-metrics/pom.xml index b109570b41..6922e33890 100644 --- a/wsmaster/che-core-api-metrics/pom.xml +++ b/wsmaster/che-core-api-metrics/pom.xml @@ -39,6 +39,10 @@ org.eclipse.che.core che-core-api-core + + org.eclipse.che.core + che-core-api-dto + org.eclipse.che.core che-core-api-model @@ -47,5 +51,24 @@ org.eclipse.che.core che-core-api-workspace-activity + + org.eclipse.che.core + che-core-api-workspace-shared + + + org.mockito + mockito-core + test + + + org.mockito + mockito-testng + test + + + org.testng + testng + test + diff --git a/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceActivityMeterBinder.java b/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceActivityMeterBinder.java index e09dbea426..f94ddb465a 100644 --- a/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceActivityMeterBinder.java +++ b/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceActivityMeterBinder.java @@ -11,6 +11,9 @@ */ package org.eclipse.che.api.metrics; +import static org.eclipse.che.api.metrics.WorkspaceBinders.withStandardTags; +import static org.eclipse.che.api.metrics.WorkspaceBinders.workspaceMetric; + import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.binder.MeterBinder; @@ -34,9 +37,8 @@ public class WorkspaceActivityMeterBinder implements MeterBinder { @Override public void bindTo(MeterRegistry registry) { for (WorkspaceStatus s : WorkspaceStatus.values()) { - Gauge.builder("che.workspace.status", () -> count(s)) - .tag("status", s.name()) - .tag("area", "workspace") + Gauge.builder(workspaceMetric("status"), () -> count(s)) + .tags(withStandardTags("status", s.name())) .description("The number of workspaces in a given status") .register(registry); } diff --git a/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceBinders.java b/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceBinders.java new file mode 100644 index 0000000000..3499b7b4df --- /dev/null +++ b/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceBinders.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.api.metrics; + +import java.util.Arrays; + +/** Utility methods for workspace meter binders. */ +final class WorkspaceBinders { + + private static final String METRIC_NAME_PREFIX = "che.workspace."; + private static final String[] STANDARD_TAGS = new String[] {"area", "workspace"}; + + private WorkspaceBinders() {} + + /** Produces a name for the workspace metric with the standard prefix. */ + static String workspaceMetric(String name) { + return METRIC_NAME_PREFIX + name; + } + + /** + * Produces a list of tags to add to the metric. The returned array always contains standard tags + * common to all workspace metrics. + * + * @param tags the additional tags to add in addition to the standard tags + * @return an array representing the tags + */ + static String[] withStandardTags(String... tags) { + if (tags.length == 0) { + return STANDARD_TAGS; + } + + String[] ret = Arrays.copyOf(STANDARD_TAGS, STANDARD_TAGS.length + tags.length); + System.arraycopy(tags, 0, ret, STANDARD_TAGS.length, tags.length); + return ret; + } +} diff --git a/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceFailureMeterBinder.java b/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceFailureMeterBinder.java new file mode 100644 index 0000000000..e41bbbceaa --- /dev/null +++ b/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceFailureMeterBinder.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.api.metrics; + +import static org.eclipse.che.api.metrics.WorkspaceBinders.withStandardTags; +import static org.eclipse.che.api.metrics.WorkspaceBinders.workspaceMetric; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.binder.MeterBinder; +import org.eclipse.che.api.core.model.workspace.WorkspaceStatus; +import org.eclipse.che.api.core.notification.EventService; +import org.eclipse.che.api.workspace.shared.dto.event.WorkspaceStatusEvent; + +/** + * Counts errors in workspaces while in different statuses. I.e. the errors while starting, running + * or stopping are counted separately. The counter IDs only differ in the "while" tag which + * specifies the workspace status in which the failure occurred. + */ +@Singleton +public class WorkspaceFailureMeterBinder implements MeterBinder { + private final EventService eventService; + + private Counter startingStoppedFailureCounter; + private Counter stoppingStoppedFailureCounter; + private Counter runningStoppedFailureCounter; + + @Inject + public WorkspaceFailureMeterBinder(EventService eventService) { + this.eventService = eventService; + } + + @Override + public void bindTo(MeterRegistry registry) { + startingStoppedFailureCounter = bindFailureFrom(WorkspaceStatus.STARTING, registry); + runningStoppedFailureCounter = bindFailureFrom(WorkspaceStatus.RUNNING, registry); + stoppingStoppedFailureCounter = bindFailureFrom(WorkspaceStatus.STOPPING, registry); + + // only subscribe to the event once we have the counters ready + eventService.subscribe( + event -> { + if (event.getError() == null || event.getStatus() != WorkspaceStatus.STOPPED) { + return; + } + + Counter counter; + switch (event.getPrevStatus()) { + case STARTING: + counter = startingStoppedFailureCounter; + break; + case RUNNING: + counter = runningStoppedFailureCounter; + break; + case STOPPING: + counter = stoppingStoppedFailureCounter; + break; + default: + return; + } + + counter.increment(); + }, + WorkspaceStatusEvent.class); + } + + private Counter bindFailureFrom(WorkspaceStatus previousState, MeterRegistry registry) { + // there's apparently a convention to suffix the counters with "_total" (which is what the name + // will end up looking like). + return Counter.builder(workspaceMetric("failure.total")) + .tags(withStandardTags("while", previousState.name())) + .description("The count of failed workspaces") + .register(registry); + } +} diff --git a/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WsMasterMetricsModule.java b/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WsMasterMetricsModule.java index 77bf43b48d..502fea2bae 100644 --- a/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WsMasterMetricsModule.java +++ b/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WsMasterMetricsModule.java @@ -27,5 +27,6 @@ public class WsMasterMetricsModule extends AbstractModule { Multibinder.newSetBinder(binder(), MeterBinder.class); meterMultibinder.addBinding().to(WorkspaceActivityMeterBinder.class); + meterMultibinder.addBinding().to(WorkspaceFailureMeterBinder.class); } } diff --git a/wsmaster/che-core-api-metrics/src/test/java/org/eclipse/che/api/metrics/WorkspaceFailureMeterBinderTest.java b/wsmaster/che-core-api-metrics/src/test/java/org/eclipse/che/api/metrics/WorkspaceFailureMeterBinderTest.java new file mode 100644 index 0000000000..aa74513c6c --- /dev/null +++ b/wsmaster/che-core-api-metrics/src/test/java/org/eclipse/che/api/metrics/WorkspaceFailureMeterBinderTest.java @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.api.metrics; + +import static java.util.Arrays.asList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.testng.Assert.assertEquals; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import org.eclipse.che.api.core.model.workspace.WorkspaceStatus; +import org.eclipse.che.api.core.notification.EventService; +import org.eclipse.che.api.core.notification.EventSubscriber; +import org.eclipse.che.api.workspace.shared.dto.event.WorkspaceStatusEvent; +import org.eclipse.che.dto.server.DtoFactory; +import org.mockito.ArgumentCaptor; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class WorkspaceFailureMeterBinderTest { + + private Collection failureCounters; + private EventSubscriber events; + + @BeforeMethod + public void setup() { + MeterRegistry registry = new SimpleMeterRegistry(); + + EventService eventService = mock(EventService.class); + + WorkspaceFailureMeterBinder meterBinder = new WorkspaceFailureMeterBinder(eventService); + + meterBinder.bindTo(registry); + + @SuppressWarnings("unchecked") + ArgumentCaptor> statusChangeEventCaptor = + ArgumentCaptor.forClass(EventSubscriber.class); + + failureCounters = registry.find("che.workspace.failure.total").counters(); + + verify(eventService) + .subscribe(statusChangeEventCaptor.capture(), eq(WorkspaceStatusEvent.class)); + + events = statusChangeEventCaptor.getValue(); + } + + @Test(dataProvider = "failureWhileInStatus") + public void shouldCollectFailureCountsPerStatus(WorkspaceStatus failureStatus) { + events.onEvent( + DtoFactory.newDto(WorkspaceStatusEvent.class) + .withPrevStatus(failureStatus) + .withStatus(WorkspaceStatus.STOPPED) + .withError("D'oh!") + .withWorkspaceId("1")); + + List restOfCounters = new ArrayList<>(failureCounters); + + Counter counter = + failureCounters + .stream() + .filter(c -> failureStatus.name().equals(c.getId().getTag("while"))) + .findAny() + .orElseThrow( + () -> + new AssertionError( + "Could not find a counter for failure status " + failureStatus)); + + restOfCounters.remove(counter); + + assertEquals(counter.count(), 1d); + restOfCounters.forEach(c -> assertEquals(c.count(), 0d)); + } + + @Test(dataProvider = "failureWhileInStatus") + public void shouldNotCollectFailureWhenNoErrorInEvent(WorkspaceStatus prevStatus) { + events.onEvent( + DtoFactory.newDto(WorkspaceStatusEvent.class) + .withPrevStatus(prevStatus) + .withStatus(WorkspaceStatus.STOPPED) + .withWorkspaceId("1")); + + failureCounters.forEach(c -> assertEquals(c.count(), 0d)); + } + + @Test(dataProvider = "allStatusTransitionsWithoutToStopped") + public void shouldNotCollectFailureWhenNotTransitioningToStopped( + WorkspaceStatus from, WorkspaceStatus to) { + // This really doesn't make much sense because the codebase always transitions the workspace + // to STOPPED on any kind of failure. This is just a precaution that a potential bug in the + // rest of the codebase doesn't affect the metric collection ;) + + events.onEvent( + DtoFactory.newDto(WorkspaceStatusEvent.class) + .withPrevStatus(from) + .withStatus(to) + .withError("D'oh!") + .withWorkspaceId("1")); + + failureCounters.forEach(c -> assertEquals(c.count(), 0d)); + } + + @DataProvider + public Object[][] failureWhileInStatus() { + return new Object[][] { + new Object[] {WorkspaceStatus.STARTING}, + new Object[] {WorkspaceStatus.RUNNING}, + new Object[] {WorkspaceStatus.STOPPING}, + }; + } + + @DataProvider + public Object[][] allStatusTransitionsWithoutToStopped() { + List> transitions = new ArrayList<>(9); + + for (WorkspaceStatus from : WorkspaceStatus.values()) { + for (WorkspaceStatus to : WorkspaceStatus.values()) { + if (from == to || to == WorkspaceStatus.STOPPED) { + continue; + } + + transitions.add(asList(from, to)); + } + } + + return transitions.stream().map(List::toArray).toArray(Object[][]::new); + } +}