Add filter to track 5xx errors for Prometheus (#12284)

core/che-core-metrics-core/pom.xml

@@ -73,6 +73,31 @@
             <artifactId>logback-classic</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.jayway.restassured</groupId>
+            <artifactId>rest-assured</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>javax.ws.rs</groupId>
+            <artifactId>javax.ws.rs-api</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.everrest</groupId>
+            <artifactId>everrest-assured</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-testng</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.testng</groupId>
             <artifactId>testng</artifactId>

core/che-core-metrics-core/src/main/java/org/eclipse/che/core/metrics/ApiResponseCounter.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2012-2018 Red Hat, Inc.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *   Red Hat, Inc. - initial API and implementation
+ */
+package org.eclipse.che.core.metrics;
+
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.binder.MeterBinder;
+import javax.inject.Singleton;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Metric binding for Che API responses, that are grouped by http status codes.
+ *
+ * @author Mykhailo Kuznietsov
+ */
+@Singleton
+public class ApiResponseCounter implements MeterBinder {
+  private static final Logger LOG = LoggerFactory.getLogger(ApiResponseCounter.class);
+
+  // package private access for visibility in tests
+  Counter informationalResponseCounter;
+  Counter successResponseCounter;
+  Counter redirectResponseCounter;
+  Counter clientErrorResponseCounter;
+  Counter serverErrorResponseCounter;
+
+  @Override
+  public void bindTo(MeterRegistry registry) {
+    informationalResponseCounter =
+        Counter.builder("che.server.api.response")
+            .description("Che Server Tomcat informational responses (1xx responses)")
+            .tag("code", "1xx")
+            .tag("area", "http")
+            .register(registry);
+    successResponseCounter =
+        Counter.builder("che.server.api.response")
+            .description("Che Server Tomcat success responses (2xx responses)")
+            .tag("code", "2xx")
+            .tag("area", "http")
+            .register(registry);
+    redirectResponseCounter =
+        Counter.builder("che.server.api.response")
+            .description("Che Server Tomcat redirect responses (3xx responses)")
+            .tag("code", "3xx")
+            .tag("area", "http")
+            .register(registry);
+    clientErrorResponseCounter =
+        Counter.builder("che.server.api.response")
+            .description("Che Server Tomcat client errors (4xx responses)")
+            .tag("code", "4xx")
+            .tag("area", "http")
+            .register(registry);
+    serverErrorResponseCounter =
+        Counter.builder("che.server.api.response")
+            .description("Che Server Tomcat server errors (5xx responses)")
+            .tag("code", "5xx")
+            .tag("area", "http")
+            .register(registry);
+  }
+
+  public void handleStatus(int status) {
+    status = status / 100;
+    switch (status) {
+      case 1:
+        informationalResponseCounter.increment();
+        break;
+      case 2:
+        successResponseCounter.increment();
+        break;
+      case 3:
+        redirectResponseCounter.increment();
+        break;
+      case 4:
+        clientErrorResponseCounter.increment();
+        break;
+      case 5:
+        serverErrorResponseCounter.increment();
+        break;
+      default:
+        // should not happen
+        LOG.warn("Unhandled HTTP status ", status);
+    }
+  }
+}

core/che-core-metrics-core/src/main/java/org/eclipse/che/core/metrics/ApiResponseMetricFilter.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2012-2018 Red Hat, Inc.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *   Red Hat, Inc. - initial API and implementation
+ */
+package org.eclipse.che.core.metrics;
+
+import java.io.IOException;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletResponse;
+
+/**
+ * Filter for tracking all HTTP requests through {@link ApiResponseCounter}
+ *
+ * @author Mykhailo Kuznietsov
+ */
+@Singleton
+public class ApiResponseMetricFilter implements Filter {
+
+  private ApiResponseCounter apiResponseCounter;
+
+  @Inject
+  public void setApiResponseCounter(ApiResponseCounter counter) {
+    this.apiResponseCounter = counter;
+  }
+
+  @Override
+  public void init(FilterConfig filterConfig) throws ServletException {}
+
+  @Override
+  public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain)
+      throws IOException, ServletException {
+    filterChain.doFilter(request, response);
+    if (response instanceof HttpServletResponse) {
+      apiResponseCounter.handleStatus(((HttpServletResponse) response).getStatus());
+    }
+  }
+
+  @Override
+  public void destroy() {}
+}

core/che-core-metrics-core/src/main/java/org/eclipse/che/core/metrics/MetricsModule.java

@@ -48,5 +48,6 @@ public class MetricsModule extends AbstractModule {
     meterMultibinder.addBinding().to(ProcessorMetrics.class);
     meterMultibinder.addBinding().to(UptimeMetrics.class);
     meterMultibinder.addBinding().to(FileStoresMeterBinder.class);
+    meterMultibinder.addBinding().to(ApiResponseCounter.class);
   }
 }

core/che-core-metrics-core/src/main/java/org/eclipse/che/core/metrics/MetricsServletModule.java

@@ -37,6 +37,7 @@ public class MetricsServletModule extends ServletModule {
     meterMultibinder.addBinding().toProvider(TomcatMetricsProvider.class);
 
     bind(Manager.class).toInstance(getManager(getServletContext()));
+    filter("/*").through(ApiResponseMetricFilter.class);
   }
 
   private Manager getManager(ServletContext servletContext) {

core/che-core-metrics-core/src/test/java/org/eclipse/che/core/metrics/ApiResponseCounterTest.java

@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2012-2018 Red Hat, Inc.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *   Red Hat, Inc. - initial API and implementation
+ */
+package org.eclipse.che.core.metrics;
+
+import static org.testng.Assert.assertEquals;
+
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+/**
+ * Test for {@link ApiResponseCounter} functionality
+ *
+ * @author Mykhailo Kuznietsov
+ */
+public class ApiResponseCounterTest {
+  private ApiResponseCounter apiResponseCounter;
+  private MeterRegistry registry;
+
+  @BeforeMethod
+  public void setup() {
+    registry = new SimpleMeterRegistry();
+
+    apiResponseCounter = new ApiResponseCounter();
+    apiResponseCounter.bindTo(registry);
+  }
+
+  @Test(dataProvider = "information")
+  public void shouldCount1xxResponses(int status) {
+    apiResponseCounter.handleStatus(status);
+
+    assertEquals(apiResponseCounter.informationalResponseCounter.count(), 1.0);
+    assertEquals(apiResponseCounter.successResponseCounter.count(), 0.0);
+    assertEquals(apiResponseCounter.redirectResponseCounter.count(), 0.0);
+    assertEquals(apiResponseCounter.clientErrorResponseCounter.count(), 0.0);
+    assertEquals(apiResponseCounter.serverErrorResponseCounter.count(), 0.0);
+  }
+
+  @Test(dataProvider = "success")
+  public void shouldCount2xxResponses(int status) {
+    apiResponseCounter.handleStatus(status);
+
+    assertEquals(apiResponseCounter.informationalResponseCounter.count(), 0.0);
+    assertEquals(apiResponseCounter.successResponseCounter.count(), 1.0);
+    assertEquals(apiResponseCounter.redirectResponseCounter.count(), 0.0);
+    assertEquals(apiResponseCounter.clientErrorResponseCounter.count(), 0.0);
+    assertEquals(apiResponseCounter.serverErrorResponseCounter.count(), 0.0);
+  }
+
+  @Test(dataProvider = "redirect")
+  public void shouldCount3xxResponses(int status) {
+    apiResponseCounter.handleStatus(status);
+
+    assertEquals(apiResponseCounter.informationalResponseCounter.count(), 0.0);
+    assertEquals(apiResponseCounter.successResponseCounter.count(), 0.0);
+    assertEquals(apiResponseCounter.redirectResponseCounter.count(), 1.0);
+    assertEquals(apiResponseCounter.clientErrorResponseCounter.count(), .0);
+    assertEquals(apiResponseCounter.serverErrorResponseCounter.count(), 0.0);
+  }
+
+  @Test(dataProvider = "clientError")
+  public void shouldCount4xxResponses(int status) {
+    apiResponseCounter.handleStatus(status);
+
+    assertEquals(apiResponseCounter.informationalResponseCounter.count(), 0.0);
+    assertEquals(apiResponseCounter.successResponseCounter.count(), 0.0);
+    assertEquals(apiResponseCounter.redirectResponseCounter.count(), 0.0);
+    assertEquals(apiResponseCounter.clientErrorResponseCounter.count(), 1.0);
+    assertEquals(apiResponseCounter.serverErrorResponseCounter.count(), 0.0);
+  }
+
+  @Test(dataProvider = "serverError")
+  public void shouldCount5xxResponses(int status) {
+    apiResponseCounter.handleStatus(status);
+
+    assertEquals(apiResponseCounter.informationalResponseCounter.count(), 0.0);
+    assertEquals(apiResponseCounter.successResponseCounter.count(), 0.0);
+    assertEquals(apiResponseCounter.redirectResponseCounter.count(), 0.0);
+    assertEquals(apiResponseCounter.clientErrorResponseCounter.count(), 0.0);
+    assertEquals(apiResponseCounter.serverErrorResponseCounter.count(), 1.0);
+  }
+
+  @DataProvider(name = "information")
+  public Object[][] information() {
+    return new Object[][] {{100}, {101}};
+  }
+
+  @DataProvider(name = "success")
+  public Object[][] success() {
+    return new Object[][] {{200}, {201}, {202}, {203}, {204}};
+  }
+
+  @DataProvider(name = "redirect")
+  public Object[][] redirect() {
+    return new Object[][] {{300}, {301}, {302}, {303}, {304}};
+  }
+
+  @DataProvider(name = "clientError")
+  public Object[][] clientError() {
+    return new Object[][] {{400}, {401}, {402}, {403}, {404}, {405}};
+  }
+
+  @DataProvider(name = "serverError")
+  public Object[][] serverError() {
+    return new Object[][] {{500}, {501}, {502}, {503}, {504}};
+  }
+}

core/che-core-metrics-core/src/test/java/org/eclipse/che/core/metrics/ApiResponseMetricFilterTest.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2012-2018 Red Hat, Inc.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *   Red Hat, Inc. - initial API and implementation
+ */
+package org.eclipse.che.core.metrics;
+
+import static com.jayway.restassured.RestAssured.given;
+import static org.everrest.assured.JettyHttpServer.ADMIN_USER_NAME;
+import static org.everrest.assured.JettyHttpServer.ADMIN_USER_PASSWORD;
+import static org.everrest.assured.JettyHttpServer.SECURE_PATH;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.verify;
+
+import org.everrest.assured.EverrestJetty;
+import org.mockito.Mock;
+import org.mockito.testng.MockitoTestNGListener;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Listeners;
+import org.testng.annotations.Test;
+
+/**
+ * Test for {@link ApiResponseMetricFilter} functionality
+ *
+ * @author Mykhailo Kuznietsov
+ */
+@Listeners({
+  MockitoTestNGListener.class,
+  EverrestJetty.class,
+})
+public class ApiResponseMetricFilterTest {
+
+  @Mock private ApiResponseCounter apiResponseCounter;
+
+  private ApiResponseMetricFilter filter;
+
+  @BeforeMethod
+  public void setUp() {
+    filter = new ApiResponseMetricFilter();
+    filter.setApiResponseCounter(apiResponseCounter);
+  }
+
+  @Test
+  public void shouldHandleStatusOnHttpRequest() {
+    // requesting a non existing resource, so 404 is expected
+    int status = 404;
+
+    given()
+        .auth()
+        .basic(ADMIN_USER_NAME, ADMIN_USER_PASSWORD)
+        .when()
+        .get(SECURE_PATH + "/service")
+        .then()
+        .statusCode(status);
+
+    verify(apiResponseCounter).handleStatus(eq(status));
+  }
+}