Metrics POC (#11990)

With CHE_METRICS_ENABLED env variable enable Prometheus metrics endpoint
6.19.x
Sergii Kabashniuk 2018-11-27 23:47:24 +02:00 committed by GitHub
parent c91cb54fb8
commit da3eefe237
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 577 additions and 0 deletions

View File

@ -229,6 +229,10 @@
<groupId>org.eclipse.che.core</groupId>
<artifactId>che-core-ide-stacks</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.che.core</groupId>
<artifactId>che-core-metrics-core</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.che.core</groupId>
<artifactId>che-core-sql-schema</artifactId>

View File

@ -276,6 +276,9 @@ public class WsMasterModule extends AbstractModule {
if (Boolean.valueOf(System.getenv("CHE_TRACING_ENABLED"))) {
install(new org.eclipse.che.core.tracing.TracingModule());
}
if (Boolean.valueOf(System.getenv("CHE_METRICS_ENABLED"))) {
install(new org.eclipse.che.core.metrics.MetricsModule());
}
}
private void configureSingleUserMode(Map<String, String> persistenceProperties) {

View File

@ -61,6 +61,10 @@ public class WsMasterServletModule extends ServletModule {
} else {
configureSingleUserMode();
}
if (Boolean.valueOf(System.getenv("CHE_METRICS_ENABLED"))) {
install(new org.eclipse.che.core.metrics.MetricsServletModule());
}
}
private void configureSingleUserMode() {

View File

@ -549,3 +549,7 @@ che.server.secure_exposer.jwtproxy.memory_limit=128mb
# Maximum size of the json processing pool
# in case if pool size would be exceeded message execution will be rejected
che.core.jsonrpc.processor_max_pool_size=100
## Port the the http server endpoint that would be exposed with Prometheus metrics
che.metrics.port=8087

View File

@ -0,0 +1,104 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (c) 2012-2018 Red Hat, Inc.
This program and the accompanying materials are made
available under the terms of the Eclipse Public License 2.0
which is available at https://www.eclipse.org/legal/epl-2.0/
SPDX-License-Identifier: EPL-2.0
Contributors:
Red Hat, Inc. - initial API and implementation
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>che-core-parent</artifactId>
<groupId>org.eclipse.che.core</groupId>
<version>6.15.0-SNAPSHOT</version>
</parent>
<artifactId>che-core-metrics-core</artifactId>
<name>Che Core :: Metrics :: Core</name>
<dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-multibindings</artifactId>
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-servlet</artifactId>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
</dependency>
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient</artifactId>
</dependency>
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_httpserver</artifactId>
</dependency>
<dependency>
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.tomcat</groupId>
<artifactId>tomcat-catalina</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>analyze</id>
<configuration>
<ignoredUsedUndeclaredDependencies>
<ignoredUsedUndeclaredDependency>org.apache.tomcat:tomcat-annotations-api</ignoredUsedUndeclaredDependency>
</ignoredUsedUndeclaredDependencies>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@ -0,0 +1,76 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.core.metrics;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tag;
import io.micrometer.core.instrument.Tags;
import io.micrometer.core.instrument.binder.MeterBinder;
import java.io.IOException;
import java.nio.file.FileStore;
import java.nio.file.FileSystems;
import java.util.function.ToDoubleFunction;
import javax.inject.Singleton;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** Bind disk usage metrics for for every {@link java.nio.file.FileStore}. */
@Singleton
public class FileStoresMeterBinder implements MeterBinder {
private static final Logger LOG = LoggerFactory.getLogger(FileStoresMeterBinder.class);
@Override
public void bindTo(MeterRegistry registry) {
for (FileStore fileStore : FileSystems.getDefault().getFileStores()) {
LOG.debug("Add gauge metric for {}", fileStore.name());
Iterable<Tag> tagsWithPath = Tags.concat(Tags.empty(), "path", fileStore.toString());
Gauge.builder("disk.free", fileStore, exceptionToNonWrapper(FileStore::getUnallocatedSpace))
.tags(tagsWithPath)
.description("Unallocated space for file store")
.baseUnit("bytes")
.strongReference(true)
.register(registry);
Gauge.builder("disk.total", fileStore, exceptionToNonWrapper(FileStore::getTotalSpace))
.tags(tagsWithPath)
.description("Total space for file store")
.baseUnit("bytes")
.strongReference(true)
.register(registry);
Gauge.builder("disk.usable", fileStore, exceptionToNonWrapper(FileStore::getUsableSpace))
.tags(tagsWithPath)
.description("Usable space for file store")
.baseUnit("bytes")
.strongReference(true)
.register(registry);
}
}
static <T> ToDoubleFunction<T> exceptionToNonWrapper(
ThrowingToDoubleFunction<T> throwingConsumer) {
return i -> {
try {
return throwingConsumer.applyAsDouble(i);
} catch (Exception ex) {
return Double.NaN;
}
};
}
@FunctionalInterface
interface ThrowingToDoubleFunction<T> {
double applyAsDouble(T t) throws IOException;
}
}

View File

@ -0,0 +1,33 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.core.metrics;
import io.micrometer.core.instrument.binder.MeterBinder;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import java.util.Set;
import javax.inject.Inject;
import javax.inject.Singleton;
/**
* Takes all {@link io.micrometer.core.instrument.binder.MeterBinder} from guice container, binded
* with {@link com.google.inject.multibindings.Multibinder}, and bind them to {@link
* io.micrometer.prometheus.PrometheusMeterRegistry} on PostConstruct.
*/
@Singleton
public class MetricsBinder {
@Inject
public void bindToRegistry(
PrometheusMeterRegistry meterRegistry, Set<MeterBinder> meterBinderList) {
meterBinderList.forEach(e -> e.bindTo(meterRegistry));
}
}

View File

@ -0,0 +1,52 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.core.metrics;
import com.google.common.annotations.Beta;
import com.google.inject.AbstractModule;
import com.google.inject.multibindings.Multibinder;
import io.micrometer.core.instrument.binder.MeterBinder;
import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics;
import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics;
import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics;
import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics;
import io.micrometer.core.instrument.binder.logging.LogbackMetrics;
import io.micrometer.core.instrument.binder.system.FileDescriptorMetrics;
import io.micrometer.core.instrument.binder.system.ProcessorMetrics;
import io.micrometer.core.instrument.binder.system.UptimeMetrics;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import io.prometheus.client.CollectorRegistry;
@Beta
public class MetricsModule extends AbstractModule {
@Override
protected void configure() {
bind(MetricsServer.class).asEagerSingleton();
bind(MetricsBinder.class).asEagerSingleton();
bind(CollectorRegistry.class).toInstance(CollectorRegistry.defaultRegistry);
bind(PrometheusMeterRegistry.class)
.toProvider(PrometheusMeterRegistryProvider.class)
.asEagerSingleton();
Multibinder<MeterBinder> meterMultibinder =
Multibinder.newSetBinder(binder(), MeterBinder.class);
meterMultibinder.addBinding().to(ClassLoaderMetrics.class);
meterMultibinder.addBinding().to(JvmMemoryMetrics.class);
meterMultibinder.addBinding().to(JvmGcMetrics.class);
meterMultibinder.addBinding().to(JvmThreadMetrics.class);
meterMultibinder.addBinding().to(LogbackMetrics.class);
meterMultibinder.addBinding().to(FileDescriptorMetrics.class);
meterMultibinder.addBinding().to(ProcessorMetrics.class);
meterMultibinder.addBinding().to(UptimeMetrics.class);
meterMultibinder.addBinding().to(FileStoresMeterBinder.class);
}
}

View File

@ -0,0 +1,54 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.core.metrics;
import io.prometheus.client.CollectorRegistry;
import io.prometheus.client.exporter.HTTPServer;
import java.io.IOException;
import java.net.InetSocketAddress;
import javax.annotation.PreDestroy;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** Bind on 8087 port http endpoint with prometheus metrics. */
@Singleton
public class MetricsServer {
private static final Logger LOG = LoggerFactory.getLogger(MetricsServer.class);
private HTTPServer server;
private final CollectorRegistry collectorRegistry;
private final Integer metricsPort;
@Inject
public MetricsServer(
CollectorRegistry collectorRegistry, @Named("che.metrics.port") Integer metricsPort) {
this.collectorRegistry = collectorRegistry;
this.metricsPort = metricsPort;
}
public void startServer() throws IOException {
this.server = new HTTPServer(new InetSocketAddress(metricsPort), collectorRegistry, true);
LOG.info("Metrics server started at port {} successfully ", metricsPort);
}
@PreDestroy
public void stopServer() {
if (server != null) {
server.stop();
LOG.info("Metrics server suspended at port {} successfully ", metricsPort);
}
}
}

View File

@ -0,0 +1,63 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.core.metrics;
import com.google.inject.multibindings.Multibinder;
import com.google.inject.servlet.ServletModule;
import io.micrometer.core.instrument.binder.MeterBinder;
import java.lang.reflect.Field;
import javax.servlet.ServletContext;
import org.apache.catalina.Manager;
import org.apache.catalina.core.ApplicationContext;
import org.apache.catalina.core.ApplicationContextFacade;
import org.apache.catalina.core.StandardContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* ServletModule is made to bind {@link org.eclipse.che.core.metrics.TomcatMetricsProvider} in guice
* container.
*/
public class MetricsServletModule extends ServletModule {
private static final Logger LOG = LoggerFactory.getLogger(TomcatMetricsProvider.class);
@Override
protected void configureServlets() {
Multibinder<MeterBinder> meterMultibinder =
Multibinder.newSetBinder(binder(), MeterBinder.class);
meterMultibinder.addBinding().toProvider(TomcatMetricsProvider.class);
bind(Manager.class).toInstance(getManager(getServletContext()));
}
private Manager getManager(ServletContext servletContext) {
try {
ApplicationContextFacade acf = (ApplicationContextFacade) servletContext;
Field applicationContextFacadeField =
ApplicationContextFacade.class.getDeclaredField("context");
applicationContextFacadeField.setAccessible(true);
ApplicationContext appContext = (ApplicationContext) applicationContextFacadeField.get(acf);
Field applicationContextField = ApplicationContext.class.getDeclaredField("context");
applicationContextField.setAccessible(true);
StandardContext stdContext = (StandardContext) applicationContextField.get(appContext);
return stdContext.getManager();
} catch (Exception e) {
// maybe not in Tomcat?
LOG.error("Unable to get catalina manager. Cause: {}", e.getMessage(), e);
throw new RuntimeException(e.getMessage(), e);
}
}
}

View File

@ -0,0 +1,42 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.core.metrics;
import io.micrometer.core.instrument.Clock;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.prometheus.PrometheusConfig;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import io.prometheus.client.CollectorRegistry;
import javax.inject.Inject;
import javax.inject.Provider;
import javax.inject.Singleton;
/**
* {@link javax.inject.Provider} of {@link io.micrometer.prometheus.PrometheusMeterRegistry}
* instances. Used constructor with PrometheusConfig#DEFAULT and Clock.SYSTEM parameters.
*/
@Singleton
public class PrometheusMeterRegistryProvider implements Provider<PrometheusMeterRegistry> {
private final PrometheusMeterRegistry prometheusMeterRegistry;
@Inject
public PrometheusMeterRegistryProvider(CollectorRegistry registry) {
prometheusMeterRegistry =
new PrometheusMeterRegistry(PrometheusConfig.DEFAULT, registry, Clock.SYSTEM);
Metrics.addRegistry(prometheusMeterRegistry);
}
@Override
public PrometheusMeterRegistry get() {
return prometheusMeterRegistry;
}
}

View File

@ -0,0 +1,40 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.core.metrics;
import io.micrometer.core.instrument.Tags;
import io.micrometer.core.instrument.binder.tomcat.TomcatMetrics;
import javax.inject.Inject;
import javax.inject.Provider;
import javax.inject.Singleton;
import org.apache.catalina.Manager;
/**
* {@link javax.inject.Provider} of {@link
* io.micrometer.core.instrument.binder.tomcat.TomcatMetrics} instance. Used constructor with empty
* {@link io.micrometer.core.instrument.Tags}
*/
@Singleton
public class TomcatMetricsProvider implements Provider<TomcatMetrics> {
private final Manager manager;
@Inject
public TomcatMetricsProvider(Manager manager) {
this.manager = manager;
}
@Override
public TomcatMetrics get() {
return new TomcatMetrics(manager, Tags.empty());
}
}

View File

@ -0,0 +1,54 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.core.metrics;
import static org.testng.Assert.*;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import java.nio.file.FileStore;
import java.nio.file.FileSystems;
import java.util.Collection;
import java.util.Iterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.annotations.Test;
public class FileStoresMeterTest {
private static final Logger LOG = LoggerFactory.getLogger(FileStoresMeterTest.class);
@Test
public void shouldBindFileStores() {
MeterRegistry registry = new SimpleMeterRegistry();
new FileStoresMeterBinder().bindTo(registry);
Collection<Gauge> dd = registry.get("disk.free").gauges();
assertNotNull(dd);
assertEquals(dd.size(), getSize());
assertTrue(registry.get("disk.free").gauge().value() >= 0);
assertTrue(registry.get("disk.total").gauge().value() >= 0);
assertTrue(registry.get("disk.usable").gauge().value() >= 0);
}
private int getSize() {
int i = 0;
Iterator<FileStore> it = FileSystems.getDefault().getFileStores().iterator();
while (it.hasNext()) {
LOG.debug("Found {} meter ", it.next().name());
i++;
}
return i;
}
}

View File

@ -0,0 +1,27 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (c) 2012-2018 Red Hat, Inc.
This program and the accompanying materials are made
available under the terms of the Eclipse Public License 2.0
which is available at https://www.eclipse.org/legal/epl-2.0/
SPDX-License-Identifier: EPL-2.0
Contributors:
Red Hat, Inc. - initial API and implementation
-->
<configuration>
<appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%-41(%date[%.15thread]) %-45([%-5level] [%.30logger{30} %L]) - %msg%n</pattern>
</encoder>
</appender>
<root level="INFO">
<appender-ref ref="stdout"/>
</root>
</configuration>

View File

@ -39,5 +39,6 @@
<module>che-core-db-vendor-postgresql</module>
<module>che-core-tracing-core</module>
<module>che-core-tracing-web</module>
<module>che-core-metrics-core</module>
</modules>
</project>

View File

@ -38,6 +38,10 @@ objects:
port: 8080
protocol: TCP
targetPort: 8080
- name: metrics
port: 8087
protocol: TCP
targetPort: 8087
selector:
app: che
- apiVersion: v1
@ -48,6 +52,8 @@ objects:
to:
kind: Service
name: che-host
port:
targetPort: http
- apiVersion: v1
kind: DeploymentConfig
metadata:
@ -153,6 +159,8 @@ objects:
value: "${CHE_WORKSPACE_PLUGIN__REGISTRY__URL}"
- name: CHE_TRACING_ENABLED
value: "${CHE_TRACING_ENABLED}"
- name: CHE_METRICS_ENABLED
value: "false"
image: ${IMAGE_CHE}:${CHE_VERSION}
imagePullPolicy: "${PULL_POLICY}"
livenessProbe:
@ -176,6 +184,9 @@ objects:
- containerPort: 8080
name: http
protocol: TCP
- containerPort: 8087
name: metrics
protocol: TCP
- containerPort: 8000
name: http-debug
- containerPort: 8888

View File

@ -742,6 +742,11 @@
<version>${che.version}</version>
<type>gwt-lib</type>
</dependency>
<dependency>
<groupId>org.eclipse.che.core</groupId>
<artifactId>che-core-metrics-core</artifactId>
<version>${che.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.che.core</groupId>
<artifactId>che-core-orion-editor</artifactId>