Decoupled monitoring configuration and deployment (#13152)

* Decoupled monitoring configuration and deployment

Signed-off-by: Sergii Kabashniuk <skabashniuk@redhat.com>

.github/CODEOWNERS

@@ -63,7 +63,7 @@ wsagent/che-core-api-project/src/test/java/org/eclipse/che/api/search/** @skabas
 
 # deploy
 deploy/** @l0rd @sleshchenko
-deploy/openshift/templates/che-monitoring.yaml @skabashnyuk @metlos
+deploy/openshift/templates/monitoring/** @skabashnyuk @metlos
 
 # selenium tests
 selenium/** @musienko-maxim @dmytro-ndp @Ohrimenko1988

deploy/openshift/deploy_che.sh

@@ -512,7 +512,10 @@ deployJaeger(){
 deployMetrics(){
     if [ "${CHE_METRICS_ENABLED}" == "true" ]; then
       echo "Deploying Grafana and Prometheus..."
-      ${OC_BINARY} new-app -f ${BASE_DIR}/templates/che-monitoring.yaml
+      ${OC_BINARY} apply -f ${BASE_DIR}/templates/monitoring/grafana-dashboards.yaml
+      ${OC_BINARY} apply -f ${BASE_DIR}/templates/monitoring/grafana-datasources.yaml
+      ${OC_BINARY} apply -f ${BASE_DIR}/templates/monitoring/prometheus-config.yaml
+      ${OC_BINARY} new-app -f ${BASE_DIR}/templates/monitoring/che-monitoring.yaml
       echo "Grafana deployment complete. $($OC_BINARY get route/grafana --namespace=${CHE_OPENSHIFT_PROJECT} -o=jsonpath={'.spec.host'})"
       echo "Prometheus deployment complete. $($OC_BINARY get route/prometheus --namespace=${CHE_OPENSHIFT_PROJECT} -o=jsonpath={'.spec.host'})"
     fi

deploy/openshift/templates/monitoring/che-monitoring.yaml

@@ -0,0 +1,164 @@
+apiVersion: v1
+kind: Template
+metadata:
+  name: che-monitoring
+objects:
+- apiVersion: v1
+  kind: Route
+  metadata:
+    labels:
+      app: prometheus
+    name: prometheus
+  spec:
+    port:
+      targetPort: 9090-tcp
+    to:
+      kind: Service
+      name: prometheus
+- apiVersion: v1
+  kind: Route
+  metadata:
+    labels:
+      app: grafana
+    name: grafana
+  spec:
+    port:
+      targetPort: 3000-tcp
+    to:
+      kind: Service
+      name: grafana
+- apiVersion: v1
+  kind: Service
+  metadata:
+    labels:
+      app: grafana
+    name: grafana
+  spec:
+    ports:
+    - name: 3000-tcp
+      port: 3000
+      protocol: TCP
+      targetPort: 3000
+    selector:
+      deploymentconfig: grafana
+- apiVersion: v1
+  kind: Service
+  metadata:
+    labels:
+      app: prometheus
+    name: prometheus
+  spec:
+    ports:
+    - name: 9090-tcp
+      port: 9090
+      protocol: TCP
+      targetPort: 9090
+    selector:
+      deploymentconfig: prometheus
+- apiVersion: v1
+  kind: DeploymentConfig
+  metadata:
+    generation: 1
+    labels:
+      app: grafana
+    name: grafana
+  spec:
+    replicas: 1
+    selector:
+      app: grafana
+      deploymentconfig: grafana
+    template:
+      metadata:
+        labels:
+          app: grafana
+          deploymentconfig: grafana
+      spec:
+        containers:
+        - image: grafana/grafana
+          imagePullPolicy: Always
+          name: grafana
+          ports:
+          - containerPort: 3000
+            protocol: TCP
+          volumeMounts:
+          - mountPath: /etc/grafana/provisioning/datasources
+            name: volume-datasources
+          - mountPath: /etc/grafana/provisioning/dashboards
+            name: volume-dashboards
+          readinessProbe:
+            httpGet:
+              path: /api/health
+              port: 3000
+          livenessProbe:
+            httpGet:
+              path: /api/health
+              port: 3000
+            initialDelaySeconds: 60
+            timeoutSeconds: 30
+            failureThreshold: 10
+        restartPolicy: Always
+        schedulerName: default-scheduler
+        terminationGracePeriodSeconds: 30
+        volumes:
+        - configMap:
+            defaultMode: 420
+            name: grafana-datasources
+          name: volume-datasources
+        - configMap:
+            defaultMode: 420
+            name: grafana-dashboards
+          name: volume-dashboards
+    test: false
+    triggers:
+      - type: ConfigChange
+- apiVersion: v1
+  kind: DeploymentConfig
+  metadata:
+    generation: 1
+    labels:
+      app: prometheus
+    name: prometheus
+  spec:
+    replicas: 1
+    selector:
+      app: prometheus
+      deploymentconfig: prometheus
+    template:
+      metadata:
+        labels:
+          app: prometheus
+          deploymentconfig: prometheus
+      spec:
+        containers:
+        - image: prom/prometheus
+          imagePullPolicy: Always
+          name: prometheus
+          ports:
+          - containerPort: 9090
+            protocol: TCP
+          volumeMounts:
+          - mountPath: /prometheus
+            name: volume-data
+          - mountPath: /etc/prometheus/prometheus.yml
+            name: volume-config
+            subPath: prometheus.yml
+          readinessProbe:
+            httpGet:
+              path: /-/ready
+              port: 9090
+          livenessProbe:
+            httpGet:
+              path: /-/healthy
+              port: 9090
+            initialDelaySeconds: 30
+            timeoutSeconds: 30
+        restartPolicy: Always
+        volumes:
+        - emptyDir: {}
+          name: volume-data
+        - configMap:
+            defaultMode: 420
+            name: prometheus-config
+          name: volume-config
+    triggers:
+    - type: ConfigChange

deploy/openshift/templates/monitoring/grafana-datasources.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-datasources
+data:
+  che-datasource.yaml: |-
+    apiVersion: 1
+    datasources:
+    - name: che
+      type: prometheus
+      access: proxy
+      url: http://prometheus:9090

deploy/openshift/templates/monitoring/prometheus-config.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-config
+data:
+  prometheus.yml: |-
+    global:
+      scrape_interval:     5s
+      evaluation_interval: 5s
+    scrape_configs:
+      - job_name: 'che'
+        static_configs:
+          - targets: ['che-host:8087']