--- apiVersion: v1 kind: List items: - apiVersion: v1 kind: ServiceAccount metadata: labels: provider: fabric8 project: prometheus version: 2.2.300 group: io.fabric8.devops.apps name: metrics - apiVersion: v1 kind: Service metadata: annotations: prometheus.io/scrape: "true" fabric8.io/app-menu: management fabric8.io/git-commit: 3b38f81a06b19e0db5badd7cebdbdadd7718ee74 fabric8.io/iconUrl: https://cdn.rawgit.com/fabric8io/fabric8-devops/master/prometheus/src/main/fabric8/icon.png fabric8.io/git-branch: release-v2.2.300 labels: provider: fabric8 expose: "true" project: prometheus version: 2.2.300 group: io.fabric8.devops.apps name: prometheus spec: ports: - name: http port: 80 protocol: TCP targetPort: 9090 selector: project: prometheus provider: fabric8 group: io.fabric8.devops.apps - apiVersion: v1 kind: PersistentVolumeClaim metadata: annotations: volume.beta.kubernetes.io/storage-class: standard labels: provider: fabric8 project: prometheus version: 2.2.300 group: io.fabric8.devops.apps name: prometheus-data spec: accessModes: - ReadWriteOnce resources: requests: storage: 1Gi - apiVersion: v1 kind: ConfigMap metadata: labels: provider: fabric8 project: prometheus version: 2.2.300 group: io.fabric8.devops.apps name: prometheus data: prometheus.yml: |- global: scrape_interval: 30s evaluation_interval: 60s rule_files: - "*.rules" # A scrape configuration for running Prometheus on a Kubernetes cluster. # This uses separate scrape configs for cluster components (i.e. API server, node) # and services to allow each to use different authentication configs. # # Kubernetes labels will be added as Prometheus labels on metrics via the # `labelmap` relabeling action. scrape_configs: # Scrape config for API servers. - job_name: 'kubernetes-apiservers' # This TLS & bearer token file config is used to connect to the actual scrape # endpoints for cluster components. This is separate to discovery auth # configuration (`in_cluster` below) because discovery & scraping are two # separate concerns in Prometheus. tls_config: # workaround for Prometheus -> Kubernetes cert issue https://github.com/fabric8io/fabric8-devops/issues/438 insecure_skip_verify: true ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token scheme: https kubernetes_sd_configs: - role: endpoints relabel_configs: - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] action: keep regex: default;kubernetes;https # Scrape config for nodes. - job_name: 'kubernetes-nodes' # This TLS & bearer token file config is used to connect to the actual scrape # endpoints for cluster components. This is separate to discovery auth # configuration (`in_cluster` below) because discovery & scraping are two # separate concerns in Prometheus. tls_config: # workaround for Prometheus -> Kubernetes cert issue https://github.com/fabric8io/fabric8-devops/issues/438 insecure_skip_verify: true ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token scheme: https kubernetes_sd_configs: - role: node relabel_configs: - action: labelmap regex: __meta_kubernetes_node_label_(.+) # Scrape config for node exporter pods deployed as a daemonset. - job_name: 'kubernetes-node-exporters' kubernetes_sd_configs: - role: pod relabel_configs: - source_labels: [__meta_kubernetes_pod_name] action: keep regex: node-exporter-.+ - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] action: replace regex: (.+):(?:\d+);(\d+) replacement: ${1}:${2} target_label: __address__ - source_labels: [__meta_kubernetes_pod_node_name] action: replace target_label: instance # Scrape config for service endpoints. # # The relabeling allows the actual service scrape endpoint to be configured # via the following annotations: # # * `prometheus.io/scrape`: Only scrape services that have a value of `true` # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need # to set this to `https` & most likely set the `tls_config` of the scrape config. # * `prometheus.io/path`: If the metrics path is not `/metrics` override this. # * `prometheus.io/port`: If the metrics are exposed on a different port to the # service then set this appropriately. - job_name: 'kubernetes-service-endpoints' kubernetes_sd_configs: - role: endpoints relabel_configs: - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] action: keep regex: true - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] action: replace target_label: __scheme__ regex: (https?) - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] action: replace regex: (.+) target_label: __metrics_path__ - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] action: replace target_label: __address__ regex: (.+)(?::\d+);(\d+) replacement: $1:$2 - action: labelmap regex: __meta_kubernetes_service_label_(.+) - source_labels: [__meta_kubernetes_service_namespace] action: replace target_label: kubernetes_namespace - source_labels: [__meta_kubernetes_service_name] action: replace target_label: kubernetes_name # Scrape config for probing services via the Blackbox Exporter. # # The relabeling allows the actual service scrape endpoint to be configured # via the following annotations: # # * `prometheus.io/probe`: Only probe services that have a value of `true` - job_name: 'kubernetes-services' metrics_path: /probe params: module: [http_2xx] kubernetes_sd_configs: - role: service relabel_configs: - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe] action: keep regex: true - source_labels: [] target_label: __address__ replacement: blackbox - source_labels: [__address__] regex: (.*)(:80)? target_label: __param_target - source_labels: [__meta_kubernetes_service_name,__meta_kubernetes_service_namespace] target_label: __param_target regex: ([^;]+);(.+) replacement: $1.$2.svc - action: labelmap regex: __meta_kubernetes_service_label_(.+) - source_labels: [__meta_kubernetes_service_namespace] action: replace target_label: kubernetes_namespace - source_labels: [__meta_kubernetes_service_name] action: replace target_label: kubernetes_name # Scrape config for pods # # The relabeling allows the actual pod scrape endpoint to be configured via the # following annotations: # # * `prometheus.io/scrape`: Only scrape pods that have a value of `true` # * `prometheus.io/port`: Scrape the pod on the indicated port instead of the default of `9102`. - job_name: 'kubernetes-pods' kubernetes_sd_configs: - role: pod relabel_configs: - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] action: keep regex: true - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme] action: replace target_label: __scheme__ regex: (https?) - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] action: replace target_label: __metrics_path__ regex: (.+) - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] action: replace regex: (.+):(?:\d+);(\d+) replacement: ${1}:${2} target_label: __address__ - action: labelmap regex: __meta_kubernetes_pod_label_(.+) pod.rules: |- pod:memory_usage_bytes = sum(container_memory_usage_bytes{kubernetes_pod_name=~".+", job="kubernetes-nodes"}) by (kubernetes_pod_name, kubernetes_namespace) pod:network_receive_bytes:1m = sum(rate(container_network_receive_bytes_total{kubernetes_pod_name=~".+", job="kubernetes-nodes"}[1m])) by (kubernetes_pod_name, kubernetes_namespace) pod:network_transmit_bytes:1m = sum(rate(container_network_transmit_bytes_total{kubernetes_pod_name=~".+", job="kubernetes-nodes"}[1m])) by (kubernetes_pod_name, kubernetes_namespace) pod:cpu_usage_seconds:1m = sum(rate(container_cpu_usage_seconds_total{kubernetes_pod_name=~".+", job="kubernetes-nodes"}[1m])) by (kubernetes_pod_name, kubernetes_namespace) ALERT ServiceDown IF probe_success{job="kubernetes-services"} == 0 FOR 10s LABELS { service = "{{$labels.kubernetes_namespace}}/{{$labels.kubernetes_name}}" } ANNOTATIONS { severity ="page", summary ="Service {{$labels.kubernetes_namespace}}/{{$labels.kubernetes_name}} down", description ="Service {{$labels.kubernetes_namespace}}/{{$labels.kubernetes_name}} has been down for more than 10 seconds!" } - apiVersion: extensions/v1beta1 kind: Deployment metadata: annotations: fabric8.io/git-commit: 3b38f81a06b19e0db5badd7cebdbdadd7718ee74 fabric8.io/iconUrl: https://cdn.rawgit.com/fabric8io/fabric8-devops/master/prometheus/src/main/fabric8/icon.png fabric8.io/git-branch: release-v2.2.300 fabric8.io/metrics-path: dashboard/file/kubernetes-pods.json/?var-project=prometheus&var-version=2.2.300 labels: provider: fabric8 project: prometheus version: 2.2.300 group: io.fabric8.devops.apps name: prometheus spec: replicas: 1 selector: matchLabels: project: prometheus provider: fabric8 group: io.fabric8.devops.apps template: metadata: annotations: fabric8.io/git-commit: 3b38f81a06b19e0db5badd7cebdbdadd7718ee74 fabric8.io/iconUrl: https://cdn.rawgit.com/fabric8io/fabric8-devops/master/prometheus/src/main/fabric8/icon.png fabric8.io/git-branch: release-v2.2.300 fabric8.io/metrics-path: dashboard/file/kubernetes-pods.json/?var-project=prometheus&var-version=2.2.300 pod.alpha.kubernetes.io/init-containers: '[{"image":"busybox","imagePullPolicy":"IfNotPresent","name":"init","command":["chmod","777","/prometheus","/etc/prometheus"],"volumeMounts":[{"mountPath":"/etc/prometheus","name":"config-volume"},{"mountPath":"/prometheus","name":"data-volume"}]}]' labels: provider: fabric8 project: prometheus version: 2.2.300 group: io.fabric8.devops.apps spec: containers: - image: prom/prometheus:v1.3.1 livenessProbe: httpGet: port: http initialDelaySeconds: 1 name: prometheus ports: - containerPort: 9090 name: http readinessProbe: httpGet: port: http initialDelaySeconds: 1 volumeMounts: - mountPath: /etc/prometheus name: config-volume - mountPath: /prometheus name: data-volume subPath: prometheus-db - args: - -volume-dir - /etc/prometheus - -webhook-url - http://localhost:9090/-/reload image: jimmidyson/configmap-reload:v0.1 name: configmap-reload volumeMounts: - mountPath: /etc/prometheus name: config-volume serviceAccountName: metrics volumes: - configMap: name: prometheus name: config-volume - name: data-volume persistentVolumeClaim: claimName: prometheus-data