Prometheus, Node Exporter & Grafana

Node Exporter :9100 ──scrape──▶ Prometheus :9090 ──query──▶ Grafana :3000
App /metrics      ──scrape──▶        │
                                     └──▶ Alertmanager

services:
  prometheus:
    image: prom/prometheus:latest
    restart: unless-stopped
    ports:
      - "127.0.0.1:9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prom-data:/prometheus
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.retention.time=15d

  node-exporter:
    image: prom/node-exporter:latest
    restart: unless-stopped
    ports:
      - "127.0.0.1:9100:9100"
    pid: host
    volumes:
      - /:/host:ro,rslave

  grafana:
    image: grafana/grafana:latest
    restart: unless-stopped
    ports:
      - "127.0.0.1:3001:3000"
    volumes:
      - grafana-data:/var/lib/grafana
    environment:
      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD}

volumes:
  prom-data:
  grafana-data:

global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: prometheus
    static_configs:
      - targets: ['localhost:9090']

  - job_name: node
    static_configs:
      - targets: ['node-exporter:9100']

  - job_name: app
    metrics_path: /metrics
    static_configs:
      - targets: ['10.10.10.11:3000']

curl -X POST http://localhost:9090/-/reload

100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

sum(rate(http_requests_total{status=~"5.."}[5m]))
/
sum(rate(http_requests_total[5m]))

groups:
  - name: host
    rules:
      - alert: InstanceDown
        expr: up == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Target {{ $labels.instance }} mati"

      - alert: DiskAlmostFull
        expr: (node_filesystem_avail_bytes / node_filesystem_size_bytes) < 0.1
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Disk {{ $labels.mountpoint }} hampir penuh"

rule_files:
  - /etc/prometheus/alerts.yml

services:
  alertmanager:
    image: prom/alertmanager:latest
    ports:
      - "127.0.0.1:9093:9093"
    volumes:
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro

route:
  receiver: default
  group_by: [alertname]
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h

receivers:
  - name: default
    webhook_configs:
      - url: http://10.10.10.50:5001/webhook