Add prometheus and grafana for monitoring

README.md

@@ -3,11 +3,14 @@
 ansible-galaxy install -r requirements.yml
 
 # Install the playbook
-ansible-playbook -i inventory -e @secrets.enc --ask-vault-pass frontend.yaml
+ansible-playbook -i inventory.yaml -e @secrets.enc --ask-vault-pass frontend.yaml
 
 # Github runner
-ansible-playbook -i inventory -e @secrets.enc --ask-vault-pass github-runner.yaml
+ansible-playbook -i inventory.yaml -e @secrets.enc --ask-vault-pass github-runner.yaml
 
 # gitea server
-ansible-playbook -i inventory -e @secrets.enc --ask-vault-pass gitea.yaml
+ansible-playbook -i inventory.yaml -e @secrets.enc --ask-vault-pass gitea.yaml
+
+# gitea server
+ansible-playbook -i inventory.yaml -e @secrets.enc --ask-vault-pass monitor.yaml
 ```

inventory.yaml

@@ -0,0 +1,28 @@
+all:
+  hosts:
+    rpi4-1:
+      ansible_host: rpi4-1.local
+    rpi4-2:
+      ansible_host: rpi4-2.local
+    rpi5-1:
+      ansible_host: rpi5-1.local
+    rpi5-2:
+      ansible_host: rpi5-2.local
+  children:
+    monitored:
+      hosts:
+        rpi4-2: {}
+        # rpi5-1: {}
+        rpi5-2: {}
+    monitoring_server:
+      hosts:
+        rpi4-2: {}
+    frontend:
+      hosts:
+        rpi4-1: {}
+    github-runners:
+      hosts:
+        rpi5-1: {}
+    gitea:
+      hosts:
+        rpi5-2: {}

monitor.yaml

@@ -0,0 +1,103 @@
+---
+# Gather facts from monitored hosts so we can use their IPs on the monitoring server
+- name: Gather facts from monitored nodes
+  hosts: monitored
+  gather_facts: true
+
+- name: Monitored nodes
+  hosts: monitored
+
+  vars:
+    username: matt
+    docker_add_repo: true
+    docker_users:
+      - "{{ username }}"
+
+  roles:
+    - role: geerlingguy.docker
+      become: true
+
+  tasks:
+    - name: Create node-exporter directory
+      ansible.builtin.file:
+        path: "{{ ansible_env.HOME }}/node-exporter"
+        state: directory
+        mode: "0755"
+
+    - name: Copy node-exporter compose file
+      ansible.builtin.copy:
+        src: node-exporter/compose.yaml
+        dest: "{{ ansible_env.HOME }}/node-exporter/compose.yaml"
+
+    - name: Start node-exporter
+      community.docker.docker_compose_v2:
+        project_src: "{{ ansible_env.HOME }}/node-exporter/"
+        state: present
+
+- name: Monitoring server
+  hosts: monitoring_server
+
+  vars:
+    username: matt
+    docker_add_repo: true
+    docker_users:
+      - "{{ username }}"
+
+  roles:
+    - role: geerlingguy.docker
+      become: true
+
+  tasks:
+    - name: Create monitoring directory
+      ansible.builtin.file:
+        path: "{{ ansible_env.HOME }}/monitoring"
+        state: directory
+        mode: "0755"
+
+    - name: Create grafana provisioning directories
+      ansible.builtin.file:
+        path: "{{ ansible_env.HOME }}/monitoring/grafana/provisioning/{{ item }}"
+        state: directory
+        mode: "0755"
+      loop:
+        - datasources
+        - dashboards
+
+    - name: Template prometheus config
+      ansible.builtin.template:
+        src: monitoring/prometheus.yml.j2
+        dest: "{{ ansible_env.HOME }}/monitoring/prometheus.yml"
+        mode: "0644"
+
+    - name: Copy compose file
+      ansible.builtin.copy:
+        src: monitoring/compose.yaml
+        dest: "{{ ansible_env.HOME }}/monitoring/compose.yaml"
+
+    - name: Copy grafana datasource config
+      ansible.builtin.copy:
+        src: monitoring/grafana/provisioning/datasources/prometheus.yaml
+        dest: "{{ ansible_env.HOME }}/monitoring/grafana/provisioning/datasources/prometheus.yaml"
+
+    - name: Copy grafana dashboard provisioning config
+      ansible.builtin.copy:
+        src: monitoring/grafana/provisioning/dashboards/dashboards.yaml
+        dest: "{{ ansible_env.HOME }}/monitoring/grafana/provisioning/dashboards/dashboards.yaml"
+
+    - name: Fetch Node Exporter Full dashboard from grafana.com
+      ansible.builtin.get_url:
+        url: "https://grafana.com/api/dashboards/1860/revisions/37/download"
+        dest: "{{ ansible_env.HOME }}/monitoring/grafana/provisioning/dashboards/node-exporter-full.json"
+        mode: "0644"
+        force: false
+
+    - name: Write .env file with secrets
+      ansible.builtin.copy:
+        content: "GRAFANA_ADMIN_PASSWORD={{ secrets.GRAFANA_ADMIN_PASSWORD }}\n"
+        dest: "{{ ansible_env.HOME }}/monitoring/.env"
+        mode: "0600"
+
+    - name: Start monitoring services
+      community.docker.docker_compose_v2:
+        project_src: "{{ ansible_env.HOME }}/monitoring/"
+        state: present

monitoring/compose.yaml

@@ -0,0 +1,34 @@
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    restart: unless-stopped
+    network_mode: host
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    restart: unless-stopped
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD}
+      - GF_INSTALL_PLUGINS=vonage-status-panel
+    volumes:
+      - grafana-data:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+    ports:
+      - "4000:3000"
+    extra_hosts:
+      - "host-gateway:host-gateway"
+    depends_on:
+      - prometheus
+
+volumes:
+  prometheus-data:
+  grafana-data:

monitoring/grafana/provisioning/dashboards/dashboards.yaml

@@ -0,0 +1,7 @@
+apiVersion: 1
+
+providers:
+  - name: default
+    type: file
+    options:
+      path: /etc/grafana/provisioning/dashboards

monitoring/grafana/provisioning/datasources/prometheus.yaml

@@ -0,0 +1,8 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://host-gateway:9090
+    isDefault: true

monitoring/prometheus.yml.j2

@@ -0,0 +1,10 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: node
+    static_configs:
+      - targets:
+{% for host in groups['monitored'] %}
+          - "{{ hostvars[host].ansible_host }}:9100"
+{% endfor %}

node-exporter/compose.yaml

@@ -0,0 +1,17 @@
+services:
+  node-exporter:
+    image: prom/node-exporter:latest
+    container_name: node-exporter
+    restart: unless-stopped
+    pid: host
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - "--path.procfs=/host/proc"
+      - "--path.rootfs=/rootfs"
+      - "--path.sysfs=/host/sys"
+      - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
+    ports:
+      - "9100:9100"

secrets.enc

@@ -1,20 +1,22 @@
 $ANSIBLE_VAULT;1.1;AES256
-63396337613935616363393830653839656434663934623938366331613430316465656439643761
+30373739346533376636623934363634373734316230386233333066666662656566613834643062
-6239373331623465373164643836326362643461623536340a383134656335376663616134393031
+3234313933626566663563646539393132323466643564350a303932373831386564613738663865
-31613730663666386135363136306339666534656533373163643933613238623233313630363032
+30393439643239316639303933376536353230393966373234336231306438386338663464333439
-3761373132373639360a366564303535396630653466366361643761306432343732353666393863
+3831353636323437650a333837366433396236383333303738363862623730636539616330666663
-32636165633363383266636637643836333731626265613863363630313430613561626530303730
+62623839616131623837336537636434353237666434386437633066346365646437366363356333
-38363463373464333833666632366636666464306265313639343963376265323438333837333733
+32356135613062366566323637383730343230623937646661666464343436666533643838333664
-30376334313062316264393663383134346362613265356462333066666535393666383936363730
+38303039633435366334613934653639386661663065386334383765393237386663353865383830
-62656334373438383139343735376538653738636530333337346633386331326565323462646236
+31366138383834643438303535383134306430383739326233643438383432366539393539386637
-31626431353265383766336564303734316165373062343036643362353364313335613237623233
+32313635383661303435643936363633326662306235303764643163323965303165353065313530
-31363537343033313138646338616532363161626133626137396638366431333266306539313134
+39613766343736653163366434383832303539646633636134313134383733303632633130376131
-30663338323366353866316434313661666137343634306231643533373636613431356337623165
+65313862653437313534663164383561653033313663383637363865386365663639306137626465
-64373338313665373635646136393736303030626134313863306634623463653038386566313030
+63653266636433333362356261653965313261666338323333393339353530356466653463303634
-33376130383161313034323563656237626430663362323265623766393830363032653934306431
+38313836663465643361316363363364343665356565303836356433363461383331336237376561
-30363364393663666131653536356135303536663966633564323135326663613663313739303635
+37316431396364626431313530306466663238303736623335336132663236353864663066383639
-62373862383030656136633338396166616431396466353432323630366436313339623430393262
+34386261353664336332363032333130613963613539623965303261363339633064663739326532
-31373938653135353231323666396166303466626233626261653631653132306561336463663366
+66323338346134303762646134663336356666326135346264343931653035323136333031323331
-32626335346431333632363061643862353736393435336536303536323736303539376566323838
+38393861383136653866666138626639613837616264633031343639323036653563383661633434
-64333163633639383366646165633231326562353933323838663639663835343936373235363335
+30343136636632613838333538393466316664316261343766343635353132666233633936363931
-633636623866386566613530633861373364
+62313430323230616138643463643239373465313435663430343433643335363362303264366464
+61663034353165323237636465303131346366626361643531616266633435303065613434396633
+6663