Add prometheus and grafana for monitoring

monitor.yaml

@@ -0,0 +1,103 @@
+---
+# Gather facts from monitored hosts so we can use their IPs on the monitoring server
+- name: Gather facts from monitored nodes
+  hosts: monitored
+  gather_facts: true
+
+- name: Monitored nodes
+  hosts: monitored
+
+  vars:
+    username: matt
+    docker_add_repo: true
+    docker_users:
+      - "{{ username }}"
+
+  roles:
+    - role: geerlingguy.docker
+      become: true
+
+  tasks:
+    - name: Create node-exporter directory
+      ansible.builtin.file:
+        path: "{{ ansible_env.HOME }}/node-exporter"
+        state: directory
+        mode: "0755"
+
+    - name: Copy node-exporter compose file
+      ansible.builtin.copy:
+        src: node-exporter/compose.yaml
+        dest: "{{ ansible_env.HOME }}/node-exporter/compose.yaml"
+
+    - name: Start node-exporter
+      community.docker.docker_compose_v2:
+        project_src: "{{ ansible_env.HOME }}/node-exporter/"
+        state: present
+
+- name: Monitoring server
+  hosts: monitoring_server
+
+  vars:
+    username: matt
+    docker_add_repo: true
+    docker_users:
+      - "{{ username }}"
+
+  roles:
+    - role: geerlingguy.docker
+      become: true
+
+  tasks:
+    - name: Create monitoring directory
+      ansible.builtin.file:
+        path: "{{ ansible_env.HOME }}/monitoring"
+        state: directory
+        mode: "0755"
+
+    - name: Create grafana provisioning directories
+      ansible.builtin.file:
+        path: "{{ ansible_env.HOME }}/monitoring/grafana/provisioning/{{ item }}"
+        state: directory
+        mode: "0755"
+      loop:
+        - datasources
+        - dashboards
+
+    - name: Template prometheus config
+      ansible.builtin.template:
+        src: monitoring/prometheus.yml.j2
+        dest: "{{ ansible_env.HOME }}/monitoring/prometheus.yml"
+        mode: "0644"
+
+    - name: Copy compose file
+      ansible.builtin.copy:
+        src: monitoring/compose.yaml
+        dest: "{{ ansible_env.HOME }}/monitoring/compose.yaml"
+
+    - name: Copy grafana datasource config
+      ansible.builtin.copy:
+        src: monitoring/grafana/provisioning/datasources/prometheus.yaml
+        dest: "{{ ansible_env.HOME }}/monitoring/grafana/provisioning/datasources/prometheus.yaml"
+
+    - name: Copy grafana dashboard provisioning config
+      ansible.builtin.copy:
+        src: monitoring/grafana/provisioning/dashboards/dashboards.yaml
+        dest: "{{ ansible_env.HOME }}/monitoring/grafana/provisioning/dashboards/dashboards.yaml"
+
+    - name: Fetch Node Exporter Full dashboard from grafana.com
+      ansible.builtin.get_url:
+        url: "https://grafana.com/api/dashboards/1860/revisions/37/download"
+        dest: "{{ ansible_env.HOME }}/monitoring/grafana/provisioning/dashboards/node-exporter-full.json"
+        mode: "0644"
+        force: false
+
+    - name: Write .env file with secrets
+      ansible.builtin.copy:
+        content: "GRAFANA_ADMIN_PASSWORD={{ secrets.GRAFANA_ADMIN_PASSWORD }}\n"
+        dest: "{{ ansible_env.HOME }}/monitoring/.env"
+        mode: "0600"
+
+    - name: Start monitoring services
+      community.docker.docker_compose_v2:
+        project_src: "{{ ansible_env.HOME }}/monitoring/"
+        state: present