diff --git a/README.md b/README.md index 92441ef..11f8195 100755 --- a/README.md +++ b/README.md @@ -3,11 +3,14 @@ ansible-galaxy install -r requirements.yml # Install the playbook -ansible-playbook -i inventory -e @secrets.enc --ask-vault-pass frontend.yaml +ansible-playbook -i inventory.yaml -e @secrets.enc --ask-vault-pass frontend.yaml # Github runner -ansible-playbook -i inventory -e @secrets.enc --ask-vault-pass github-runner.yaml +ansible-playbook -i inventory.yaml -e @secrets.enc --ask-vault-pass github-runner.yaml # gitea server -ansible-playbook -i inventory -e @secrets.enc --ask-vault-pass gitea.yaml +ansible-playbook -i inventory.yaml -e @secrets.enc --ask-vault-pass gitea.yaml + +# gitea server +ansible-playbook -i inventory.yaml -e @secrets.enc --ask-vault-pass monitor.yaml ``` diff --git a/inventory.yaml b/inventory.yaml new file mode 100755 index 0000000..93ccd26 --- /dev/null +++ b/inventory.yaml @@ -0,0 +1,28 @@ +all: + hosts: + rpi4-1: + ansible_host: rpi4-1.local + rpi4-2: + ansible_host: rpi4-2.local + rpi5-1: + ansible_host: rpi5-1.local + rpi5-2: + ansible_host: rpi5-2.local + children: + monitored: + hosts: + rpi4-2: {} + # rpi5-1: {} + rpi5-2: {} + monitoring_server: + hosts: + rpi4-2: {} + frontend: + hosts: + rpi4-1: {} + github-runners: + hosts: + rpi5-1: {} + gitea: + hosts: + rpi5-2: {} \ No newline at end of file diff --git a/monitor.yaml b/monitor.yaml new file mode 100644 index 0000000..ed1e907 --- /dev/null +++ b/monitor.yaml @@ -0,0 +1,103 @@ +--- +# Gather facts from monitored hosts so we can use their IPs on the monitoring server +- name: Gather facts from monitored nodes + hosts: monitored + gather_facts: true + +- name: Monitored nodes + hosts: monitored + + vars: + username: matt + docker_add_repo: true + docker_users: + - "{{ username }}" + + roles: + - role: geerlingguy.docker + become: true + + tasks: + - name: Create node-exporter directory + ansible.builtin.file: + path: "{{ ansible_env.HOME }}/node-exporter" + state: directory + mode: "0755" + + - name: Copy node-exporter compose file + ansible.builtin.copy: + src: node-exporter/compose.yaml + dest: "{{ ansible_env.HOME }}/node-exporter/compose.yaml" + + - name: Start node-exporter + community.docker.docker_compose_v2: + project_src: "{{ ansible_env.HOME }}/node-exporter/" + state: present + +- name: Monitoring server + hosts: monitoring_server + + vars: + username: matt + docker_add_repo: true + docker_users: + - "{{ username }}" + + roles: + - role: geerlingguy.docker + become: true + + tasks: + - name: Create monitoring directory + ansible.builtin.file: + path: "{{ ansible_env.HOME }}/monitoring" + state: directory + mode: "0755" + + - name: Create grafana provisioning directories + ansible.builtin.file: + path: "{{ ansible_env.HOME }}/monitoring/grafana/provisioning/{{ item }}" + state: directory + mode: "0755" + loop: + - datasources + - dashboards + + - name: Template prometheus config + ansible.builtin.template: + src: monitoring/prometheus.yml.j2 + dest: "{{ ansible_env.HOME }}/monitoring/prometheus.yml" + mode: "0644" + + - name: Copy compose file + ansible.builtin.copy: + src: monitoring/compose.yaml + dest: "{{ ansible_env.HOME }}/monitoring/compose.yaml" + + - name: Copy grafana datasource config + ansible.builtin.copy: + src: monitoring/grafana/provisioning/datasources/prometheus.yaml + dest: "{{ ansible_env.HOME }}/monitoring/grafana/provisioning/datasources/prometheus.yaml" + + - name: Copy grafana dashboard provisioning config + ansible.builtin.copy: + src: monitoring/grafana/provisioning/dashboards/dashboards.yaml + dest: "{{ ansible_env.HOME }}/monitoring/grafana/provisioning/dashboards/dashboards.yaml" + + - name: Fetch Node Exporter Full dashboard from grafana.com + ansible.builtin.get_url: + url: "https://grafana.com/api/dashboards/1860/revisions/37/download" + dest: "{{ ansible_env.HOME }}/monitoring/grafana/provisioning/dashboards/node-exporter-full.json" + mode: "0644" + force: false + + - name: Write .env file with secrets + ansible.builtin.copy: + content: "GRAFANA_ADMIN_PASSWORD={{ secrets.GRAFANA_ADMIN_PASSWORD }}\n" + dest: "{{ ansible_env.HOME }}/monitoring/.env" + mode: "0600" + + - name: Start monitoring services + community.docker.docker_compose_v2: + project_src: "{{ ansible_env.HOME }}/monitoring/" + state: present diff --git a/monitoring/compose.yaml b/monitoring/compose.yaml new file mode 100644 index 0000000..e78ca41 --- /dev/null +++ b/monitoring/compose.yaml @@ -0,0 +1,34 @@ +services: + prometheus: + image: prom/prometheus:latest + container_name: prometheus + restart: unless-stopped + network_mode: host + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + + grafana: + image: grafana/grafana:latest + container_name: grafana + restart: unless-stopped + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD} + - GF_INSTALL_PLUGINS=vonage-status-panel + volumes: + - grafana-data:/var/lib/grafana + - ./grafana/provisioning:/etc/grafana/provisioning:ro + ports: + - "4000:3000" + extra_hosts: + - "host-gateway:host-gateway" + depends_on: + - prometheus + +volumes: + prometheus-data: + grafana-data: diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yaml b/monitoring/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..970ffbe --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,7 @@ +apiVersion: 1 + +providers: + - name: default + type: file + options: + path: /etc/grafana/provisioning/dashboards diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yaml b/monitoring/grafana/provisioning/datasources/prometheus.yaml new file mode 100644 index 0000000..fe20b57 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://host-gateway:9090 + isDefault: true diff --git a/monitoring/prometheus.yml.j2 b/monitoring/prometheus.yml.j2 new file mode 100644 index 0000000..3c623c5 --- /dev/null +++ b/monitoring/prometheus.yml.j2 @@ -0,0 +1,10 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: node + static_configs: + - targets: +{% for host in groups['monitored'] %} + - "{{ hostvars[host].ansible_host }}:9100" +{% endfor %} diff --git a/node-exporter/compose.yaml b/node-exporter/compose.yaml new file mode 100644 index 0000000..3bf6d70 --- /dev/null +++ b/node-exporter/compose.yaml @@ -0,0 +1,17 @@ +services: + node-exporter: + image: prom/node-exporter:latest + container_name: node-exporter + restart: unless-stopped + pid: host + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - "--path.procfs=/host/proc" + - "--path.rootfs=/rootfs" + - "--path.sysfs=/host/sys" + - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)" + ports: + - "9100:9100" diff --git a/secrets.enc b/secrets.enc index 088be1c..e912458 100644 --- a/secrets.enc +++ b/secrets.enc @@ -1,20 +1,22 @@ $ANSIBLE_VAULT;1.1;AES256 -63396337613935616363393830653839656434663934623938366331613430316465656439643761 -6239373331623465373164643836326362643461623536340a383134656335376663616134393031 -31613730663666386135363136306339666534656533373163643933613238623233313630363032 -3761373132373639360a366564303535396630653466366361643761306432343732353666393863 -32636165633363383266636637643836333731626265613863363630313430613561626530303730 -38363463373464333833666632366636666464306265313639343963376265323438333837333733 -30376334313062316264393663383134346362613265356462333066666535393666383936363730 -62656334373438383139343735376538653738636530333337346633386331326565323462646236 -31626431353265383766336564303734316165373062343036643362353364313335613237623233 -31363537343033313138646338616532363161626133626137396638366431333266306539313134 -30663338323366353866316434313661666137343634306231643533373636613431356337623165 -64373338313665373635646136393736303030626134313863306634623463653038386566313030 -33376130383161313034323563656237626430663362323265623766393830363032653934306431 -30363364393663666131653536356135303536663966633564323135326663613663313739303635 -62373862383030656136633338396166616431396466353432323630366436313339623430393262 -31373938653135353231323666396166303466626233626261653631653132306561336463663366 -32626335346431333632363061643862353736393435336536303536323736303539376566323838 -64333163633639383366646165633231326562353933323838663639663835343936373235363335 -633636623866386566613530633861373364 +30373739346533376636623934363634373734316230386233333066666662656566613834643062 +3234313933626566663563646539393132323466643564350a303932373831386564613738663865 +30393439643239316639303933376536353230393966373234336231306438386338663464333439 +3831353636323437650a333837366433396236383333303738363862623730636539616330666663 +62623839616131623837336537636434353237666434386437633066346365646437366363356333 +32356135613062366566323637383730343230623937646661666464343436666533643838333664 +38303039633435366334613934653639386661663065386334383765393237386663353865383830 +31366138383834643438303535383134306430383739326233643438383432366539393539386637 +32313635383661303435643936363633326662306235303764643163323965303165353065313530 +39613766343736653163366434383832303539646633636134313134383733303632633130376131 +65313862653437313534663164383561653033313663383637363865386365663639306137626465 +63653266636433333362356261653965313261666338323333393339353530356466653463303634 +38313836663465643361316363363364343665356565303836356433363461383331336237376561 +37316431396364626431313530306466663238303736623335336132663236353864663066383639 +34386261353664336332363032333130613963613539623965303261363339633064663739326532 +66323338346134303762646134663336356666326135346264343931653035323136333031323331 +38393861383136653866666138626639613837616264633031343639323036653563383661633434 +30343136636632613838333538393466316664316261343766343635353132666233633936363931 +62313430323230616138643463643239373465313435663430343433643335363362303264366464 +61663034353165323237636465303131346366626361643531616266633435303065613434396633 +6663