Ansible Monitoring: Integrate with Prometheus, Grafana & Alerting (Complete Guide)

---
- name: Deploy Prometheus
  hosts: monitoring
  become: true
  vars:
    prometheus_version: "2.53.0"
  tasks:
    - name: Create prometheus user
      ansible.builtin.user:
        name: prometheus
        system: true
        shell: /usr/sbin/nologin

    - name: Download Prometheus
      ansible.builtin.get_url:
        url: "https://github.com/prometheus/prometheus/releases/download/v{{ prometheus_version }}/prometheus-{{ prometheus_version }}.linux-amd64.tar.gz"
        dest: /tmp/prometheus.tar.gz

    - name: Extract Prometheus
      ansible.builtin.unarchive:
        src: /tmp/prometheus.tar.gz
        dest: /opt/
        remote_src: true

    - name: Deploy Prometheus config
      ansible.builtin.template:
        src: prometheus.yml.j2
        dest: /etc/prometheus/prometheus.yml
      notify: restart prometheus

    - name: Create systemd service
      ansible.builtin.template:
        src: prometheus.service.j2
        dest: /etc/systemd/system/prometheus.service
      notify: restart prometheus

    - name: Start Prometheus
      ansible.builtin.systemd:
        name: prometheus
        state: started
        enabled: true
        daemon_reload: true

  handlers:
    - name: restart prometheus
      ansible.builtin.systemd:
        name: prometheus
        state: restarted

- name: Deploy Node Exporter
  hosts: all
  become: true
  vars:
    node_exporter_version: "1.8.0"
  tasks:
    - name: Download Node Exporter
      ansible.builtin.get_url:
        url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/node_exporter-{{ node_exporter_version }}.linux-amd64.tar.gz"
        dest: /tmp/node_exporter.tar.gz

    - name: Extract and install
      ansible.builtin.unarchive:
        src: /tmp/node_exporter.tar.gz
        dest: /usr/local/bin/
        remote_src: true
        extra_opts: [--strip-components=1]
        creates: /usr/local/bin/node_exporter

    - name: Create systemd service
      ansible.builtin.copy:
        content: |
          [Unit]
          Description=Node Exporter
          After=network.target

          [Service]
          User=node_exporter
          ExecStart=/usr/local/bin/node_exporter
          Restart=always

          [Install]
          WantedBy=multi-user.target
        dest: /etc/systemd/system/node_exporter.service

    - name: Start Node Exporter
      ansible.builtin.systemd:
        name: node_exporter
        state: started
        enabled: true
        daemon_reload: true

- name: Deploy Grafana
  hosts: monitoring
  become: true
  tasks:
    - name: Add Grafana repository
      ansible.builtin.apt_repository:
        repo: "deb https://apt.grafana.com stable main"
        state: present

    - name: Add Grafana GPG key
      ansible.builtin.apt_key:
        url: https://apt.grafana.com/gpg.key
        state: present

    - name: Install Grafana
      ansible.builtin.apt:
        name: grafana
        state: present
        update_cache: true

    - name: Configure Grafana datasource
      ansible.builtin.copy:
        content: |
          apiVersion: 1
          datasources:
            - name: Prometheus
              type: prometheus
              url: http://localhost:9090
              isDefault: true
              access: proxy
        dest: /etc/grafana/provisioning/datasources/prometheus.yml

    - name: Start Grafana
      ansible.builtin.systemd:
        name: grafana-server
        state: started
        enabled: true

# ansible.cfg
[defaults]
callbacks_enabled = ansible.builtin.timer, ansible.builtin.profile_tasks

[defaults]
callbacks_enabled = ansible.builtin.json
log_path = /var/log/ansible/playbook.log

# Push playbook metrics to Prometheus Pushgateway
- name: Report playbook metrics
  hosts: localhost
  tasks:
    - name: Push execution metrics
      ansible.builtin.uri:
        url: "http://pushgateway:9091/metrics/job/ansible/instance/{{ playbook_dir | basename }}"
        method: POST
        body: |
          ansible_playbook_duration_seconds {{ ansible_date_time.epoch | int - start_time }}
          ansible_playbook_hosts_total {{ ansible_play_hosts | length }}
          ansible_playbook_success 1
        headers:
          Content-Type: text/plain

# Prometheus alerting rule
groups:
  - name: ansible
    rules:
      - alert: AnsiblePlaybookFailed
        expr: ansible_playbook_success == 0
        for: 0m
        labels:
          severity: critical
        annotations:
          summary: "Ansible playbook failed"

# In playbook — notify on failure
- name: Deploy application
  hosts: webservers
  tasks:
    - name: Deploy
      ansible.builtin.include_role:
        name: deploy
  
  post_tasks:
    - name: Notify success
      ansible.builtin.uri:
        url: "{{ slack_webhook }}"
        method: POST
        body_format: json
        body:
          text: "✅ Deployment to {{ ansible_play_hosts | length }} hosts completed successfully"
      run_once: true
      when: ansible_play_hosts_all | length == ansible_play_hosts | length
      delegate_to: localhost

- name: Auto-remediate common issues
  hosts: all
  tasks:
    - name: Check disk space
      ansible.builtin.shell: df -h / | tail -1 | awk '{print $5}' | tr -d '%'
      register: disk_usage
      changed_when: false

    - name: Clean up if disk usage > 85%
      when: disk_usage.stdout | int > 85
      block:
        - name: Clean apt cache
          ansible.builtin.apt:
            autoclean: true

        - name: Remove old logs
          ansible.builtin.shell: find /var/log -name "*.gz" -mtime +30 -delete
          changed_when: true

    - name: Check if services are running
      ansible.builtin.systemd:
        name: "{{ item }}"
        state: started
      loop:
        - nginx
        - postgresql
      register: service_results
      failed_when: false

    - name: Alert on service failures
      ansible.builtin.uri:
        url: "{{ alertmanager_webhook }}"
        method: POST
        body_format: json
        body:
          alerts:
            - labels:
                alertname: "ServiceDown"
                host: "{{ inventory_hostname }}"
                service: "{{ item.item }}"
              annotations:
                summary: "{{ item.item }} is not running on {{ inventory_hostname }}"
      loop: "{{ service_results.results }}"
      when: item.failed | default(false)
      delegate_to: localhost