Move prometheus to on disk

playbooks/nodes/observability-hub.yml

@@ -6,8 +6,8 @@
     # - docker/install
     # - docker/portainer
     # - observability/grafana
-    # - observability/prometheus
-    - observability/beszel-hub
+    - observability/prometheus
+    # - observability/beszel-hub
   tasks:
     # - name: Deploy promlens
     #   community.docker.docker_container:

playbooks/server/base.yml

@@ -4,7 +4,7 @@
   become: true
   roles:
     # - role: observability/prometheus-node-exporter
-    - role: observability/beszel-agent
+    # - role: observability/beszel-agent
     # - role: docker/install
     # - role: docker/portainer
     # - role: server/setup/sshkey

playbooks/util/prometheus-config.yml

@@ -6,7 +6,7 @@
     - name: Generate Prometheus config from template
       ansible.builtin.template:
         src: templates/prometheus.yml.j2
-        dest: "{{ container_data_base_path }}/prometheus/prometheus.yml" 
+        dest: "/home/docker/prometheus/prometheus.yml" 
 
     - name: Restart prometheus container
       community.docker.docker_container:

playbooks/util/templates/prometheus.yml.j2

@@ -1,7 +1,7 @@
 #jinja2: trim_blocks: True, lstrip_blocks: True
 
 global:
-  scrape_interval: "15s"
+  scrape_interval: "5s"
 
 scrape_configs:
   - job_name: node
@@ -11,5 +11,5 @@ scrape_configs:
       {% for host in groups['vms'] %}
       - targets: ['{{ hostvars[host].ansible_host }}:9100']
         labels:
-          instance_name: '{{ host }}'
+          name: '{{ host }}'
       {% endfor %}

roles/observability/beszel-hub/tasks/main.yml

@@ -3,6 +3,7 @@
     name: beszel
     image: henrygd/beszel:latest
     restart_policy: unless-stopped
+    state: started
     ports:
       - 8090:8090
     volumes:
@@ -10,11 +11,12 @@
       - ./beszel_socket:/beszel_socket
 
 - name: Deploy local agent
-  community.docker.docker_config:
+  community.docker.docker_container:
     name: beszel-agent
     image: henrygd/beszel-agent:latest
     restart_policy: unless-stopped
     network_mode: host
+    state: started
     volumes:
       - ./beszel_socket:/beszel_socket
       - /var/run/docker.sock:/var/run/docker.sock:ro

roles/observability/prometheus-node-exporter/tasks/main.yml

@@ -4,6 +4,7 @@
     image: quay.io/prometheus/node-exporter:latest
     command: "--path.rootfs=/host"
     state: started
+    restart: true
     network_mode: host
     pid_mode: host
     restart_policy: unless-stopped

roles/observability/prometheus/tasks/main.yml

@@ -1,12 +1,12 @@
-- name: Create data folder on NAS
-  ansible.builtin.include_role:
-    role: docker/container-data
-  vars:
-    dir_name: "prometheus"
+- name: Create data folder
+  ansible.builtin.file:
+    dest: "/home/docker/prometheus/data"
+    state: directory
+    mode: '0777'
 
 - name: Set facts
   ansible.builtin.set_fact:
-    data_dir: "{{ container_data_base_path }}/prometheus" 
+    data_dir: "/home/docker/prometheus" 
 
 - name: Create config file
   ansible.builtin.file:
@@ -19,11 +19,12 @@
     name: prometheus
     image: prom/prometheus
     state: started
+    restart: true
     restart_policy: unless-stopped
     # env:
     #   GF_SERVER_ROOT_URL: "https://dash.lan.thegrind.dev" 
     ports:
       - '9090:9090'
     volumes:
-      - "{{ data_dir }}:/prometheus"
+      - "{{ data_dir }}/data:/prometheus"
       - "{{ data_dir }}/prometheus.yml:/etc/prometheus/prometheus.yml"