[feature] replace promtail with grafana agent

improve grafana provisioning

.env.example

@@ -7,6 +7,10 @@ LOKI_STACK_VERSION=2.6.0
 LOKI_HOST=loki.docker.localhost
 LOKI_PROMTAIL_HOST=promtail.docker.localhost
 
+AGENT_VERSION=v0.39.0
+AGENT_LOKI_URL=loki:3100
+AGENT_TEMPO_URL=tempo:4317
+
 TEMPO_VERSION=2.3.1
 TEMPO_CONFIG_FILE=/etc/tempo/tempo.yml
 TEMPO_SERVER_LOG_LEVEL=info

agent/agent.yaml

@@ -0,0 +1,49 @@
+logs:
+  configs:
+    - name: local
+      clients:
+        - url: "http://${AGENT_LOKI_URL}/loki/api/v1/push"
+      scrape_configs:
+        - docker_sd_configs:
+            - host: unix:///var/run/docker.sock
+              refresh_interval: 10s
+          job_name: docker
+          pipeline_stages:
+            - drop:
+                drop_counter_reason: reject_old_samples
+                older_than: 30m
+            - static_labels:
+                job: docker
+          relabel_configs:
+            - regex: /(.*)
+              source_labels: [__meta_docker_container_name]
+              target_label: container_name
+            - source_labels: [__meta_docker_container_id]
+              target_label: container_id
+            - source_labels: [__meta_docker_container_log_stream]
+              target_label: stream
+            - source_labels: [__meta_docker_container_label_com_docker_compose_project]
+              target_label: compose_project
+            - source_labels: [__meta_docker_container_label_org_label_schema_vcs_ref]
+              target_label: vcs_ref
+            - source_labels: [__meta_docker_container_label_org_label_schema_version]
+              target_label: image_version
+  positions_directory: /opt/positions
+
+traces:
+  configs:
+    - name: tempo
+      remote_write:
+        - endpoint: "http://${AGENT_TEMPO_URL}"
+          insecure: true
+      receivers:
+        otlp:
+          protocols:
+            http:
+            grpc:
+      automatic_logging:
+        backend: "logs_instance"
+        logs_instance_name: "local"
+        roots: true
+server:
+  log_level: info

docker-compose.yml

@@ -30,6 +30,22 @@ services:
     profiles:
       - grafana
 
+  agent:
+    image: grafana/agent:${AGENT_VERSION}
+    environment:
+      WAL_DATA_DIRECTORY: /opt/agent/data
+      CONFIG_FILE_PATH: /etc/agent/agent.yaml
+      AGENT_LOKI_URL: "${AGENT_LOKI_URL}"
+      AGENT_TEMPO_URL: "${AGENT_TEMPO_URL}"
+    volumes:
+      - ./agent:/etc/agent/
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    command:
+      - "-config.file=/etc/agent/agent.yaml"
+      - "-config.expand-env"
+    profiles:
+      - agent
+
   tempo:
     image: grafana/tempo:${TEMPO_VERSION}
     command:
@@ -50,22 +66,6 @@ services:
     profiles:
       - tempo
 
-  promtail:
-    image: grafana/promtail:${LOKI_STACK_VERSION}
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.grafana.rule=Host(`${LOKI_PROMTAIL_HOST}`)"
-      - "traefik.http.services.grafana.loadbalancer.server.port=9080"
-    expose:
-      - 9080
-    volumes:
-      - /var/lib/docker/containers:/var/lib/docker/containers:ro
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-      - ./promtail:/opt/promtail
-    command: -config.file=/opt/promtail/config.yml
-    profiles:
-      - promtail
-
   prometheus:
     image: prom/prometheus:${PROMETHEUS_VERSION}
     labels:

grafana/provisioning/datasources/datasource.yml

@@ -8,6 +8,7 @@ deleteDatasources:
 datasources:
   - name: Loki
     type: loki
+    uid: loki-local
     orgId: 1
     access: proxy
     url: http://loki:3100
@@ -16,13 +17,27 @@ datasources:
     editable: true
   - name: Prometheus
     type: prometheus
+    uid: prometheus-local
     orgId: 1
     url: http://prometheus:9090
     editable: true
     version: 1
   - name: Tempo
     type: tempo
+    uid: tempo-local
     orgId: 1
     access: proxy
     url: http://tempo:3200
     editable: true
+    jsonData:
+      tracesToLogsV2:
+        datasourceUid: "loki-local"
+        customQuery: true
+        filtersBySpanID: false
+        filtersByTraceID: false
+        query: "{job=\"docker\"} |= \"${__trace.traceId}\" |= \"${__span.spanId}\""
+        spanEndTimeShift: "30m"
+        spanStartTimeShift: "-30m"
+        tags: []
+      lokiSearch:
+        datasourceUid: "loki-local"

promtail/config.yml

@@ -1,30 +0,0 @@
----
-server:
-  http_listen_address: 0.0.0.0
-  http_listen_port: 9080
-
-positions:
-  filename: /tmp/positions.yaml
-
-clients:
-  - url: http://loki:3100/loki/api/v1/push
-
-scrape_configs:
-  - job_name: docker
-    docker_sd_configs:
-      - host: unix:///var/run/docker.sock
-        refresh_interval: 5s
-        filters:
-        - name: label
-          values: ["loki.log=true"]
-    pipeline_stages:
-      - docker: {}
-      - static_labels:
-          job: "promtail"
-    relabel_configs:
-      - source_labels: ['__meta_docker_container_name']
-        regex: '/(.*)'
-        target_label: 'container'
-      - source_labels: ['__meta_docker_container_log_stream']
-        regex: '/(.*)'
-        target_label: 'source_stream'

readme.md

@@ -6,23 +6,14 @@
 
 1. Скопировать [.env.example](/.env.example) в `.env`
 
-#### With promtail
+#### With grafana agent
 
-1. Добавить новый лейбл к контейнеру, чьи логи необходимо отсылать в loki
+1. Добавить в `COMPOSE_PROFILES` значение `agent` в файле [.env](/.env)
+2. Запустить docker-compose-стэк:
+  ```shell
+  docker compose up -d
+  ```
 
-	Например, достаточно просто добавить новый лейбл в `docker-compose`-файл искомого стека:
-	```yaml
-	services:
-	  my-service:
-	    labels:
-	      - "loki.log=true"
-	```
-
-2. Запустить весь стек `docker-compose`
-
-	```shell
-	docker compose up -d
-	```
 
 #### With docker plugin
 
@@ -57,7 +48,7 @@
 	  	<<: *loki-log-config
 	```
 
-3. Убрать профиль `promtail` из значения `COMPOSE_PROFILES` в файле [.env](/.env)
+3. Убрать профиль `agent` из значения `COMPOSE_PROFILES` в файле [.env](/.env)
 4. Запустить стек `grafana` + `loki`
 
 	```shell