From 83a2756f3ce40e2e4e3fe4243749be188750ca20 Mon Sep 17 00:00:00 2001 From: dannc Date: Mon, 5 Feb 2024 14:46:29 +0700 Subject: [PATCH] [feature] replace promtail with grafana agent improve grafana provisioning --- .env.example | 4 ++ agent/agent.yaml | 49 +++++++++++++++++++ docker-compose.yml | 36 +++++++------- .../provisioning/datasources/datasource.yml | 15 ++++++ promtail/config.yml | 30 ------------ readme.md | 23 +++------ 6 files changed, 93 insertions(+), 64 deletions(-) create mode 100644 agent/agent.yaml delete mode 100644 promtail/config.yml diff --git a/.env.example b/.env.example index 59b6ea8..e36bf58 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,10 @@ LOKI_STACK_VERSION=2.6.0 LOKI_HOST=loki.docker.localhost LOKI_PROMTAIL_HOST=promtail.docker.localhost +AGENT_VERSION=v0.39.0 +AGENT_LOKI_URL=loki:3100 +AGENT_TEMPO_URL=tempo:4317 + TEMPO_VERSION=2.3.1 TEMPO_CONFIG_FILE=/etc/tempo/tempo.yml TEMPO_SERVER_LOG_LEVEL=info diff --git a/agent/agent.yaml b/agent/agent.yaml new file mode 100644 index 0000000..22f32ed --- /dev/null +++ b/agent/agent.yaml @@ -0,0 +1,49 @@ +logs: + configs: + - name: local + clients: + - url: "http://${AGENT_LOKI_URL}/loki/api/v1/push" + scrape_configs: + - docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 10s + job_name: docker + pipeline_stages: + - drop: + drop_counter_reason: reject_old_samples + older_than: 30m + - static_labels: + job: docker + relabel_configs: + - regex: /(.*) + source_labels: [__meta_docker_container_name] + target_label: container_name + - source_labels: [__meta_docker_container_id] + target_label: container_id + - source_labels: [__meta_docker_container_log_stream] + target_label: stream + - source_labels: [__meta_docker_container_label_com_docker_compose_project] + target_label: compose_project + - source_labels: [__meta_docker_container_label_org_label_schema_vcs_ref] + target_label: vcs_ref + - source_labels: [__meta_docker_container_label_org_label_schema_version] + target_label: image_version + positions_directory: /opt/positions + +traces: + configs: + - name: tempo + remote_write: + - endpoint: "http://${AGENT_TEMPO_URL}" + insecure: true + receivers: + otlp: + protocols: + http: + grpc: + automatic_logging: + backend: "logs_instance" + logs_instance_name: "local" + roots: true +server: + log_level: info diff --git a/docker-compose.yml b/docker-compose.yml index 1cdef0f..8eb5359 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,10 +29,26 @@ services: command: --config.file=/loki/config/config.yml profiles: - grafana - + + agent: + image: grafana/agent:${AGENT_VERSION} + environment: + WAL_DATA_DIRECTORY: /opt/agent/data + CONFIG_FILE_PATH: /etc/agent/agent.yaml + AGENT_LOKI_URL: "${AGENT_LOKI_URL}" + AGENT_TEMPO_URL: "${AGENT_TEMPO_URL}" + volumes: + - ./agent:/etc/agent/ + - /var/run/docker.sock:/var/run/docker.sock:ro + command: + - "-config.file=/etc/agent/agent.yaml" + - "-config.expand-env" + profiles: + - agent + tempo: image: grafana/tempo:${TEMPO_VERSION} - command: + command: - "-config.file=${TEMPO_CONFIG_FILE:-/etc/tempo/tempo.yml}" - "-config.expand-env=true" expose: @@ -50,22 +66,6 @@ services: profiles: - tempo - promtail: - image: grafana/promtail:${LOKI_STACK_VERSION} - labels: - - "traefik.enable=true" - - "traefik.http.routers.grafana.rule=Host(`${LOKI_PROMTAIL_HOST}`)" - - "traefik.http.services.grafana.loadbalancer.server.port=9080" - expose: - - 9080 - volumes: - - /var/lib/docker/containers:/var/lib/docker/containers:ro - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./promtail:/opt/promtail - command: -config.file=/opt/promtail/config.yml - profiles: - - promtail - prometheus: image: prom/prometheus:${PROMETHEUS_VERSION} labels: diff --git a/grafana/provisioning/datasources/datasource.yml b/grafana/provisioning/datasources/datasource.yml index 641d2be..ca98678 100644 --- a/grafana/provisioning/datasources/datasource.yml +++ b/grafana/provisioning/datasources/datasource.yml @@ -8,6 +8,7 @@ deleteDatasources: datasources: - name: Loki type: loki + uid: loki-local orgId: 1 access: proxy url: http://loki:3100 @@ -16,13 +17,27 @@ datasources: editable: true - name: Prometheus type: prometheus + uid: prometheus-local orgId: 1 url: http://prometheus:9090 editable: true version: 1 - name: Tempo type: tempo + uid: tempo-local orgId: 1 access: proxy url: http://tempo:3200 editable: true + jsonData: + tracesToLogsV2: + datasourceUid: "loki-local" + customQuery: true + filtersBySpanID: false + filtersByTraceID: false + query: "{job=\"docker\"} |= \"${__trace.traceId}\" |= \"${__span.spanId}\"" + spanEndTimeShift: "30m" + spanStartTimeShift: "-30m" + tags: [] + lokiSearch: + datasourceUid: "loki-local" diff --git a/promtail/config.yml b/promtail/config.yml deleted file mode 100644 index 4c787ad..0000000 --- a/promtail/config.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -server: - http_listen_address: 0.0.0.0 - http_listen_port: 9080 - -positions: - filename: /tmp/positions.yaml - -clients: - - url: http://loki:3100/loki/api/v1/push - -scrape_configs: - - job_name: docker - docker_sd_configs: - - host: unix:///var/run/docker.sock - refresh_interval: 5s - filters: - - name: label - values: ["loki.log=true"] - pipeline_stages: - - docker: {} - - static_labels: - job: "promtail" - relabel_configs: - - source_labels: ['__meta_docker_container_name'] - regex: '/(.*)' - target_label: 'container' - - source_labels: ['__meta_docker_container_log_stream'] - regex: '/(.*)' - target_label: 'source_stream' diff --git a/readme.md b/readme.md index 63e48d4..086ba88 100644 --- a/readme.md +++ b/readme.md @@ -6,23 +6,14 @@ 1. Скопировать [.env.example](/.env.example) в `.env` -#### With promtail +#### With grafana agent -1. Добавить новый лейбл к контейнеру, чьи логи необходимо отсылать в loki +1. Добавить в `COMPOSE_PROFILES` значение `agent` в файле [.env](/.env) +2. Запустить docker-compose-стэк: + ```shell + docker compose up -d + ``` - Например, достаточно просто добавить новый лейбл в `docker-compose`-файл искомого стека: - ```yaml - services: - my-service: - labels: - - "loki.log=true" - ``` - -2. Запустить весь стек `docker-compose` - - ```shell - docker compose up -d - ``` #### With docker plugin @@ -57,7 +48,7 @@ <<: *loki-log-config ``` -3. Убрать профиль `promtail` из значения `COMPOSE_PROFILES` в файле [.env](/.env) +3. Убрать профиль `agent` из значения `COMPOSE_PROFILES` в файле [.env](/.env) 4. Запустить стек `grafana` + `loki` ```shell