add grafana tempo

.env.example

@@ -1,12 +1,18 @@
 COMPOSE_PROFILES=grafana
 
-GRAFANA_VERSION=9.1.7
+GRAFANA_VERSION=9.5.6
 GRAFANA_HOST=grafana.docker.localhost
 
 LOKI_STACK_VERSION=2.6.0
 LOKI_HOST=loki.docker.localhost
 LOKI_PROMTAIL_HOST=promtail.docker.localhost
 
+TEMPO_VERSION=2.3.1
+TEMPO_CONFIG_FILE=/etc/tempo/tempo.yml
+TEMPO_SERVER_LOG_LEVEL=info
+TEMPO_STORAGE_PATH=/opt/tempo
+TEMPO_PROMETHEUS_URL=prometheus:9090
+
 PROMETHEUS_VERSION=v2.43.0
 PROMETHEUS_NODE_EXPORTER_VERSION=v1.5.0
 PROMETHEUS_HOST=prometheus.docker.localhost

docker-compose.yml

@@ -29,6 +29,26 @@ services:
     command: --config.file=/loki/config/config.yml
     profiles:
       - grafana
+  
+  tempo:
+    image: grafana/tempo:${TEMPO_VERSION}
+    command: 
+      - "-config.file=${TEMPO_CONFIG_FILE:-/etc/tempo/tempo.yml}"
+      - "-config.expand-env=true"
+    expose:
+      - 3200 # tempo
+      - 9095 # tempo grpc
+      - 4317 # otlp grpc
+      - 4318 # otlp http
+    environment:
+      TEMPO_SERVER_LOG_LEVEL: ${TEMPO_SERVER_LOG_LEVEL:-info}
+      TEMPO_STORAGE_PATH: ${TEMPO_STORAGE_PATH:-/opt/tempo}
+      TEMPO_PROMETHEUS_URL: ${TEMPO_PROMETHEUS_URL}
+    volumes:
+      - ./tempo:/etc/tempo
+      - tempo-local-storage:${TEMPO_STORAGE_PATH:-/opt/tempo}
+    profiles:
+      - tempo
 
   promtail:
     image: grafana/promtail:${LOKI_STACK_VERSION}
@@ -85,4 +105,5 @@ services:
 
 volumes:
   grafana-storage:
+  tempo-local-storage:
   prometheus-data:

grafana/provisioning/datasources/datasource.yml

@@ -20,3 +20,9 @@ datasources:
     url: http://prometheus:9090
     editable: true
     version: 1
+  - name: Tempo
+    type: tempo
+    orgId: 1
+    access: proxy
+    url: http://tempo:3200
+    editable: true

tempo/tempo.yml

@@ -0,0 +1,56 @@
+---
+stream_over_http_enabled: true
+
+server:
+  http_listen_port: 3200
+  log_level: "${TEMPO_SERVER_LOG_LEVEL}"
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+# this configuration will listen on all ports and protocols that tempo is capable of.
+# the receives all come from the OpenTelemetry collector.  more configuration information can
+# be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
+#
+# for a production deployment you should only enable the receivers you need!
+distributor:
+  receivers:                           
+    otlp:
+      protocols:
+        http:
+        grpc:
+
+ingester:
+  max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 1h # overall Tempo trace retention. set for demo purposes
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: "${TEMPO_STORAGE_PATH}/generator/wal"
+    remote_write:
+      - url: http://${TEMPO_PROMETHEUS_URL}/api/v1/write
+        send_exemplars: true
+
+storage:
+  trace:
+    backend: local # backend configuration to use
+    wal:
+      path: "${TEMPO_STORAGE_PATH}/wal" # where to store the the wal locally
+    local:
+      path: "${TEMPO_STORAGE_PATH}/blocks"
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics] # enables metrics generator