Devenv: self-instrumentation (#63730)

* Devenv: self instrumentation

.github/CODEOWNERS

@@ -163,6 +163,7 @@
 # Logs code, developers environment
 /devenv/docker/blocks/loki* @grafana/observability-logs
 /devenv/docker/blocks/elastic* @grafana/observability-logs
+/devenv/docker/blocks/self-instrumentation* @grafana/observability-metrics
 
 /devenv/bulk-dashboards/ @grafana/dashboards-squad
 /devenv/bulk_alerting_dashboards/ @grafana/alerting-backend-product

devenv/datasources.yaml

@@ -248,6 +248,9 @@ datasources:
       - targetUID: gdev-zipkin
         label: "Zipkin traces"
         description: "Related traces stored in Zipkin"
+      - targetUID: gdev-tempo
+        label: "Tempo traces"
+        description: "Related traces stored in Tempo"
       - targetUID: gdev-prometheus
         label: "Logs to metrics"
         description: "Related metrics stored in Prometheus"
@@ -267,6 +270,10 @@ datasources:
           matcherRegex: "traceID=(\\w+)"
           url: "$${__value.raw}"
           datasourceUid: gdev-zipkin
+        - name: "traceID"
+          matcherRegex: "traceID=(\\w+)"
+          url: "$${__value.raw}"
+          datasourceUid: gdev-tempo
 
   - name: gdev-jaeger
     type: jaeger

devenv/docker/blocks/self-instrumentation/agent.yaml

@@ -0,0 +1,40 @@
+logs:
+  positions_directory: /tmp/loki-pos
+  configs:
+    - name: default
+      scrape_configs: []
+      clients:
+        - url: http://host.docker.internal:3100/loki/api/v1/push
+traces:
+  configs:
+    - name: default
+      remote_write:
+        - endpoint: http://host.docker.internal:3200
+          insecure: true
+      receivers:
+        otlp:
+          protocols:
+            grpc:
+            http:
+              cors:
+                allowed_origins:
+                  - "*"
+integrations:
+  app_agent_receiver_configs:
+    - autoscrape:
+        enable: false
+      instance: "frontend"
+      logs_instance: "default"
+      traces_instance: "default"
+      server:
+        host: 0.0.0.0
+        port: 12347
+        cors_allowed_origins:
+          - "*"
+      logs_labels: # labels to add to loki log record
+        app: frontend # static value
+        kind: # value will be taken from log items. exception, log, measurement, etc
+      logs_send_timeout: 5000
+      sourcemaps:
+        download: true # will download source file, extract source map location,
+        # download source map and use it to transform stack trace locations

devenv/docker/blocks/self-instrumentation/docker-compose.yaml

@@ -0,0 +1,45 @@
+  prometheus:
+    image: prom/prometheus:v2.45.0
+    ports:
+      - "9090:9090"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    command: >
+      --enable-feature=remote-write-receiver
+      --config.file=/etc/prometheus/prometheus.yml
+      --storage.tsdb.path=/prometheus
+    volumes:
+      - ./docker/blocks/self-instrumentation/prometheus.yaml:/etc/prometheus/prometheus.yml
+  loki:
+    image: grafana/loki:2.8.2
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+  promtail:
+    image: grafana/promtail:2.8.2
+    command: -config.file=/etc/promtail/promtail.yaml
+    volumes:
+      - ./docker/blocks/self-instrumentation/promtail.yaml:/etc/promtail/promtail.yaml
+      - ../data/log:/var/log/grafana
+  tempo:
+    image: grafana/tempo:2.1.1
+    command: -config.file=/etc/tempo.yaml
+    volumes:
+      - ./docker/blocks/self-instrumentation/tempo.yaml:/etc/tempo.yaml
+    ports:
+      - "14268:14268"
+      - "3200:3200"
+  agent:
+    image: grafana/agent:v0.34.3
+    entrypoint:
+      - /bin/grafana-agent
+      - -server.http.address=0.0.0.0:12345
+      - -config.file=/etc/agent/agent.yaml
+      - -enable-features=integrations-next
+    volumes:
+      - ./docker/blocks/self-instrumentation/agent.yaml:/etc/agent/agent.yaml
+    ports:
+      - "12345:12345"
+      - "12347:12347"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"

devenv/docker/blocks/self-instrumentation/prometheus.yaml

@@ -0,0 +1,9 @@
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: grafana
+    static_configs:
+      - targets:
+        - host.docker.internal:3000

devenv/docker/blocks/self-instrumentation/promtail.yaml

@@ -0,0 +1,18 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+- job_name: hosted-grafana/grafana
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: hosted-grafana/grafana
+      __path__: /var/log/grafana/*log

devenv/docker/blocks/self-instrumentation/readme.md

@@ -0,0 +1,19 @@
+# Self Instrumentation
+
+To run this source, in the Grafana repo root:
+
+```
+make devenv sources=self-instrumentation
+```
+
+This will setup Prometheus, Loki and Tempo.
+
+You then need to run Grafana with those added config:
+
+```ini
+[log.frontend]
+enabled = true
+
+[tracing.opentelemetry.jaeger]
+address = http://localhost:14268/api/traces
+```

devenv/docker/blocks/self-instrumentation/tempo.yaml

@@ -0,0 +1,44 @@
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:                           # this configuration will listen on all ports and protocols that tempo is capable of.
+    jaeger:                            # the receives all come from the OpenTelemetry collector.  more configuration information can
+      protocols:                       # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
+        thrift_http:                   #
+        grpc:                          # for a production deployment you should only enable the receivers you need!
+        thrift_binary:
+        thrift_compact:
+    zipkin:
+    otlp:
+      protocols:
+        http:
+        grpc:
+    opencensus:
+
+compactor:
+  compaction:
+    block_retention: 1h  
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /tmp/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+
+storage:
+  trace:
+    backend: local                     # backend configuration to use
+    wal:
+      path: /tmp/tempo/wal             # where to store the the wal locally
+    local:
+      path: /tmp/tempo/blocks
+
+overrides:
+  metrics_generator_processors: [service-graphs, span-metrics]
+