From 8c8fdadb783e63c271c970717ecb62c81af3bd22 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Mon, 11 Sep 2017 20:57:08 +0200
Subject: [PATCH] trace failed executions

---
 pkg/services/alerting/engine.go | 16 ++++++++++------
 pkg/tsdb/graphite/graphite.go   |  2 +-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go
index d7e9867db0a..e01a01360b5 100644
--- a/pkg/services/alerting/engine.go
+++ b/pkg/services/alerting/engine.go
@@ -2,9 +2,12 @@ package alerting
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	"github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/ext"
+	tlog "github.com/opentracing/opentracing-go/log"
 
 	"github.com/benbjohnson/clock"
 	"github.com/grafana/grafana/pkg/log"
@@ -101,20 +104,21 @@ func (e *Engine) processJob(grafanaCtx context.Context, job *Job) error {
 	}()
 
 	alertCtx, cancelFn := context.WithTimeout(context.Background(), alertTimeout)
+	span := opentracing.StartSpan("alert execution")
+	alertCtx = opentracing.ContextWithSpan(alertCtx, span)
 
 	job.Running = true
 	evalContext := NewEvalContext(alertCtx, job.Rule)
-
-	done := make(chan struct{})
-
-	span := opentracing.StartSpan("alerting")
-	alertCtx = opentracing.ContextWithSpan(alertCtx, span)
 	evalContext.Ctx = alertCtx
 
+	done := make(chan struct{})
 	go func() {
 		defer func() {
 			if err := recover(); err != nil {
 				e.log.Error("Alert Panic", "error", err, "stack", log.Stack(1))
+				ext.Error.Set(span, true)
+				span.LogFields(tlog.Error(fmt.Errorf("error: %v", err)))
+				span.Finish()
 				close(done)
 			}
 		}()
@@ -126,8 +130,8 @@ func (e *Engine) processJob(grafanaCtx context.Context, job *Job) error {
 		span.SetTag("dashboardId", evalContext.Rule.DashboardId)
 		span.SetTag("firing", evalContext.Firing)
 
-		close(done)
 		span.Finish()
+		close(done)
 	}()
 
 	var err error = nil
diff --git a/pkg/tsdb/graphite/graphite.go b/pkg/tsdb/graphite/graphite.go
index 7e7a25910d6..a3683d789f3 100644
--- a/pkg/tsdb/graphite/graphite.go
+++ b/pkg/tsdb/graphite/graphite.go
@@ -81,7 +81,7 @@ func (e *GraphiteExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice,
 		return result
 	}
 
-	span, ctx := opentracing.StartSpanFromContext(ctx, "alerting.graphite")
+	span, ctx := opentracing.StartSpanFromContext(ctx, "graphite query")
 	span.SetTag("target", target)
 	span.SetTag("from", from)
 	span.SetTag("until", until)