From 82b1a2abc2978e2779ce1753f69ca98e84dea7d6 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Thu, 10 Nov 2016 13:40:27 -0500 Subject: [PATCH] Add graph transformation recording The external api provided here is simply dag.Graph.SetDebugWriter(io.Writer). When a writer is provided to a Graph, it will immediately encode itself to the stream, and subsequently encode any additional transformations to the graph. This will allow easier logging of graph transformations without writing complete graphs to the logs at every step. Since the marshalGraph can also be dot encoded, this will allow translation from the JSON logs to dot graphs. --- dag/dag.go | 8 + dag/graph.go | 38 +++-- dag/marshal.go | 360 +++++++++++++++++++++++++++++++++++++++----- dag/marshal_test.go | 175 +++++++++++++++++++++ 4 files changed, 531 insertions(+), 50 deletions(-) create mode 100644 dag/marshal_test.go diff --git a/dag/dag.go b/dag/dag.go index b58b407177..5d1ca8d954 100644 --- a/dag/dag.go +++ b/dag/dag.go @@ -103,6 +103,8 @@ func (g *AcyclicGraph) TransitiveReduction() { // v such that the edge (u,v) exists (v is a direct descendant of u). // // For each v-prime reachable from v, remove the edge (u, v-prime). + defer g.debug.BeginReduction().End() + for _, u := range g.Vertices() { uTargets := g.DownEdges(u) vs := AsVertexList(g.DownEdges(u)) @@ -165,6 +167,8 @@ func (g *AcyclicGraph) Cycles() [][]Vertex { // This will walk nodes in parallel if it can. Because the walk is done // in parallel, the error returned will be a multierror. func (g *AcyclicGraph) Walk(cb WalkFunc) error { + defer g.debug.BeginWalk().End() + // Cache the vertices since we use it multiple times vertices := g.Vertices() @@ -274,6 +278,8 @@ type vertexAtDepth struct { // the vertices in start. This is not exported now but it would make sense // to export this publicly at some point. func (g *AcyclicGraph) DepthFirstWalk(start []Vertex, f DepthWalkFunc) error { + defer g.debug.BeginDepthFirstWalk().End() + seen := make(map[Vertex]struct{}) frontier := make([]*vertexAtDepth, len(start)) for i, v := range start { @@ -316,6 +322,8 @@ func (g *AcyclicGraph) DepthFirstWalk(start []Vertex, f DepthWalkFunc) error { // reverseDepthFirstWalk does a depth-first walk _up_ the graph starting from // the vertices in start. func (g *AcyclicGraph) ReverseDepthFirstWalk(start []Vertex, f DepthWalkFunc) error { + defer g.debug.BeginReverseDepthFirstWalk().End() + seen := make(map[Vertex]struct{}) frontier := make([]*vertexAtDepth, len(start)) for i, v := range start { diff --git a/dag/graph.go b/dag/graph.go index ff9c9bd894..4e4f5be284 100644 --- a/dag/graph.go +++ b/dag/graph.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "io" "sort" "sync" ) @@ -15,6 +16,9 @@ type Graph struct { downEdges map[interface{}]*Set upEdges map[interface{}]*Set once sync.Once + + // JSON encoder for recording debug information + debug *encoder } // Subgrapher allows a Vertex to be a Graph itself, by returning a Grapher. @@ -106,6 +110,7 @@ func (g *Graph) HasEdge(e Edge) bool { func (g *Graph) Add(v Vertex) Vertex { g.once.Do(g.init) g.vertices.Add(v) + g.debug.Add(v) return v } @@ -114,6 +119,7 @@ func (g *Graph) Add(v Vertex) Vertex { func (g *Graph) Remove(v Vertex) Vertex { // Delete the vertex itself g.vertices.Delete(v) + g.debug.Remove(v) // Delete the edges to non-existent things for _, target := range g.DownEdges(v).List() { @@ -135,6 +141,8 @@ func (g *Graph) Replace(original, replacement Vertex) bool { return false } + defer g.debug.BeginReplace().End() + // If they're the same, then don't do anything if original == replacement { return true @@ -158,6 +166,7 @@ func (g *Graph) Replace(original, replacement Vertex) bool { // RemoveEdge removes an edge from the graph. func (g *Graph) RemoveEdge(edge Edge) { g.once.Do(g.init) + g.debug.RemoveEdge(edge) // Delete the edge from the set g.edges.Delete(edge) @@ -189,6 +198,7 @@ func (g *Graph) UpEdges(v Vertex) *Set { // value of the edge itself. func (g *Graph) Connect(edge Edge) { g.once.Do(g.init) + g.debug.Connect(edge) source := edge.Source() target := edge.Target() @@ -301,15 +311,6 @@ func (g *Graph) String() string { return buf.String() } -func (g *Graph) Dot(opts *DotOpts) []byte { - return newMarshalGraph("", g).Dot(opts) -} - -func (g *Graph) MarshalJSON() ([]byte, error) { - dg := newMarshalGraph("", g) - return json.MarshalIndent(dg, "", " ") -} - func (g *Graph) init() { g.vertices = new(Set) g.edges = new(Set) @@ -317,6 +318,25 @@ func (g *Graph) init() { g.upEdges = make(map[interface{}]*Set) } +// Dot returns a dot-formatted representation of the Graph. +func (g *Graph) Dot(opts *DotOpts) []byte { + return newMarshalGraph("", g).Dot(opts) +} + +// MarshalJSON returns a JSON representation of the entire Graph. +func (g *Graph) MarshalJSON() ([]byte, error) { + dg := newMarshalGraph("root", g) + return json.MarshalIndent(dg, "", " ") +} + +// SetDebugWriter sets the io.Writer where the Graph will record debug +// information. After this is set, the graph will immediately encode itself to +// the stream, and continue to record all subsequent operations. +func (g *Graph) SetDebugWriter(w io.Writer) { + g.debug = &encoder{w} + g.debug.Encode(newMarshalGraph("root", g)) +} + // VertexName returns the name of a vertex. func VertexName(raw Vertex) string { switch v := raw.(type) { diff --git a/dag/marshal.go b/dag/marshal.go index 95790ba3d7..722fb4b930 100644 --- a/dag/marshal.go +++ b/dag/marshal.go @@ -1,7 +1,10 @@ package dag import ( + "encoding/json" "fmt" + "io" + "log" "reflect" "sort" "strconv" @@ -9,6 +12,10 @@ import ( // the marshal* structs are for serialization of the graph data. type marshalGraph struct { + // Type is always "Graph", for identification as a top level object in the + // JSON stream. + Type string + // Each marshal structure requires a unique ID so that it can be referenced // by other structures. ID string `json:",omitempty"` @@ -33,6 +40,36 @@ type marshalGraph struct { Cycles [][]*marshalVertex `json:",omitempty"` } +// The add, remove, connect, removeEdge methods mirror the basic Graph +// manipulations to reconstruct a marshalGraph from a debug log. +func (g *marshalGraph) add(v *marshalVertex) { + g.Vertices = append(g.Vertices, v) + sort.Sort(vertices(g.Vertices)) +} + +func (g *marshalGraph) remove(v *marshalVertex) { + for i, existing := range g.Vertices { + if v.ID == existing.ID { + g.Vertices = append(g.Vertices[:i], g.Vertices[i+1:]...) + return + } + } +} + +func (g *marshalGraph) connect(e *marshalEdge) { + g.Edges = append(g.Edges, e) + sort.Sort(edges(g.Edges)) +} + +func (g *marshalGraph) removeEdge(e *marshalEdge) { + for i, existing := range g.Edges { + if e.Source == existing.Source && e.Target == existing.Target { + g.Edges = append(g.Edges[:i], g.Edges[i+1:]...) + return + } + } +} + func (g *marshalGraph) vertexByID(id string) *marshalVertex { for _, v := range g.Vertices { if id == v.ID { @@ -57,6 +94,27 @@ type marshalVertex struct { graphNodeDotter bool } +func newMarshalVertex(v Vertex) *marshalVertex { + return &marshalVertex{ + ID: marshalVertexID(v), + Name: VertexName(v), + Attrs: make(map[string]string), + graphNodeDotter: isDotter(v), + } +} + +func isDotter(v Vertex) bool { + dn, isDotter := v.(GraphNodeDotter) + dotOpts := &DotOpts{ + Verbose: true, + DrawCycles: true, + } + if isDotter && dn.DotNode("fake", dotOpts) == nil { + isDotter = false + } + return isDotter +} + // vertices is a sort.Interface implementation for sorting vertices by ID type vertices []*marshalVertex @@ -75,6 +133,15 @@ type marshalEdge struct { Attrs map[string]string `json:",omitempty"` } +func newMarshalEdge(e Edge) *marshalEdge { + return &marshalEdge{ + Name: fmt.Sprintf("%s|%s", VertexName(e.Source()), VertexName(e.Target())), + Source: marshalVertexID(e.Source()), + Target: marshalVertexID(e.Target()), + Attrs: make(map[string]string), + } +} + // edges is a sort.Interface implementation for sorting edges by Source ID type edges []*marshalEdge @@ -84,69 +151,42 @@ func (e edges) Swap(i, j int) { e[i], e[j] = e[j], e[i] } // build a marshalGraph structure from a *Graph func newMarshalGraph(name string, g *Graph) *marshalGraph { - dg := &marshalGraph{ + mg := &marshalGraph{ + Type: "Graph", Name: name, Attrs: make(map[string]string), } for _, v := range g.Vertices() { - // We only care about nodes that yield non-empty Dot strings. - dn, isDotter := v.(GraphNodeDotter) - dotOpts := &DotOpts{ - Verbose: true, - DrawCycles: true, - } - if isDotter && dn.DotNode("fake", dotOpts) == nil { - isDotter = false - } - id := marshalVertexID(v) if sg, ok := marshalSubgrapher(v); ok { - - sdg := newMarshalGraph(VertexName(v), sg) - sdg.ID = id - dg.Subgraphs = append(dg.Subgraphs, sdg) + smg := newMarshalGraph(VertexName(v), sg) + smg.ID = id + mg.Subgraphs = append(mg.Subgraphs, smg) } - dv := &marshalVertex{ - ID: id, - Name: VertexName(v), - Attrs: make(map[string]string), - graphNodeDotter: isDotter, - } - - dg.Vertices = append(dg.Vertices, dv) + mv := newMarshalVertex(v) + mg.Vertices = append(mg.Vertices, mv) } - sort.Sort(vertices(dg.Vertices)) + sort.Sort(vertices(mg.Vertices)) for _, e := range g.Edges() { - de := &marshalEdge{ - Name: fmt.Sprintf("%s|%s", VertexName(e.Source()), VertexName(e.Target())), - Source: marshalVertexID(e.Source()), - Target: marshalVertexID(e.Target()), - Attrs: make(map[string]string), - } - dg.Edges = append(dg.Edges, de) + mg.Edges = append(mg.Edges, newMarshalEdge(e)) } - sort.Sort(edges(dg.Edges)) + sort.Sort(edges(mg.Edges)) for _, c := range (&AcyclicGraph{*g}).Cycles() { var cycle []*marshalVertex for _, v := range c { - dv := &marshalVertex{ - ID: marshalVertexID(v), - Name: VertexName(v), - Attrs: make(map[string]string), - } - - cycle = append(cycle, dv) + mv := newMarshalVertex(v) + cycle = append(cycle, mv) } - dg.Cycles = append(dg.Cycles, cycle) + mg.Cycles = append(mg.Cycles, cycle) } - return dg + return mg } // Attempt to return a unique ID for any vertex. @@ -189,3 +229,241 @@ func marshalSubgrapher(v Vertex) (*Graph, bool) { return nil, false } + +// ender provides a way to call any End* method expression via an End method +type ender func() + +func (e ender) End() { e() } + +// encoder provides methods to write debug data to an io.Writer, and is a noop +// when no writer is present +type encoder struct { + w io.Writer +} + +// Encode is analogous to json.Encoder.Encode +func (e *encoder) Encode(i interface{}) { + if e == nil || e.w == nil { + return + } + + js, err := json.Marshal(i) + if err != nil { + log.Println("[ERROR] dag:", err) + return + } + js = append(js, '\n') + + _, err = e.w.Write(js) + if err != nil { + log.Println("[ERROR] dag:", err) + return + } +} + +func (e *encoder) Add(v Vertex) { + e.Encode(marshalTransform{ + Type: "Transform", + AddVertex: newMarshalVertex(v), + }) +} + +// Remove records the removal of Vertex v. +func (e *encoder) Remove(v Vertex) { + e.Encode(marshalTransform{ + Type: "Transform", + RemoveVertex: newMarshalVertex(v), + }) +} + +func (e *encoder) Connect(edge Edge) { + e.Encode(marshalTransform{ + Type: "Transform", + AddEdge: newMarshalEdge(edge), + }) +} + +func (e *encoder) RemoveEdge(edge Edge) { + e.Encode(marshalTransform{ + Type: "Transform", + RemoveEdge: newMarshalEdge(edge), + }) +} + +// BeginReplace marks the start of a replace operation, and returns the encoder +// to chain the EndReplace call. +func (e *encoder) BeginReplace() ender { + e.Encode(marshalOperation{ + Type: "Operation", + Begin: newString("Replace"), + }) + return e.EndReplace +} + +func (e *encoder) EndReplace() { + e.Encode(marshalOperation{ + Type: "Operation", + End: newString("Replace"), + }) +} + +// BeginReduction marks the start of a replace operation, and returns the encoder +// to chain the EndReduction call. +func (e *encoder) BeginReduction() ender { + e.Encode(marshalOperation{ + Type: "Operation", + Begin: newString("Reduction"), + }) + return e.EndReduction +} + +func (e *encoder) EndReduction() { + e.Encode(marshalOperation{ + Type: "Operation", + End: newString("Reduction"), + }) +} + +// BeginDepthFirstWalk marks the start of a replace operation, and returns the +// encoder to chain the EndDepthFirstWalk call. +func (e *encoder) BeginDepthFirstWalk() ender { + e.Encode(marshalOperation{ + Type: "Operation", + Begin: newString("DepthFirstWalk"), + }) + return e.EndDepthFirstWalk +} + +func (e *encoder) EndDepthFirstWalk() { + e.Encode(marshalOperation{ + Type: "Operation", + End: newString("DepthFirstWalk"), + }) +} + +// BeginReverseDepthFirstWalk marks the start of a replace operation, and +// returns the encoder to chain the EndReverseDepthFirstWalk call. +func (e *encoder) BeginReverseDepthFirstWalk() ender { + e.Encode(marshalOperation{ + Type: "Operation", + Begin: newString("ReverseDepthFirstWalk"), + }) + return e.EndReverseDepthFirstWalk +} + +func (e *encoder) EndReverseDepthFirstWalk() { + e.Encode(marshalOperation{ + Type: "Operation", + End: newString("ReverseDepthFirstWalk"), + }) +} + +// BeginWalk marks the start of a replace operation, and returns the encoder +// to chain the EndWalk call. +func (e *encoder) BeginWalk() ender { + e.Encode(marshalOperation{ + Type: "Operation", + Begin: newString("Walk"), + }) + return e.EndWalk +} + +func (e *encoder) EndWalk() { + e.Encode(marshalOperation{ + Type: "Operation", + End: newString("Walk"), + }) +} + +// structure for recording graph transformations +type marshalTransform struct { + // Type: "Transform" + Type string + AddEdge *marshalEdge `json:",omitempty"` + RemoveEdge *marshalEdge `json:",omitempty"` + AddVertex *marshalVertex `json:",omitempty"` + RemoveVertex *marshalVertex `json:",omitempty"` +} + +func (t marshalTransform) Transform(g *marshalGraph) { + switch { + case t.AddEdge != nil: + g.connect(t.AddEdge) + case t.RemoveEdge != nil: + g.removeEdge(t.RemoveEdge) + case t.AddVertex != nil: + g.add(t.AddVertex) + case t.RemoveVertex != nil: + g.remove(t.RemoveVertex) + } +} + +// this structure allows us to decode any object in the json stream for +// inspection, then re-decode it into a proper struct if needed. +type streamDecode struct { + Type string + Map map[string]interface{} + JSON []byte +} + +func (s *streamDecode) UnmarshalJSON(d []byte) error { + s.JSON = d + err := json.Unmarshal(d, &s.Map) + if err != nil { + return err + } + + if t, ok := s.Map["Type"]; ok { + s.Type, _ = t.(string) + } + return nil +} + +// structure for recording the beginning and end of any multi-step +// transformations. These are informational, and not required to reproduce the +// graph state. +type marshalOperation struct { + Type string + Begin *string `json:",omitempty"` + End *string `json:",omitempty"` +} + +func newBool(b bool) *bool { return &b } + +func newString(s string) *string { return &s } + +// decodeGraph decodes a marshalGraph from an encoded graph stream. +func decodeGraph(r io.Reader) (*marshalGraph, error) { + dec := json.NewDecoder(r) + + // a stream should always start with a graph + g := &marshalGraph{} + + err := dec.Decode(g) + if err != nil { + return nil, err + } + + // now replay any operations that occurred on the original graph + for dec.More() { + s := &streamDecode{} + err := dec.Decode(s) + if err != nil { + return g, err + } + + // the only Type we're concerned with here is Transform to complete the + // Graph + if s.Type != "Transform" { + continue + } + + t := &marshalTransform{} + err = json.Unmarshal(s.JSON, t) + if err != nil { + return g, err + } + t.Transform(g) + } + return g, nil +} diff --git a/dag/marshal_test.go b/dag/marshal_test.go new file mode 100644 index 0000000000..945d12f7c2 --- /dev/null +++ b/dag/marshal_test.go @@ -0,0 +1,175 @@ +package dag + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +func TestGraphDot_empty(t *testing.T) { + var g Graph + g.Add(1) + g.Add(2) + g.Add(3) + + actual := strings.TrimSpace(string(g.Dot(nil))) + expected := strings.TrimSpace(testGraphDotEmptyStr) + if actual != expected { + t.Fatalf("bad: %s", actual) + } +} + +func TestGraphDot_basic(t *testing.T) { + var g Graph + g.Add(1) + g.Add(2) + g.Add(3) + g.Connect(BasicEdge(1, 3)) + + actual := strings.TrimSpace(string(g.Dot(nil))) + expected := strings.TrimSpace(testGraphDotBasicStr) + if actual != expected { + t.Fatalf("bad: %s", actual) + } +} + +const testGraphDotBasicStr = `digraph { + compound = "true" + newrank = "true" + subgraph "root" { + "[root] 1" -> "[root] 3" + } +} +` + +const testGraphDotEmptyStr = `digraph { + compound = "true" + newrank = "true" + subgraph "root" { + } +}` + +func TestGraphJSON_empty(t *testing.T) { + var g Graph + g.Add(1) + g.Add(2) + g.Add(3) + + js, err := g.MarshalJSON() + if err != nil { + t.Fatal(err) + } + + actual := strings.TrimSpace(string(js)) + expected := strings.TrimSpace(testGraphJSONEmptyStr) + if actual != expected { + t.Fatalf("bad: %s", actual) + } +} + +func TestGraphJSON_basic(t *testing.T) { + var g Graph + g.Add(1) + g.Add(2) + g.Add(3) + g.Connect(BasicEdge(1, 3)) + + js, err := g.MarshalJSON() + if err != nil { + t.Fatal(err) + } + actual := strings.TrimSpace(string(js)) + expected := strings.TrimSpace(testGraphJSONBasicStr) + if actual != expected { + t.Fatalf("bad: %s", actual) + } +} + +// record some graph transformations, and make sure we get the same graph when +// they're replayed +func TestGraphJSON_basicRecord(t *testing.T) { + var g Graph + var buf bytes.Buffer + g.SetDebugWriter(&buf) + + g.Add(1) + g.Add(2) + g.Add(3) + g.Connect(BasicEdge(1, 2)) + g.Connect(BasicEdge(1, 3)) + g.Connect(BasicEdge(2, 3)) + (&AcyclicGraph{g}).TransitiveReduction() + + recorded := buf.Bytes() + // the Walk doesn't happen in a determined order, so just count operations + // for now to make sure we wrote stuff out. + if len(bytes.Split(recorded, []byte{'\n'})) != 17 { + t.Fatalf("bad: %s", recorded) + } + + original, err := g.MarshalJSON() + if err != nil { + t.Fatal(err) + } + + // replay the logs, and marshal the graph back out again + m, err := decodeGraph(bytes.NewReader(buf.Bytes())) + if err != nil { + t.Fatal(err) + } + + replayed, err := json.MarshalIndent(m, "", " ") + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(original, replayed) { + t.Fatalf("\noriginal: %s\nreplayed: %s", original, replayed) + } +} + +const testGraphJSONEmptyStr = `{ + "Type": "Graph", + "Name": "root", + "Vertices": [ + { + "ID": "1", + "Name": "1" + }, + { + "ID": "2", + "Name": "2" + }, + { + "ID": "3", + "Name": "3" + } + ] +}` + +const testGraphJSONBasicStr = `{ + "Type": "Graph", + "Name": "root", + "Vertices": [ + { + "ID": "1", + "Name": "1" + }, + { + "ID": "2", + "Name": "2" + }, + { + "ID": "3", + "Name": "3" + } + ], + "Edges": [ + { + "Name": "1|3", + "Source": "1", + "Target": "3" + } + ] +}`