diff --git a/dag/dag.go b/dag/dag.go index b58b407177..5d1ca8d954 100644 --- a/dag/dag.go +++ b/dag/dag.go @@ -103,6 +103,8 @@ func (g *AcyclicGraph) TransitiveReduction() { // v such that the edge (u,v) exists (v is a direct descendant of u). // // For each v-prime reachable from v, remove the edge (u, v-prime). + defer g.debug.BeginReduction().End() + for _, u := range g.Vertices() { uTargets := g.DownEdges(u) vs := AsVertexList(g.DownEdges(u)) @@ -165,6 +167,8 @@ func (g *AcyclicGraph) Cycles() [][]Vertex { // This will walk nodes in parallel if it can. Because the walk is done // in parallel, the error returned will be a multierror. func (g *AcyclicGraph) Walk(cb WalkFunc) error { + defer g.debug.BeginWalk().End() + // Cache the vertices since we use it multiple times vertices := g.Vertices() @@ -274,6 +278,8 @@ type vertexAtDepth struct { // the vertices in start. This is not exported now but it would make sense // to export this publicly at some point. func (g *AcyclicGraph) DepthFirstWalk(start []Vertex, f DepthWalkFunc) error { + defer g.debug.BeginDepthFirstWalk().End() + seen := make(map[Vertex]struct{}) frontier := make([]*vertexAtDepth, len(start)) for i, v := range start { @@ -316,6 +322,8 @@ func (g *AcyclicGraph) DepthFirstWalk(start []Vertex, f DepthWalkFunc) error { // reverseDepthFirstWalk does a depth-first walk _up_ the graph starting from // the vertices in start. func (g *AcyclicGraph) ReverseDepthFirstWalk(start []Vertex, f DepthWalkFunc) error { + defer g.debug.BeginReverseDepthFirstWalk().End() + seen := make(map[Vertex]struct{}) frontier := make([]*vertexAtDepth, len(start)) for i, v := range start { diff --git a/dag/graph.go b/dag/graph.go index ff9c9bd894..4e4f5be284 100644 --- a/dag/graph.go +++ b/dag/graph.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "io" "sort" "sync" ) @@ -15,6 +16,9 @@ type Graph struct { downEdges map[interface{}]*Set upEdges map[interface{}]*Set once sync.Once + + // JSON encoder for recording debug information + debug *encoder } // Subgrapher allows a Vertex to be a Graph itself, by returning a Grapher. @@ -106,6 +110,7 @@ func (g *Graph) HasEdge(e Edge) bool { func (g *Graph) Add(v Vertex) Vertex { g.once.Do(g.init) g.vertices.Add(v) + g.debug.Add(v) return v } @@ -114,6 +119,7 @@ func (g *Graph) Add(v Vertex) Vertex { func (g *Graph) Remove(v Vertex) Vertex { // Delete the vertex itself g.vertices.Delete(v) + g.debug.Remove(v) // Delete the edges to non-existent things for _, target := range g.DownEdges(v).List() { @@ -135,6 +141,8 @@ func (g *Graph) Replace(original, replacement Vertex) bool { return false } + defer g.debug.BeginReplace().End() + // If they're the same, then don't do anything if original == replacement { return true @@ -158,6 +166,7 @@ func (g *Graph) Replace(original, replacement Vertex) bool { // RemoveEdge removes an edge from the graph. func (g *Graph) RemoveEdge(edge Edge) { g.once.Do(g.init) + g.debug.RemoveEdge(edge) // Delete the edge from the set g.edges.Delete(edge) @@ -189,6 +198,7 @@ func (g *Graph) UpEdges(v Vertex) *Set { // value of the edge itself. func (g *Graph) Connect(edge Edge) { g.once.Do(g.init) + g.debug.Connect(edge) source := edge.Source() target := edge.Target() @@ -301,15 +311,6 @@ func (g *Graph) String() string { return buf.String() } -func (g *Graph) Dot(opts *DotOpts) []byte { - return newMarshalGraph("", g).Dot(opts) -} - -func (g *Graph) MarshalJSON() ([]byte, error) { - dg := newMarshalGraph("", g) - return json.MarshalIndent(dg, "", " ") -} - func (g *Graph) init() { g.vertices = new(Set) g.edges = new(Set) @@ -317,6 +318,25 @@ func (g *Graph) init() { g.upEdges = make(map[interface{}]*Set) } +// Dot returns a dot-formatted representation of the Graph. +func (g *Graph) Dot(opts *DotOpts) []byte { + return newMarshalGraph("", g).Dot(opts) +} + +// MarshalJSON returns a JSON representation of the entire Graph. +func (g *Graph) MarshalJSON() ([]byte, error) { + dg := newMarshalGraph("root", g) + return json.MarshalIndent(dg, "", " ") +} + +// SetDebugWriter sets the io.Writer where the Graph will record debug +// information. After this is set, the graph will immediately encode itself to +// the stream, and continue to record all subsequent operations. +func (g *Graph) SetDebugWriter(w io.Writer) { + g.debug = &encoder{w} + g.debug.Encode(newMarshalGraph("root", g)) +} + // VertexName returns the name of a vertex. func VertexName(raw Vertex) string { switch v := raw.(type) { diff --git a/dag/marshal.go b/dag/marshal.go index 95790ba3d7..722fb4b930 100644 --- a/dag/marshal.go +++ b/dag/marshal.go @@ -1,7 +1,10 @@ package dag import ( + "encoding/json" "fmt" + "io" + "log" "reflect" "sort" "strconv" @@ -9,6 +12,10 @@ import ( // the marshal* structs are for serialization of the graph data. type marshalGraph struct { + // Type is always "Graph", for identification as a top level object in the + // JSON stream. + Type string + // Each marshal structure requires a unique ID so that it can be referenced // by other structures. ID string `json:",omitempty"` @@ -33,6 +40,36 @@ type marshalGraph struct { Cycles [][]*marshalVertex `json:",omitempty"` } +// The add, remove, connect, removeEdge methods mirror the basic Graph +// manipulations to reconstruct a marshalGraph from a debug log. +func (g *marshalGraph) add(v *marshalVertex) { + g.Vertices = append(g.Vertices, v) + sort.Sort(vertices(g.Vertices)) +} + +func (g *marshalGraph) remove(v *marshalVertex) { + for i, existing := range g.Vertices { + if v.ID == existing.ID { + g.Vertices = append(g.Vertices[:i], g.Vertices[i+1:]...) + return + } + } +} + +func (g *marshalGraph) connect(e *marshalEdge) { + g.Edges = append(g.Edges, e) + sort.Sort(edges(g.Edges)) +} + +func (g *marshalGraph) removeEdge(e *marshalEdge) { + for i, existing := range g.Edges { + if e.Source == existing.Source && e.Target == existing.Target { + g.Edges = append(g.Edges[:i], g.Edges[i+1:]...) + return + } + } +} + func (g *marshalGraph) vertexByID(id string) *marshalVertex { for _, v := range g.Vertices { if id == v.ID { @@ -57,6 +94,27 @@ type marshalVertex struct { graphNodeDotter bool } +func newMarshalVertex(v Vertex) *marshalVertex { + return &marshalVertex{ + ID: marshalVertexID(v), + Name: VertexName(v), + Attrs: make(map[string]string), + graphNodeDotter: isDotter(v), + } +} + +func isDotter(v Vertex) bool { + dn, isDotter := v.(GraphNodeDotter) + dotOpts := &DotOpts{ + Verbose: true, + DrawCycles: true, + } + if isDotter && dn.DotNode("fake", dotOpts) == nil { + isDotter = false + } + return isDotter +} + // vertices is a sort.Interface implementation for sorting vertices by ID type vertices []*marshalVertex @@ -75,6 +133,15 @@ type marshalEdge struct { Attrs map[string]string `json:",omitempty"` } +func newMarshalEdge(e Edge) *marshalEdge { + return &marshalEdge{ + Name: fmt.Sprintf("%s|%s", VertexName(e.Source()), VertexName(e.Target())), + Source: marshalVertexID(e.Source()), + Target: marshalVertexID(e.Target()), + Attrs: make(map[string]string), + } +} + // edges is a sort.Interface implementation for sorting edges by Source ID type edges []*marshalEdge @@ -84,69 +151,42 @@ func (e edges) Swap(i, j int) { e[i], e[j] = e[j], e[i] } // build a marshalGraph structure from a *Graph func newMarshalGraph(name string, g *Graph) *marshalGraph { - dg := &marshalGraph{ + mg := &marshalGraph{ + Type: "Graph", Name: name, Attrs: make(map[string]string), } for _, v := range g.Vertices() { - // We only care about nodes that yield non-empty Dot strings. - dn, isDotter := v.(GraphNodeDotter) - dotOpts := &DotOpts{ - Verbose: true, - DrawCycles: true, - } - if isDotter && dn.DotNode("fake", dotOpts) == nil { - isDotter = false - } - id := marshalVertexID(v) if sg, ok := marshalSubgrapher(v); ok { - - sdg := newMarshalGraph(VertexName(v), sg) - sdg.ID = id - dg.Subgraphs = append(dg.Subgraphs, sdg) + smg := newMarshalGraph(VertexName(v), sg) + smg.ID = id + mg.Subgraphs = append(mg.Subgraphs, smg) } - dv := &marshalVertex{ - ID: id, - Name: VertexName(v), - Attrs: make(map[string]string), - graphNodeDotter: isDotter, - } - - dg.Vertices = append(dg.Vertices, dv) + mv := newMarshalVertex(v) + mg.Vertices = append(mg.Vertices, mv) } - sort.Sort(vertices(dg.Vertices)) + sort.Sort(vertices(mg.Vertices)) for _, e := range g.Edges() { - de := &marshalEdge{ - Name: fmt.Sprintf("%s|%s", VertexName(e.Source()), VertexName(e.Target())), - Source: marshalVertexID(e.Source()), - Target: marshalVertexID(e.Target()), - Attrs: make(map[string]string), - } - dg.Edges = append(dg.Edges, de) + mg.Edges = append(mg.Edges, newMarshalEdge(e)) } - sort.Sort(edges(dg.Edges)) + sort.Sort(edges(mg.Edges)) for _, c := range (&AcyclicGraph{*g}).Cycles() { var cycle []*marshalVertex for _, v := range c { - dv := &marshalVertex{ - ID: marshalVertexID(v), - Name: VertexName(v), - Attrs: make(map[string]string), - } - - cycle = append(cycle, dv) + mv := newMarshalVertex(v) + cycle = append(cycle, mv) } - dg.Cycles = append(dg.Cycles, cycle) + mg.Cycles = append(mg.Cycles, cycle) } - return dg + return mg } // Attempt to return a unique ID for any vertex. @@ -189,3 +229,241 @@ func marshalSubgrapher(v Vertex) (*Graph, bool) { return nil, false } + +// ender provides a way to call any End* method expression via an End method +type ender func() + +func (e ender) End() { e() } + +// encoder provides methods to write debug data to an io.Writer, and is a noop +// when no writer is present +type encoder struct { + w io.Writer +} + +// Encode is analogous to json.Encoder.Encode +func (e *encoder) Encode(i interface{}) { + if e == nil || e.w == nil { + return + } + + js, err := json.Marshal(i) + if err != nil { + log.Println("[ERROR] dag:", err) + return + } + js = append(js, '\n') + + _, err = e.w.Write(js) + if err != nil { + log.Println("[ERROR] dag:", err) + return + } +} + +func (e *encoder) Add(v Vertex) { + e.Encode(marshalTransform{ + Type: "Transform", + AddVertex: newMarshalVertex(v), + }) +} + +// Remove records the removal of Vertex v. +func (e *encoder) Remove(v Vertex) { + e.Encode(marshalTransform{ + Type: "Transform", + RemoveVertex: newMarshalVertex(v), + }) +} + +func (e *encoder) Connect(edge Edge) { + e.Encode(marshalTransform{ + Type: "Transform", + AddEdge: newMarshalEdge(edge), + }) +} + +func (e *encoder) RemoveEdge(edge Edge) { + e.Encode(marshalTransform{ + Type: "Transform", + RemoveEdge: newMarshalEdge(edge), + }) +} + +// BeginReplace marks the start of a replace operation, and returns the encoder +// to chain the EndReplace call. +func (e *encoder) BeginReplace() ender { + e.Encode(marshalOperation{ + Type: "Operation", + Begin: newString("Replace"), + }) + return e.EndReplace +} + +func (e *encoder) EndReplace() { + e.Encode(marshalOperation{ + Type: "Operation", + End: newString("Replace"), + }) +} + +// BeginReduction marks the start of a replace operation, and returns the encoder +// to chain the EndReduction call. +func (e *encoder) BeginReduction() ender { + e.Encode(marshalOperation{ + Type: "Operation", + Begin: newString("Reduction"), + }) + return e.EndReduction +} + +func (e *encoder) EndReduction() { + e.Encode(marshalOperation{ + Type: "Operation", + End: newString("Reduction"), + }) +} + +// BeginDepthFirstWalk marks the start of a replace operation, and returns the +// encoder to chain the EndDepthFirstWalk call. +func (e *encoder) BeginDepthFirstWalk() ender { + e.Encode(marshalOperation{ + Type: "Operation", + Begin: newString("DepthFirstWalk"), + }) + return e.EndDepthFirstWalk +} + +func (e *encoder) EndDepthFirstWalk() { + e.Encode(marshalOperation{ + Type: "Operation", + End: newString("DepthFirstWalk"), + }) +} + +// BeginReverseDepthFirstWalk marks the start of a replace operation, and +// returns the encoder to chain the EndReverseDepthFirstWalk call. +func (e *encoder) BeginReverseDepthFirstWalk() ender { + e.Encode(marshalOperation{ + Type: "Operation", + Begin: newString("ReverseDepthFirstWalk"), + }) + return e.EndReverseDepthFirstWalk +} + +func (e *encoder) EndReverseDepthFirstWalk() { + e.Encode(marshalOperation{ + Type: "Operation", + End: newString("ReverseDepthFirstWalk"), + }) +} + +// BeginWalk marks the start of a replace operation, and returns the encoder +// to chain the EndWalk call. +func (e *encoder) BeginWalk() ender { + e.Encode(marshalOperation{ + Type: "Operation", + Begin: newString("Walk"), + }) + return e.EndWalk +} + +func (e *encoder) EndWalk() { + e.Encode(marshalOperation{ + Type: "Operation", + End: newString("Walk"), + }) +} + +// structure for recording graph transformations +type marshalTransform struct { + // Type: "Transform" + Type string + AddEdge *marshalEdge `json:",omitempty"` + RemoveEdge *marshalEdge `json:",omitempty"` + AddVertex *marshalVertex `json:",omitempty"` + RemoveVertex *marshalVertex `json:",omitempty"` +} + +func (t marshalTransform) Transform(g *marshalGraph) { + switch { + case t.AddEdge != nil: + g.connect(t.AddEdge) + case t.RemoveEdge != nil: + g.removeEdge(t.RemoveEdge) + case t.AddVertex != nil: + g.add(t.AddVertex) + case t.RemoveVertex != nil: + g.remove(t.RemoveVertex) + } +} + +// this structure allows us to decode any object in the json stream for +// inspection, then re-decode it into a proper struct if needed. +type streamDecode struct { + Type string + Map map[string]interface{} + JSON []byte +} + +func (s *streamDecode) UnmarshalJSON(d []byte) error { + s.JSON = d + err := json.Unmarshal(d, &s.Map) + if err != nil { + return err + } + + if t, ok := s.Map["Type"]; ok { + s.Type, _ = t.(string) + } + return nil +} + +// structure for recording the beginning and end of any multi-step +// transformations. These are informational, and not required to reproduce the +// graph state. +type marshalOperation struct { + Type string + Begin *string `json:",omitempty"` + End *string `json:",omitempty"` +} + +func newBool(b bool) *bool { return &b } + +func newString(s string) *string { return &s } + +// decodeGraph decodes a marshalGraph from an encoded graph stream. +func decodeGraph(r io.Reader) (*marshalGraph, error) { + dec := json.NewDecoder(r) + + // a stream should always start with a graph + g := &marshalGraph{} + + err := dec.Decode(g) + if err != nil { + return nil, err + } + + // now replay any operations that occurred on the original graph + for dec.More() { + s := &streamDecode{} + err := dec.Decode(s) + if err != nil { + return g, err + } + + // the only Type we're concerned with here is Transform to complete the + // Graph + if s.Type != "Transform" { + continue + } + + t := &marshalTransform{} + err = json.Unmarshal(s.JSON, t) + if err != nil { + return g, err + } + t.Transform(g) + } + return g, nil +} diff --git a/dag/marshal_test.go b/dag/marshal_test.go new file mode 100644 index 0000000000..945d12f7c2 --- /dev/null +++ b/dag/marshal_test.go @@ -0,0 +1,175 @@ +package dag + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +func TestGraphDot_empty(t *testing.T) { + var g Graph + g.Add(1) + g.Add(2) + g.Add(3) + + actual := strings.TrimSpace(string(g.Dot(nil))) + expected := strings.TrimSpace(testGraphDotEmptyStr) + if actual != expected { + t.Fatalf("bad: %s", actual) + } +} + +func TestGraphDot_basic(t *testing.T) { + var g Graph + g.Add(1) + g.Add(2) + g.Add(3) + g.Connect(BasicEdge(1, 3)) + + actual := strings.TrimSpace(string(g.Dot(nil))) + expected := strings.TrimSpace(testGraphDotBasicStr) + if actual != expected { + t.Fatalf("bad: %s", actual) + } +} + +const testGraphDotBasicStr = `digraph { + compound = "true" + newrank = "true" + subgraph "root" { + "[root] 1" -> "[root] 3" + } +} +` + +const testGraphDotEmptyStr = `digraph { + compound = "true" + newrank = "true" + subgraph "root" { + } +}` + +func TestGraphJSON_empty(t *testing.T) { + var g Graph + g.Add(1) + g.Add(2) + g.Add(3) + + js, err := g.MarshalJSON() + if err != nil { + t.Fatal(err) + } + + actual := strings.TrimSpace(string(js)) + expected := strings.TrimSpace(testGraphJSONEmptyStr) + if actual != expected { + t.Fatalf("bad: %s", actual) + } +} + +func TestGraphJSON_basic(t *testing.T) { + var g Graph + g.Add(1) + g.Add(2) + g.Add(3) + g.Connect(BasicEdge(1, 3)) + + js, err := g.MarshalJSON() + if err != nil { + t.Fatal(err) + } + actual := strings.TrimSpace(string(js)) + expected := strings.TrimSpace(testGraphJSONBasicStr) + if actual != expected { + t.Fatalf("bad: %s", actual) + } +} + +// record some graph transformations, and make sure we get the same graph when +// they're replayed +func TestGraphJSON_basicRecord(t *testing.T) { + var g Graph + var buf bytes.Buffer + g.SetDebugWriter(&buf) + + g.Add(1) + g.Add(2) + g.Add(3) + g.Connect(BasicEdge(1, 2)) + g.Connect(BasicEdge(1, 3)) + g.Connect(BasicEdge(2, 3)) + (&AcyclicGraph{g}).TransitiveReduction() + + recorded := buf.Bytes() + // the Walk doesn't happen in a determined order, so just count operations + // for now to make sure we wrote stuff out. + if len(bytes.Split(recorded, []byte{'\n'})) != 17 { + t.Fatalf("bad: %s", recorded) + } + + original, err := g.MarshalJSON() + if err != nil { + t.Fatal(err) + } + + // replay the logs, and marshal the graph back out again + m, err := decodeGraph(bytes.NewReader(buf.Bytes())) + if err != nil { + t.Fatal(err) + } + + replayed, err := json.MarshalIndent(m, "", " ") + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(original, replayed) { + t.Fatalf("\noriginal: %s\nreplayed: %s", original, replayed) + } +} + +const testGraphJSONEmptyStr = `{ + "Type": "Graph", + "Name": "root", + "Vertices": [ + { + "ID": "1", + "Name": "1" + }, + { + "ID": "2", + "Name": "2" + }, + { + "ID": "3", + "Name": "3" + } + ] +}` + +const testGraphJSONBasicStr = `{ + "Type": "Graph", + "Name": "root", + "Vertices": [ + { + "ID": "1", + "Name": "1" + }, + { + "ID": "2", + "Name": "2" + }, + { + "ID": "3", + "Name": "3" + } + ], + "Edges": [ + { + "Name": "1|3", + "Source": "1", + "Target": "3" + } + ] +}`