health/graph: Improve graph

Mark local, trusted peers. Add peernames. Improve display.
2019-09-18 11:38:13 +05:30 · 2019-09-18 11:38:13 +05:30 · 4d8ef92b3d
commit 4d8ef92b3d
parent ab4f2b4c41
7 changed files with 194 additions and 76 deletions
--- a/api/types.go
+++ b/api/types.go
@ -291,11 +291,14 @@ type Version struct {
 // then id will be a key of IPFSLinks.  In the event of a SwarmPeers error
 // IPFSLinks[id] == [].
 type ConnectGraph struct {
-	ClusterID peer.ID
+	ClusterID    peer.ID           `json:"cluster_id" codec:"id"`
+	IDtoPeername map[string]string `json:"id_to_peername" codec:"ip,omitempty"`
 	// ipfs to ipfs links
 	IPFSLinks map[string][]peer.ID `json:"ipfs_links" codec:"il,omitempty"`
 	// cluster to cluster links
 	ClusterLinks map[string][]peer.ID `json:"cluster_links" codec:"cl,omitempty"`
+	// cluster trust links
+	ClusterTrustLinks map[string]bool `json:"cluster_trust_links" codec:"ctl,omitempty"`
 	// cluster to ipfs links
 	ClustertoIPFS map[string]peer.ID `json:"cluster_to_ipfs" codec:"ci,omitempty"`
 }
--- a/cmd/ipfs-cluster-ctl/graph.go
+++ b/cmd/ipfs-cluster-ctl/graph.go
@ -6,8 +6,8 @@ import (
 	"io"
 	"sort"

+	dot "github.com/kishansagathiya/go-dot"
 	peer "github.com/libp2p/go-libp2p-peer"
-	dot "github.com/zenground0/go-dot"

 	"github.com/ipfs/ipfs-cluster/api"
 )
@ -31,8 +31,10 @@ import (
 type nodeType int

 const (
-	tCluster nodeType = iota // The cluster node type
-	tIpfs                    // The IPFS node type
+	tSelfCluster    nodeType = iota //  cluster self node
+	tCluster                        //  cluster node
+	tTrustedCluster                 //  trusted cluster node
+	tIPFS                           //  IPFS node linked to a Cluster node
 )

 var errUnfinishedWrite = errors.New("could not complete write of line to output")
@ -62,6 +64,9 @@ func makeDot(cg *api.ConnectGraph, w io.Writer, allIpfs bool) error {
 	dW := dotWriter{
 		w:                w,
 		dotGraph:         dot.NewGraph("cluster"),
+		self:             peer.IDB58Encode(cg.ClusterID),
+		trustMap:         cg.ClusterTrustLinks,
+		idToPeername:     cg.IDtoPeername,
 		ipfsEdges:        ipfsEdges,
 		clusterEdges:     cg.ClusterLinks,
 		clusterIpfsEdges: cg.ClustertoIPFS,
@ -78,53 +83,114 @@ type dotWriter struct {
 	w        io.Writer
 	dotGraph dot.Graph

+	self             string
+	idToPeername     map[string]string
+	trustMap         map[string]bool
 	ipfsEdges        map[string][]peer.ID
 	clusterEdges     map[string][]peer.ID
 	clusterIpfsEdges map[string]peer.ID
 }

+func (dW *dotWriter) addSubGraph(sGraph dot.Graph, rank string) {
+	sGraph.IsSubGraph = true
+	sGraph.Rank = rank
+	dW.dotGraph.AddSubGraph(&sGraph)
+}
+
 // writes nodes to dot file output and creates and stores an ordering over nodes
-func (dW *dotWriter) addNode(id string, nT nodeType) error {
-	var node dot.VertexDescription
-	pid, _ := peer.IDB58Decode(id)
-	node.Label = pid.ShortString()
+func (dW *dotWriter) addNode(graph *dot.Graph, id string, nT nodeType) error {
+	node := dot.NewVertexDescription("")
+	node.Group = id
+	node.ColorScheme = "brbg11"
+	node.FontName = "Ariel"
+	node.Shape = "ellipse"
+	node.Style = "filled"
 	switch nT {
-	case tCluster:
+	case tSelfCluster:
+		node.Label = label(dW.idToPeername[id], shorten(id))
 		node.ID = fmt.Sprintf("C%d", len(dW.clusterNodes))
-		node.Color = "blue2"
+		node.Color = "11"
+		node.Peripheries = 2
+		node.FontColor = "6"
 		dW.clusterNodes[id] = &node
-	case tIpfs:
+	case tTrustedCluster:
+		node.Label = label(dW.idToPeername[id], shorten(id))
+		node.ID = fmt.Sprintf("T%d", len(dW.clusterNodes))
+		node.Color = "11"
+		node.FontColor = "6"
+		dW.clusterNodes[id] = &node
+	case tCluster:
+		node.Label = label(dW.idToPeername[id], shorten(id))
+		node.ID = fmt.Sprintf("C%d", len(dW.clusterNodes))
+		node.Color = "9"
+		node.FontColor = "6"
+		dW.clusterNodes[id] = &node
+	case tIPFS:
 		node.ID = fmt.Sprintf("I%d", len(dW.ipfsNodes))
-		node.Color = "goldenrod"
-		dW.ipfsNodes[id] = &node
+		node.Shape = "box"
+		ipfsID, ok := dW.clusterIpfsEdges[id]
+		if !ok {
+			node.Label = label("IPFS", "Errored")
+			node.ColorScheme = "X11"
+			node.Color = "orangered"
+			node.FontColor = "white"
+			dW.ipfsNodes[id] = &node
+		} else {
+			ipfsIDStr := peer.IDB58Encode(ipfsID)
+			node.Label = label("IPFS", shorten(ipfsIDStr))
+			node.Color = "1"
+			node.FontColor = "6"
+			dW.ipfsNodes[ipfsIDStr] = &node
+		}
 	default:
 		return errUnknownNodeType
 	}
-	dW.dotGraph.AddVertex(&node)

+	graph.AddVertex(&node)
 	return nil
 }

+func shorten(id string) string {
+	return id[:2] + "*" + id[len(id)-6:]
+}
+
+func label(peername, id string) string {
+	return fmt.Sprintf("< <B> %s </B> <BR/> <B> %s </B> >", peername, id)
+}
+
 func (dW *dotWriter) print() error {
 	dW.dotGraph.AddComment("The nodes of the connectivity graph")
 	dW.dotGraph.AddComment("The cluster-service peers")
 	// Write cluster nodes, use sorted order for consistent labels
+	sGraphCluster := dot.NewGraph("")
+	sGraphCluster.IsSubGraph = true
 	for _, k := range sortedKeys(dW.clusterEdges) {
-		err := dW.addNode(k, tCluster)
+		var err error
+		if k == dW.self {
+			err = dW.addNode(&sGraphCluster, k, tSelfCluster)
+		} else if dW.trustMap[k] {
+			err = dW.addNode(&sGraphCluster, k, tTrustedCluster)
+		} else {
+			err = dW.addNode(&sGraphCluster, k, tCluster)
+		}
 		if err != nil {
 			return err
 		}
 	}
+	dW.addSubGraph(sGraphCluster, "min")
 	dW.dotGraph.AddNewLine()

-	dW.dotGraph.AddComment("The ipfs peers")
+	dW.dotGraph.AddComment("The ipfs peers linked to cluster peers")
+	sGraphIPFS := dot.NewGraph("")
+	sGraphIPFS.IsSubGraph = true
 	// Write ipfs nodes, use sorted order for consistent labels
-	for _, k := range sortedKeys(dW.ipfsEdges) {
-		err := dW.addNode(k, tIpfs)
+	for _, k := range sortedKeys(dW.clusterEdges) {
+		err := dW.addNode(&sGraphIPFS, k, tIPFS)
 		if err != nil {
 			return err
 		}
 	}
+	dW.addSubGraph(sGraphIPFS, "max")
 	dW.dotGraph.AddNewLine()

 	dW.dotGraph.AddComment("Edges representing active connections in the cluster")
@ -134,30 +200,52 @@ func (dW *dotWriter) print() error {
 		for _, id := range v {
 			toNode := dW.clusterNodes[k]
 			fromNode := dW.clusterNodes[peer.IDB58Encode(id)]
-			dW.dotGraph.AddEdge(toNode, fromNode, true)
+			dW.dotGraph.AddEdge(toNode, fromNode, true, "")
 		}
 	}
 	dW.dotGraph.AddNewLine()

 	dW.dotGraph.AddComment("The connections between cluster peers and their ipfs daemons")
 	// Write cluster to ipfs edges
-	for k, id := range dW.clusterIpfsEdges {
+	for k := range dW.clusterEdges {
+		var fromNode *dot.VertexDescription
 		toNode := dW.clusterNodes[k]
-		fromNode := dW.ipfsNodes[peer.IDB58Encode(id)]
-		dW.dotGraph.AddEdge(toNode, fromNode, true)
+		ipfsID, ok := dW.clusterIpfsEdges[k]
+		if !ok {
+			fromNode, ok2 := dW.ipfsNodes[k]
+			if !ok2 {
+				logger.Warning("expected a node at this id")
+				continue
+			}
+			dW.dotGraph.AddEdge(toNode, fromNode, true, "dotted")
+			continue
+		}
+
+		fromNode, ok = dW.ipfsNodes[peer.IDB58Encode(ipfsID)]
+		if !ok {
+			logger.Warning("expected a node at this id")
+			continue
+		}
+		dW.dotGraph.AddEdge(toNode, fromNode, true, "")
 	}
 	dW.dotGraph.AddNewLine()

 	dW.dotGraph.AddComment("The swarm peer connections among ipfs daemons in the cluster")
 	// Write ipfs edges
-	for k, v := range dW.ipfsEdges {
+	for _, k := range sortedKeys(dW.ipfsEdges) {
+		v := dW.ipfsEdges[k]
+		toNode := dW.ipfsNodes[k]
 		for _, id := range v {
-			toNode := dW.ipfsNodes[k]
-			fromNode := dW.ipfsNodes[peer.IDB58Encode(id)]
-			dW.dotGraph.AddEdge(toNode, fromNode, true)
+			idStr := peer.IDB58Encode(id)
+			fromNode, ok := dW.ipfsNodes[idStr]
+			if !ok {
+				logger.Warning("expected a node here")
+				continue
+			}
+			dW.dotGraph.AddEdge(toNode, fromNode, true, "")
 		}
 	}
-	return dW.dotGraph.WriteDot(dW.w)
+	return dW.dotGraph.Write(dW.w)
 }

 func sortedKeys(dict map[string][]peer.ID) []string {
--- a/cmd/ipfs-cluster-ctl/graph_test.go
+++ b/cmd/ipfs-cluster-ctl/graph_test.go
@ -32,17 +32,24 @@ func verifyOutput(t *testing.T, outStr string, trueStr string) {
 }

 var simpleIpfs = `digraph cluster {
-
 /* The nodes of the connectivity graph */
 /* The cluster-service peers */
-C0 [label="<peer.ID Qm*eqhEhD>" color="blue2"]
-C1 [label="<peer.ID Qm*cgHDQJ>" color="blue2"]
-C2 [label="<peer.ID Qm*6MQmJu>" color="blue2"]
+subgraph  {
+rank="min"
+C0 [label=< <B>  </B> <BR/> <B> Qm*EhD </B> > group="QmUBuxVHoNNjfmNpTad36UeaFQv3gXAtCv9r6KhmeqhEhD" color="11" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="ellipse" peripheries="2" ]
+C1 [label=< <B>  </B> <BR/> <B> Qm*DQJ </B> > group="QmV35LjbEGPfN7KfMAJp43VV2enwXqqQf5esx4vUcgHDQJ" color="9" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="ellipse" ]
+C2 [label=< <B>  </B> <BR/> <B> Qm*mJu </B> > group="QmZ2ckU7G35MYyJgMTwMUnicsGqSy3YUxGBX7qny6MQmJu" color="9" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="ellipse" ]
+}

-/* The ipfs peers */
-I0 [label="<peer.ID Qm*N5LSsq>" color="goldenrod"]
-I1 [label="<peer.ID Qm*R3DZDV>" color="goldenrod"]
-I2 [label="<peer.ID Qm*wbBsuL>" color="goldenrod"]
+/* The ipfs peers linked to cluster peers */
+subgraph  {
+rank="max"
+I0 [label=< <B> IPFS </B> <BR/> <B> Qm*ZDV </B> > group="QmUBuxVHoNNjfmNpTad36UeaFQv3gXAtCv9r6KhmeqhEhD" color="1" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="box" ]
+I1 [label=< <B> IPFS </B> <BR/> <B> Qm*Ssq </B> > group="QmV35LjbEGPfN7KfMAJp43VV2enwXqqQf5esx4vUcgHDQJ" color="1" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="box" ]
+I2 [label=< <B> IPFS </B> <BR/> <B> Qm*suL </B> > group="QmZ2ckU7G35MYyJgMTwMUnicsGqSy3YUxGBX7qny6MQmJu" color="1" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="box" ]
+}
+
+/* The ipfs swarm peers */

 /* Edges representing active connections in the cluster */
 /* The connections among cluster-service peers */
@ -54,19 +61,18 @@ C2 -> C0
 C2 -> C1

 /* The connections between cluster peers and their ipfs daemons */
-C0 -> I1
-C1 -> I0
+C0 -> I0
+C1 -> I1
 C2 -> I2

 /* The swarm peer connections among ipfs daemons in the cluster */
-I0 -> I1
-I0 -> I2
 I1 -> I0
 I1 -> I2
+I0 -> I1
+I0 -> I2
 I2 -> I0
 I2 -> I1
-
- }`
+}`

 var (
 	pid1, _ = peer.IDB58Decode("QmUBuxVHoNNjfmNpTad36UeaFQv3gXAtCv9r6KhmeqhEhD")
@ -127,20 +133,27 @@ func TestSimpleIpfsGraphs(t *testing.T) {
 }

 var allIpfs = `digraph cluster {
-
 /* The nodes of the connectivity graph */
 /* The cluster-service peers */
-C0 [label="<peer.ID Qm*eqhEhD>" color="blue2"]
-C1 [label="<peer.ID Qm*cgHDQJ>" color="blue2"]
-C2 [label="<peer.ID Qm*6MQmJu>" color="blue2"]
+subgraph  {
+rank="min"
+C0 [label=< <B>  </B> <BR/> <B> Qm*EhD </B> > group="QmUBuxVHoNNjfmNpTad36UeaFQv3gXAtCv9r6KhmeqhEhD" color="11" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="ellipse" peripheries="2" ]
+C1 [label=< <B>  </B> <BR/> <B> Qm*DQJ </B> > group="QmV35LjbEGPfN7KfMAJp43VV2enwXqqQf5esx4vUcgHDQJ" color="9" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="ellipse" ]
+C2 [label=< <B>  </B> <BR/> <B> Qm*mJu </B> > group="QmZ2ckU7G35MYyJgMTwMUnicsGqSy3YUxGBX7qny6MQmJu" color="9" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="ellipse" ]
+}

-/* The ipfs peers */
-I0 [label="<peer.ID Qm*N5LSsq>" color="goldenrod"]
-I1 [label="<peer.ID Qm*S8xccb>" color="goldenrod"]
-I2 [label="<peer.ID Qm*aaanM8>" color="goldenrod"]
-I3 [label="<peer.ID Qm*R3DZDV>" color="goldenrod"]
-I4 [label="<peer.ID Qm*wbBsuL>" color="goldenrod"]
-I5 [label="<peer.ID Qm*tWZdeD>" color="goldenrod"]
+/* The ipfs peers linked to cluster peers */
+subgraph  {
+rank="max"
+I0 [label=< <B> IPFS </B> <BR/> <B> Qm*ZDV </B> > group="QmUBuxVHoNNjfmNpTad36UeaFQv3gXAtCv9r6KhmeqhEhD" color="1" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="box" ]
+I1 [label=< <B> IPFS </B> <BR/> <B> Qm*Ssq </B> > group="QmV35LjbEGPfN7KfMAJp43VV2enwXqqQf5esx4vUcgHDQJ" color="1" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="box" ]
+I2 [label=< <B> IPFS </B> <BR/> <B> Qm*suL </B> > group="QmZ2ckU7G35MYyJgMTwMUnicsGqSy3YUxGBX7qny6MQmJu" color="1" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="box" ]
+}
+
+/* The ipfs swarm peers */
+I3 [label=< <B> IPFS </B> <BR/> <B> Qm*ccb </B> > group="QmQsdAdCHs4PRLi5tcoLfasYppryqQENxgAy4b2aS8xccb" color="5" style="filled" colorscheme="brbg11" fontcolor="1" fontname="Ariel" shape="box" ]
+I4 [label=< <B> IPFS </B> <BR/> <B> Qm*nM8 </B> > group="QmVV2enwXqqQf5esx4v36UeaFQvFehSPzNfi8aaaaaanM8" color="5" style="filled" colorscheme="brbg11" fontcolor="1" fontname="Ariel" shape="box" ]
+I5 [label=< <B> IPFS </B> <BR/> <B> Qm*deD </B> > group="QmfCHNQ2vbUmAuJZhE2hEpgiJq4sL1XScWEKnUrVtWZdeD" color="5" style="filled" colorscheme="brbg11" fontcolor="1" fontname="Ariel" shape="box" ]

 /* Edges representing active connections in the cluster */
 /* The connections among cluster-service peers */
@ -152,28 +165,27 @@ C1 -> C0
 C1 -> C2

 /* The connections between cluster peers and their ipfs daemons */
-C0 -> I3
-C1 -> I0
-C2 -> I4
+C0 -> I0
+C1 -> I1
+C2 -> I2

 /* The swarm peer connections among ipfs daemons in the cluster */
+I1 -> I0
+I1 -> I2
+I1 -> I3
+I1 -> I4
+I1 -> I5
 I0 -> I1
 I0 -> I2
 I0 -> I3
 I0 -> I4
 I0 -> I5
-I3 -> I0
-I3 -> I1
-I3 -> I2
-I3 -> I4
-I3 -> I5
-I4 -> I0
-I4 -> I1
-I4 -> I2
-I4 -> I3
-I4 -> I5
-
- }`
+I2 -> I0
+I2 -> I1
+I2 -> I3
+I2 -> I4
+I2 -> I5
+}`

 func TestIpfsAllGraphs(t *testing.T) {
 	cg := api.ConnectGraph{
--- a/connect_graph.go
+++ b/connect_graph.go
@ -10,22 +10,29 @@ import (
 )

 // ConnectGraph returns a description of which cluster peers and ipfs
-// daemons are connected to each other
+// daemons are connected to each other.
 func (c *Cluster) ConnectGraph() (api.ConnectGraph, error) {
 	ctx, span := trace.StartSpan(c.ctx, "cluster/ConnectGraph")
 	defer span.End()

 	cg := api.ConnectGraph{
-		ClusterID:     c.host.ID(),
-		IPFSLinks:     make(map[string][]peer.ID),
-		ClusterLinks:  make(map[string][]peer.ID),
-		ClustertoIPFS: make(map[string]peer.ID),
+		ClusterID:         c.host.ID(),
+		IDtoPeername:      make(map[string]string),
+		IPFSLinks:         make(map[string][]peer.ID),
+		ClusterLinks:      make(map[string][]peer.ID),
+		ClusterTrustLinks: make(map[string]bool),
+		ClustertoIPFS:     make(map[string]peer.ID),
 	}
 	members, err := c.consensus.Peers(ctx)
 	if err != nil {
 		return cg, err
 	}

+	for _, member := range members {
+		// one of the entries is for itself, but that shouldn't hurt
+		cg.ClusterTrustLinks[peer.IDB58Encode(member)] = c.consensus.IsTrustedPeer(ctx, member)
+	}
+
 	peers := make([][]*api.ID, len(members), len(members))

 	ctxs, cancels := rpcutil.CtxsWithCancel(ctx, len(members))
@ -49,7 +56,7 @@ func (c *Cluster) ConnectGraph() (api.ConnectGraph, error) {
 		}

 		selfConnection, pID := c.recordClusterLinks(&cg, p, peers[i])
-
+		cg.IDtoPeername[p] = pID.Peername
 		// IPFS connections
 		if !selfConnection {
 			logger.Warningf("cluster peer %s not its own peer.  No ipfs info ", p)
--- a/go.mod
+++ b/go.mod
@ -39,6 +39,7 @@ require (
 	github.com/ipfs/go-path v0.0.7
 	github.com/ipfs/go-unixfs v0.2.2
 	github.com/kelseyhightower/envconfig v1.4.0
+	github.com/kishansagathiya/go-dot v0.0.2-0.20191016070847-1c5b432788b0
 	github.com/lanzafame/go-libp2p-ocgorpc v0.1.1
 	github.com/libp2p/go-libp2p v0.4.0
 	github.com/libp2p/go-libp2p-autonat-svc v0.1.0
--- a/go.sum
+++ b/go.sum
@ -351,6 +351,10 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E
 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
 github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
 github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
+github.com/kishansagathiya/go-dot v0.0.2-0.20191013111036-f1da912e6fcb h1:K72Zy5/WevkrSEK9R9lyGowoEol8UsoSIWbfEtp0HeY=
+github.com/kishansagathiya/go-dot v0.0.2-0.20191013111036-f1da912e6fcb/go.mod h1:U1dCUFzZ+KnBgkaCWPj2JFUQygVepVudkINK9QRsxMs=
+github.com/kishansagathiya/go-dot v0.0.2-0.20191016070847-1c5b432788b0 h1:PaBS3DH0aJ7E7+YZevHQGyElNOEwZW7Q5ljITIBDA88=
+github.com/kishansagathiya/go-dot v0.0.2-0.20191016070847-1c5b432788b0/go.mod h1:U1dCUFzZ+KnBgkaCWPj2JFUQygVepVudkINK9QRsxMs=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
@ -855,7 +859,6 @@ github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:
 github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg=
 github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/zenground0/go-dot v0.0.0-20180912213407-94a425d4984e h1:GN1PUQ/MNDdtiZZhCAnZ4PwTcslUM8qWVz8q2bLkDeM=
 github.com/zenground0/go-dot v0.0.0-20180912213407-94a425d4984e/go.mod h1:T00FaxHq4SlnicuZFole4yRAgcjWtqbMcUXBfXAYvaI=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg=
--- a/ipfscluster_test.go
+++ b/ipfscluster_test.go
@ -1877,9 +1877,9 @@ func TestClustersRebalanceOnPeerDown(t *testing.T) {
 	}
 }

-// Helper function for verifying cluster graph.  Will only pass if exactly the
+// Helper function for verifying cluster graph. Will only pass if exactly the
 // peers in clusterIDs are fully connected to each other and the expected ipfs
-// mock connectivity exists.  Cluster peers not in clusterIDs are assumed to
+// mock connectivity exists. Cluster peers not in clusterIDs are assumed to
 // be disconnected and the graph should reflect this
 func validateClusterGraph(t *testing.T, graph api.ConnectGraph, clusterIDs map[string]struct{}, peerNum int) {
 	// Check that all cluster peers see each other as peers
@ -1916,6 +1916,10 @@ func validateClusterGraph(t *testing.T, graph api.ConnectGraph, clusterIDs map[s
 		}
 	}

+	if len(graph.ClusterTrustLinks) != peerNum {
+		t.Errorf("Unexpected number of trust links in graph")
+	}
+
 	// Check that the mocked ipfs swarm is recorded
 	if len(graph.IPFSLinks) != 1 {
 		t.Error("Expected exactly one ipfs peer for all cluster nodes, the mocked peer")