diff --git a/.travis.yml b/.travis.yml index 93c95a98..305dd09c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ jobs: - stage: "Testing stage" name: "Tests (all modules) + Coverage" script: - - go test -v -failfast -timeout 15m -coverprofile=coverage.txt -covermode=atomic ./... + - travis_wait go test -v -timeout 15m -coverprofile=coverage.txt -covermode=atomic ./... after_success: - bash <(curl -s https://codecov.io/bash) - name: "Main Tests with crdt consensus" diff --git a/Dockerfile b/Dockerfile index 58c82316..c7c83768 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,10 +8,6 @@ ENV SRC_PATH $GOPATH/src/github.com/ipfs/ipfs-cluster ENV GO111MODULE on ENV GOPROXY=https://proxy.golang.org -COPY . $SRC_PATH -WORKDIR $SRC_PATH -RUN make install - ENV SUEXEC_VERSION v0.2 ENV TINI_VERSION v0.16.1 RUN set -x \ @@ -27,6 +23,14 @@ RUN set -x \ # Get the TLS CA certificates, they're not provided by busybox. RUN apt-get update && apt-get install -y ca-certificates +COPY go.* $SRC_PATH/ +WORKDIR $SRC_PATH +RUN go mod download + +COPY . $SRC_PATH +RUN make install + + #------------------------------------------------------ FROM busybox:1-glibc MAINTAINER Hector Sanjuan diff --git a/Dockerfile-bundle b/Dockerfile-bundle index d60db4a3..6c91fad2 100644 --- a/Dockerfile-bundle +++ b/Dockerfile-bundle @@ -7,9 +7,14 @@ MAINTAINER Hector Sanjuan # This builder just builds the cluster binaries ENV GOPATH /go ENV SRC_PATH $GOPATH/src/github.com/ipfs/ipfs-cluster +ENV GO111MODULE on +ENV GOPROXY=https://proxy.golang.org + +COPY go.* $SRC_PATH/ +WORKDIR $SRC_PATH +RUN go mod download COPY . $SRC_PATH -WORKDIR $SRC_PATH RUN make install #------------------------------------------------------ diff --git a/Dockerfile-test b/Dockerfile-test index 61e28589..4b9476dc 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -8,14 +8,16 @@ ENV SRC_PATH $GOPATH/src/github.com/ipfs/ipfs-cluster ENV GO111MODULE on ENV GOPROXY=https://proxy.golang.org -COPY . $SRC_PATH -WORKDIR $SRC_PATH -RUN make install - RUN cd /tmp && \ wget https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 && \ chmod +x jq-linux64 - + +COPY go.* $SRC_PATH/ +WORKDIR $SRC_PATH +RUN go mod download + +COPY . $SRC_PATH +RUN make install #------------------------------------------------------ FROM ipfs/go-ipfs:master diff --git a/Makefile b/Makefile index 723883e3..eade2f6c 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,6 @@ install: $(MAKE) -C cmd/ipfs-cluster-ctl install build: - go build -ldflags "-X ipfscluster.Commit=$(shell git rev-parse HEAD)" $(MAKE) -C cmd/ipfs-cluster-service build $(MAKE) -C cmd/ipfs-cluster-ctl build diff --git a/api/add.go b/api/add.go index 26e361e5..298d1f56 100644 --- a/api/add.go +++ b/api/add.go @@ -172,9 +172,15 @@ func AddParamsFromQuery(query url.Values) (*AddParams, error) { } // ToQueryString returns a url query string (key=value&key2=value2&...) -func (p *AddParams) ToQueryString() string { - pinOptsQuery := p.PinOptions.ToQuery() - query, _ := url.ParseQuery(pinOptsQuery) +func (p *AddParams) ToQueryString() (string, error) { + pinOptsQuery, err := p.PinOptions.ToQuery() + if err != nil { + return "", err + } + query, err := url.ParseQuery(pinOptsQuery) + if err != nil { + return "", err + } query.Set("shard", fmt.Sprintf("%t", p.Shard)) query.Set("local", fmt.Sprintf("%t", p.Local)) query.Set("recursive", fmt.Sprintf("%t", p.Recursive)) @@ -188,7 +194,7 @@ func (p *AddParams) ToQueryString() string { query.Set("hash", p.HashFun) query.Set("stream-channels", fmt.Sprintf("%t", p.StreamChannels)) query.Set("nocopy", fmt.Sprintf("%t", p.NoCopy)) - return query.Encode() + return query.Encode(), nil } // Equals checks if p equals p2. diff --git a/api/add_test.go b/api/add_test.go index e8d38ee8..d302d787 100644 --- a/api/add_test.go +++ b/api/add_test.go @@ -35,11 +35,13 @@ func TestAddParams_ToQueryString(t *testing.T) { p.Name = "something" p.RawLeaves = true p.ShardSize = 1020 - qstr := p.ToQueryString() - + qstr, err := p.ToQueryString() + if err != nil { + t.Fatal(err) + } q, err := url.ParseQuery(qstr) if err != nil { - t.Fatal() + t.Fatal(err) } p2, err := AddParamsFromQuery(q) diff --git a/api/pb/types.pb.go b/api/pb/types.pb.go index 432d8654..a76a1544 100644 --- a/api/pb/types.pb.go +++ b/api/pb/types.pb.go @@ -140,6 +140,7 @@ type PinOptions struct { ShardSize uint64 `protobuf:"varint,4,opt,name=ShardSize,proto3" json:"ShardSize,omitempty"` Metadata map[string]string `protobuf:"bytes,6,rep,name=Metadata,proto3" json:"Metadata,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` PinUpdate []byte `protobuf:"bytes,7,opt,name=PinUpdate,proto3" json:"PinUpdate,omitempty"` + ExpireAt uint64 `protobuf:"varint,8,opt,name=ExpireAt,proto3" json:"ExpireAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -212,6 +213,13 @@ func (m *PinOptions) GetPinUpdate() []byte { return nil } +func (m *PinOptions) GetExpireAt() uint64 { + if m != nil { + return m.ExpireAt + } + return 0 +} + func init() { proto.RegisterEnum("api.pb.Pin_PinType", Pin_PinType_name, Pin_PinType_value) proto.RegisterType((*Pin)(nil), "api.pb.Pin") @@ -222,30 +230,31 @@ func init() { func init() { proto.RegisterFile("types.proto", fileDescriptor_d938547f84707355) } var fileDescriptor_d938547f84707355 = []byte{ - // 391 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x52, 0xcd, 0x8e, 0xd3, 0x30, - 0x10, 0xc6, 0x49, 0x9a, 0x34, 0x93, 0xee, 0xaa, 0x3b, 0xec, 0xc1, 0x5a, 0x71, 0xb0, 0x7a, 0x21, - 0x07, 0x94, 0x43, 0xb8, 0x20, 0xe0, 0xb2, 0x6c, 0x01, 0x09, 0xa9, 0x50, 0x79, 0xe9, 0x03, 0xb8, - 0x8d, 0x51, 0x2d, 0x42, 0x62, 0xa5, 0x2e, 0x6a, 0x78, 0x1b, 0x1e, 0x86, 0xf7, 0x42, 0xb6, 0xfb, - 0x87, 0xe8, 0x21, 0xd2, 0x7c, 0xdf, 0xcc, 0xe7, 0x99, 0xf9, 0x32, 0x90, 0x99, 0x5e, 0xcb, 0x4d, - 0xa1, 0xbb, 0xd6, 0xb4, 0x18, 0x0b, 0xad, 0x0a, 0xbd, 0x9c, 0xfc, 0x0e, 0x20, 0x9c, 0xab, 0x06, - 0xc7, 0x10, 0x3e, 0xa8, 0x8a, 0x12, 0x46, 0xf2, 0x11, 0xb7, 0x21, 0x3e, 0x87, 0xe8, 0x6b, 0xaf, - 0x25, 0x0d, 0x18, 0xc9, 0xaf, 0xcb, 0xa7, 0x85, 0x17, 0x14, 0x73, 0xd5, 0xd8, 0xcf, 0xa6, 0xb8, - 0x2b, 0x40, 0x06, 0xd9, 0x7d, 0x5d, 0xb7, 0x2b, 0x61, 0x54, 0xdb, 0x6c, 0x68, 0xc8, 0xc2, 0x7c, - 0xc4, 0xcf, 0x29, 0xbc, 0x83, 0xe1, 0x4c, 0xec, 0xa6, 0x52, 0x9b, 0x35, 0x8d, 0x18, 0xc9, 0x6f, - 0xf8, 0x11, 0xe3, 0x33, 0x48, 0xb9, 0xfc, 0x26, 0x3b, 0xd9, 0xac, 0x24, 0x1d, 0xb8, 0xf6, 0x27, - 0x02, 0x5f, 0x40, 0xf2, 0x45, 0xfb, 0x77, 0x63, 0x46, 0xf2, 0xac, 0xc4, 0xb3, 0x39, 0xf6, 0x19, - 0x7e, 0x28, 0x99, 0x2c, 0x20, 0xd9, 0x8f, 0x86, 0x19, 0x24, 0xef, 0x44, 0x65, 0xc3, 0xf1, 0x13, - 0x1c, 0xc1, 0x70, 0x2a, 0x8c, 0x70, 0x88, 0x58, 0x34, 0x93, 0x7b, 0x14, 0x20, 0xc2, 0xf5, 0x43, - 0xbd, 0xdd, 0x18, 0xd9, 0x4d, 0xef, 0x3f, 0x3a, 0x2e, 0xc4, 0x2b, 0x48, 0x1f, 0xd7, 0xa2, 0xf3, - 0xf2, 0x68, 0xf2, 0x27, 0x00, 0x38, 0xb5, 0xc3, 0x12, 0x6e, 0xb9, 0xd4, 0xb5, 0xf2, 0xdb, 0x7d, - 0x10, 0x2b, 0xd3, 0x76, 0x33, 0xd5, 0x38, 0xef, 0x6e, 0xf8, 0xc5, 0xdc, 0x65, 0x8d, 0xd8, 0x39, - 0x73, 0x2f, 0x6a, 0xc4, 0x0e, 0x11, 0xa2, 0xcf, 0xe2, 0x87, 0xa4, 0x21, 0x23, 0x79, 0xca, 0x5d, - 0x6c, 0xdd, 0x72, 0x93, 0x3d, 0xaa, 0x5f, 0xd2, 0x59, 0x19, 0xf1, 0x13, 0x81, 0x6f, 0xfd, 0x66, - 0x95, 0x30, 0x82, 0xc6, 0x2c, 0xcc, 0xb3, 0x92, 0xfd, 0x6f, 0x57, 0x71, 0x28, 0x79, 0xdf, 0x98, - 0xae, 0xe7, 0x47, 0x85, 0x7d, 0x7b, 0xae, 0x9a, 0x85, 0xae, 0x84, 0x91, 0x34, 0xf1, 0x7f, 0xe2, - 0x48, 0xdc, 0xbd, 0x81, 0xab, 0x7f, 0x84, 0xf6, 0x62, 0xbe, 0xcb, 0xde, 0x6d, 0x9d, 0x72, 0x1b, - 0xe2, 0x2d, 0x0c, 0x7e, 0x8a, 0x7a, 0xeb, 0x4f, 0x26, 0xe5, 0x1e, 0xbc, 0x0e, 0x5e, 0x91, 0x4f, - 0xd1, 0x70, 0x30, 0x8e, 0x97, 0xb1, 0x3b, 0xbd, 0x97, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xfe, - 0x90, 0x29, 0x9a, 0x89, 0x02, 0x00, 0x00, + // 405 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x52, 0xc1, 0x6e, 0xd3, 0x40, + 0x10, 0x65, 0x6d, 0xc7, 0xb1, 0xc7, 0x69, 0x95, 0x0e, 0x3d, 0xac, 0x2a, 0x0e, 0xab, 0x5c, 0xf0, + 0x01, 0xf9, 0x60, 0x2e, 0x08, 0xb8, 0x84, 0xa6, 0x20, 0x21, 0x05, 0xa2, 0x2d, 0xfd, 0x80, 0x6d, + 0xbc, 0xa8, 0x2b, 0x8c, 0xbd, 0x72, 0xb6, 0xc8, 0xe6, 0x6f, 0xf8, 0x34, 0xfe, 0x04, 0xed, 0x6e, + 0xea, 0x14, 0x35, 0x07, 0x4b, 0xf3, 0xde, 0xcc, 0x9b, 0x19, 0xbf, 0x1d, 0xc8, 0xcc, 0xa0, 0xe5, + 0xae, 0xd0, 0x5d, 0x6b, 0x5a, 0x8c, 0x85, 0x56, 0x85, 0xbe, 0x5d, 0xfc, 0x09, 0x20, 0xdc, 0xa8, + 0x06, 0xe7, 0x10, 0x5e, 0xaa, 0x8a, 0x12, 0x46, 0xf2, 0x19, 0xb7, 0x21, 0xbe, 0x84, 0xe8, 0xdb, + 0xa0, 0x25, 0x0d, 0x18, 0xc9, 0x4f, 0xcb, 0xe7, 0x85, 0x17, 0x14, 0x1b, 0xd5, 0xd8, 0xcf, 0xa6, + 0xb8, 0x2b, 0x40, 0x06, 0xd9, 0xb2, 0xae, 0xdb, 0xad, 0x30, 0xaa, 0x6d, 0x76, 0x34, 0x64, 0x61, + 0x3e, 0xe3, 0x8f, 0x29, 0xbc, 0x80, 0x64, 0x2d, 0xfa, 0x95, 0xd4, 0xe6, 0x8e, 0x46, 0x8c, 0xe4, + 0x67, 0x7c, 0xc4, 0xf8, 0x02, 0x52, 0x2e, 0xbf, 0xcb, 0x4e, 0x36, 0x5b, 0x49, 0x27, 0x6e, 0xfc, + 0x81, 0xc0, 0x57, 0x30, 0xfd, 0xaa, 0x7d, 0xdf, 0x98, 0x91, 0x3c, 0x2b, 0xf1, 0xd1, 0x1e, 0xfb, + 0x0c, 0x7f, 0x28, 0x59, 0xdc, 0xc0, 0x74, 0xbf, 0x1a, 0x66, 0x30, 0xfd, 0x20, 0x2a, 0x1b, 0xce, + 0x9f, 0xe1, 0x0c, 0x92, 0x95, 0x30, 0xc2, 0x21, 0x62, 0xd1, 0x5a, 0xee, 0x51, 0x80, 0x08, 0xa7, + 0x97, 0xf5, 0xfd, 0xce, 0xc8, 0x6e, 0xb5, 0xfc, 0xe4, 0xb8, 0x10, 0x4f, 0x20, 0xbd, 0xbe, 0x13, + 0x9d, 0x97, 0x47, 0x8b, 0xbf, 0x01, 0xc0, 0x61, 0x1c, 0x96, 0x70, 0xce, 0xa5, 0xae, 0x95, 0xff, + 0xbb, 0x8f, 0x62, 0x6b, 0xda, 0x6e, 0xad, 0x1a, 0xe7, 0xdd, 0x19, 0x3f, 0x9a, 0x3b, 0xae, 0x11, + 0xbd, 0x33, 0xf7, 0xa8, 0x46, 0xf4, 0x88, 0x10, 0x7d, 0x11, 0x3f, 0x25, 0x0d, 0x19, 0xc9, 0x53, + 0xee, 0x62, 0xeb, 0x96, 0xdb, 0xec, 0x5a, 0xfd, 0x96, 0xce, 0xca, 0x88, 0x1f, 0x08, 0x7c, 0xef, + 0xff, 0xac, 0x12, 0x46, 0xd0, 0x98, 0x85, 0x79, 0x56, 0xb2, 0xa7, 0x76, 0x15, 0x0f, 0x25, 0x57, + 0x8d, 0xe9, 0x06, 0x3e, 0x2a, 0x6c, 0xef, 0x8d, 0x6a, 0x6e, 0x74, 0x25, 0x8c, 0xa4, 0x53, 0xff, + 0x12, 0x23, 0x61, 0xdf, 0xf0, 0xaa, 0xd7, 0xaa, 0x93, 0x4b, 0x43, 0x13, 0x37, 0x78, 0xc4, 0x17, + 0xef, 0xe0, 0xe4, 0xbf, 0xa6, 0xf6, 0x9a, 0x7e, 0xc8, 0xc1, 0x39, 0x92, 0x72, 0x1b, 0xe2, 0x39, + 0x4c, 0x7e, 0x89, 0xfa, 0xde, 0x9f, 0x53, 0xca, 0x3d, 0x78, 0x1b, 0xbc, 0x21, 0x9f, 0xa3, 0x64, + 0x32, 0x8f, 0x6f, 0x63, 0x77, 0x96, 0xaf, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0xaf, 0x59, 0x74, + 0x86, 0xa5, 0x02, 0x00, 0x00, } diff --git a/api/pb/types.proto b/api/pb/types.proto index 70ad76fe..bc73b5d6 100644 --- a/api/pb/types.proto +++ b/api/pb/types.proto @@ -25,4 +25,5 @@ message PinOptions { reserved 5; // reserved for UserAllocations map Metadata = 6; bytes PinUpdate = 7; + uint64 ExpireAt = 8; } \ No newline at end of file diff --git a/api/rest/client/client.go b/api/rest/client/client.go index c76e68f5..ea5fd59b 100644 --- a/api/rest/client/client.go +++ b/api/rest/client/client.go @@ -120,6 +120,9 @@ type Client interface { // for the current cluster peers. Metrics(ctx context.Context, name string) ([]*api.Metric, error) + // MetricNames returns the list of metric types. + MetricNames(ctx context.Context) ([]string, error) + // RepoGC runs garbage collection on IPFS daemons of cluster peers and // returns collected CIDs. If local is true, it would garbage collect // only on contacted peer, otherwise on all peers' IPFS daemons. diff --git a/api/rest/client/lbclient.go b/api/rest/client/lbclient.go index bde667ea..c9698e16 100644 --- a/api/rest/client/lbclient.go +++ b/api/rest/client/lbclient.go @@ -369,6 +369,20 @@ func (lc *loadBalancingClient) Metrics(ctx context.Context, name string) ([]*api return metrics, err } +// MetricNames returns the list of metric types. +func (lc *loadBalancingClient) MetricNames(ctx context.Context) ([]string, error) { + var metricNames []string + call := func(c Client) error { + var err error + metricNames, err = c.MetricNames(ctx) + return err + } + + err := lc.retry(0, call) + + return metricNames, err +} + // RepoGC runs garbage collection on IPFS daemons of cluster peers and // returns collected CIDs. If local is true, it would garbage collect // only on contacted peer, otherwise on all peers' IPFS daemons. diff --git a/api/rest/client/methods.go b/api/rest/client/methods.go index 04c01653..532aba8b 100644 --- a/api/rest/client/methods.go +++ b/api/rest/client/methods.go @@ -78,14 +78,18 @@ func (c *defaultClient) Pin(ctx context.Context, ci cid.Cid, opts api.PinOptions ctx, span := trace.StartSpan(ctx, "client/Pin") defer span.End() + query, err := opts.ToQuery() + if err != nil { + return nil, err + } var pin api.Pin - err := c.do( + err = c.do( ctx, "POST", fmt.Sprintf( "/pins/%s?%s", ci.String(), - opts.ToQuery(), + query, ), nil, nil, @@ -119,14 +123,17 @@ func (c *defaultClient) PinPath(ctx context.Context, path string, opts api.PinOp if err != nil { return nil, err } - + query, err := opts.ToQuery() + if err != nil { + return nil, err + } err = c.do( ctx, "POST", fmt.Sprintf( "/pins%s?%s", ipfspath.String(), - opts.ToQuery(), + query, ), nil, nil, @@ -334,6 +341,16 @@ func (c *defaultClient) Metrics(ctx context.Context, name string) ([]*api.Metric return metrics, err } +// MetricNames lists names of all metrics. +func (c *defaultClient) MetricNames(ctx context.Context) ([]string, error) { + ctx, span := trace.StartSpan(ctx, "client/MetricNames") + defer span.End() + + var metricsNames []string + err := c.do(ctx, "GET", "/monitor/metrics", nil, nil, &metricsNames) + return metricsNames, err +} + // RepoGC runs garbage collection on IPFS daemons of cluster peers and // returns collected CIDs. If local is true, it would garbage collect // only on contacted peer, otherwise on all peers' IPFS daemons. @@ -582,7 +599,10 @@ func (c *defaultClient) AddMultiFile( // This method must run with StreamChannels set. params.StreamChannels = true - queryStr := params.ToQueryString() + queryStr, err := params.ToQueryString() + if err != nil { + return err + } // our handler decodes an AddedOutput and puts it // in the out channel. @@ -599,7 +619,7 @@ func (c *defaultClient) AddMultiFile( return nil } - err := c.doStream(ctx, + err = c.doStream(ctx, "POST", "/add?"+queryStr, headers, diff --git a/api/rest/client/methods_test.go b/api/rest/client/methods_test.go index 4b2c5012..561f24e8 100644 --- a/api/rest/client/methods_test.go +++ b/api/rest/client/methods_test.go @@ -489,6 +489,25 @@ func TestMetrics(t *testing.T) { testClients(t, api, testF) } +func TestMetricNames(t *testing.T) { + ctx := context.Background() + api := testAPI(t) + defer shutdown(api) + + testF := func(t *testing.T, c Client) { + m, err := c.MetricNames(ctx) + if err != nil { + t.Fatal(err) + } + + if len(m) == 0 { + t.Fatal("No metric names found") + } + } + + testClients(t, api, testF) +} + type waitService struct { l sync.Mutex pinStart time.Time diff --git a/api/rest/client/transports.go b/api/rest/client/transports.go index b72cea26..3a9f406a 100644 --- a/api/rest/client/transports.go +++ b/api/rest/client/transports.go @@ -14,6 +14,9 @@ import ( ipnet "github.com/libp2p/go-libp2p-core/pnet" p2phttp "github.com/libp2p/go-libp2p-http" pnet "github.com/libp2p/go-libp2p-pnet" + libp2pquic "github.com/libp2p/go-libp2p-quic-transport" + secio "github.com/libp2p/go-libp2p-secio" + libp2ptls "github.com/libp2p/go-libp2p-tls" madns "github.com/multiformats/go-multiaddr-dns" ) @@ -63,7 +66,13 @@ func (c *defaultClient) enableLibp2p() error { } } - h, err := libp2p.New(c.ctx, libp2p.PrivateNetwork(prot)) + h, err := libp2p.New(c.ctx, + libp2p.PrivateNetwork(prot), + libp2p.Security(libp2ptls.ID, libp2ptls.New), + libp2p.Security(secio.ID, secio.New), + libp2p.Transport(libp2pquic.NewTransport), + libp2p.DefaultTransports, + ) if err != nil { return err } diff --git a/api/rest/restapi.go b/api/rest/restapi.go index 9854294e..b79dd8c0 100644 --- a/api/rest/restapi.go +++ b/api/rest/restapi.go @@ -35,6 +35,9 @@ import ( rpc "github.com/libp2p/go-libp2p-gorpc" gostream "github.com/libp2p/go-libp2p-gostream" p2phttp "github.com/libp2p/go-libp2p-http" + libp2pquic "github.com/libp2p/go-libp2p-quic-transport" + secio "github.com/libp2p/go-libp2p-secio" + libp2ptls "github.com/libp2p/go-libp2p-tls" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -238,6 +241,10 @@ func (api *API) setupLibp2p() error { context.Background(), libp2p.Identity(api.config.PrivateKey), libp2p.ListenAddrs([]ma.Multiaddr{api.config.Libp2pListenAddr}...), + libp2p.Security(libp2ptls.ID, libp2ptls.New), + libp2p.Security(secio.ID, secio.New), + libp2p.Transport(libp2pquic.NewTransport), + libp2p.DefaultTransports, ) if err != nil { return err @@ -474,6 +481,12 @@ func (api *API) routes() []route { "/monitor/metrics/{name}", api.metricsHandler, }, + { + "MetricNames", + "GET", + "/monitor/metrics", + api.metricNamesHandler, + }, } } @@ -633,6 +646,19 @@ func (api *API) metricsHandler(w http.ResponseWriter, r *http.Request) { api.sendResponse(w, autoStatus, err, metrics) } +func (api *API) metricNamesHandler(w http.ResponseWriter, r *http.Request) { + var metricNames []string + err := api.rpcClient.CallContext( + r.Context(), + "", + "PeerMonitor", + "MetricNames", + struct{}{}, + &metricNames, + ) + api.sendResponse(w, autoStatus, err, metricNames) +} + func (api *API) addHandler(w http.ResponseWriter, r *http.Request) { reader, err := r.MultipartReader() if err != nil { diff --git a/api/rest/restapi_test.go b/api/rest/restapi_test.go index d197fe72..39a27c58 100644 --- a/api/rest/restapi_test.go +++ b/api/rest/restapi_test.go @@ -601,8 +601,12 @@ type pathCase struct { expectedCid string } -func (p *pathCase) WithQuery() string { - return p.path + "?" + p.opts.ToQuery() +func (p *pathCase) WithQuery(t *testing.T) string { + query, err := p.opts.ToQuery() + if err != nil { + t.Fatal(err) + } + return p.path + "?" + query } var testPinOpts = api.PinOptions{ @@ -610,6 +614,7 @@ var testPinOpts = api.PinOptions{ ReplicationFactorMin: 6, Name: "hello there", UserAllocations: []peer.ID{test.PeerID1, test.PeerID2}, + ExpireAt: time.Now().Add(30 * time.Second), } var pathTestCases = []pathCase{ @@ -660,7 +665,8 @@ func TestAPIPinEndpointWithPath(t *testing.T) { if testCase.wantErr { errResp := api.Error{} - makePost(t, rest, url(rest)+"/pins"+testCase.WithQuery(), []byte{}, &errResp) + q := testCase.WithQuery(t) + makePost(t, rest, url(rest)+"/pins"+q, []byte{}, &errResp) if errResp.Code != testCase.code { t.Errorf( "status code: expected: %d, got: %d, path: %s\n", @@ -672,7 +678,8 @@ func TestAPIPinEndpointWithPath(t *testing.T) { continue } pin := api.Pin{} - makePost(t, rest, url(rest)+"/pins"+testCase.WithQuery(), []byte{}, &pin) + q := testCase.WithQuery(t) + makePost(t, rest, url(rest)+"/pins"+q, []byte{}, &pin) if !pin.Equals(resultantPin) { t.Errorf("pin: expected: %+v", resultantPin) t.Errorf("pin: got: %+v", pin) @@ -826,6 +833,22 @@ func TestAPIMetricsEndpoint(t *testing.T) { testBothEndpoints(t, tf) } +func TestAPIMetricNamesEndpoint(t *testing.T) { + ctx := context.Background() + rest := testAPI(t) + defer rest.Shutdown(ctx) + + tf := func(t *testing.T, url urlF) { + var resp []string + makeGet(t, rest, url(rest)+"/monitor/metrics", &resp) + if len(resp) == 0 { + t.Fatal("No metric names found") + } + } + + testBothEndpoints(t, tf) +} + func TestAPIStatusAllEndpoint(t *testing.T) { ctx := context.Background() rest := testAPI(t) diff --git a/api/types.go b/api/types.go index d2b76dee..a7aea655 100644 --- a/api/types.go +++ b/api/types.go @@ -10,7 +10,6 @@ package api import ( "encoding/json" - "errors" "fmt" "net/url" "sort" @@ -32,10 +31,13 @@ import ( _ "github.com/multiformats/go-multiaddr-dns" proto "github.com/gogo/protobuf/proto" + "github.com/pkg/errors" ) var logger = logging.Logger("apitypes") +var unixZero = time.Unix(0, 0) + func init() { // Use /p2p/ multiaddresses multiaddr.SwapToP2pMultiaddrs() @@ -289,11 +291,14 @@ type Version struct { // then id will be a key of IPFSLinks. In the event of a SwarmPeers error // IPFSLinks[id] == []. type ConnectGraph struct { - ClusterID peer.ID + ClusterID peer.ID `json:"cluster_id" codec:"id"` + IDtoPeername map[string]string `json:"id_to_peername" codec:"ip,omitempty"` // ipfs to ipfs links IPFSLinks map[string][]peer.ID `json:"ipfs_links" codec:"il,omitempty"` // cluster to cluster links ClusterLinks map[string][]peer.ID `json:"cluster_links" codec:"cl,omitempty"` + // cluster trust links + ClusterTrustLinks map[string]bool `json:"cluster_trust_links" codec:"ctl,omitempty"` // cluster to ipfs links ClustertoIPFS map[string]peer.ID `json:"cluster_to_ipfs" codec:"ci,omitempty"` } @@ -464,6 +469,7 @@ type PinOptions struct { Name string `json:"name" codec:"n,omitempty"` ShardSize uint64 `json:"shard_size" codec:"s,omitempty"` UserAllocations []peer.ID `json:"user_allocations" codec:"ua,omitempty"` + ExpireAt time.Time `json:"expire_at" codec:"e,omitempty"` Metadata map[string]string `json:"metadata" codec:"m,omitempty"` PinUpdate cid.Cid `json:"pin_update,omitempty" codec:"pu,omitempty"` } @@ -510,6 +516,10 @@ func (po *PinOptions) Equals(po2 *PinOptions) bool { return false } + if !po.ExpireAt.Equal(po2.ExpireAt) { + return false + } + for k, v := range po.Metadata { v2 := po2.Metadata[k] if k != "" && v != v2 { @@ -523,13 +533,20 @@ func (po *PinOptions) Equals(po2 *PinOptions) bool { } // ToQuery returns the PinOption as query arguments. -func (po *PinOptions) ToQuery() string { +func (po *PinOptions) ToQuery() (string, error) { q := url.Values{} q.Set("replication-min", fmt.Sprintf("%d", po.ReplicationFactorMin)) q.Set("replication-max", fmt.Sprintf("%d", po.ReplicationFactorMax)) q.Set("name", po.Name) q.Set("shard-size", fmt.Sprintf("%d", po.ShardSize)) q.Set("user-allocations", strings.Join(PeersToStrings(po.UserAllocations), ",")) + if !po.ExpireAt.IsZero() { + v, err := po.ExpireAt.MarshalText() + if err != nil { + return "", err + } + q.Set("expire-at", string(v)) + } for k, v := range po.Metadata { if k == "" { continue @@ -539,7 +556,7 @@ func (po *PinOptions) ToQuery() string { if po.PinUpdate != cid.Undef { q.Set("pin-update", po.PinUpdate.String()) } - return q.Encode() + return q.Encode(), nil } // FromQuery is the inverse of ToQuery(). @@ -573,6 +590,24 @@ func (po *PinOptions) FromQuery(q url.Values) error { po.UserAllocations = StringsToPeers(strings.Split(allocs, ",")) } + if v := q.Get("expire-at"); v != "" { + var tm time.Time + err := tm.UnmarshalText([]byte(v)) + if err != nil { + return errors.Wrap(err, "expire-at cannot be parsed") + } + po.ExpireAt = tm + } else if v = q.Get("expire-in"); v != "" { + d, err := time.ParseDuration(v) + if err != nil { + return errors.Wrap(err, "expire-in cannot be parsed") + } + if d < time.Second { + return errors.New("expire-in duration too short") + } + po.ExpireAt = time.Now().Add(d) + } + po.Metadata = make(map[string]string) for k := range q { if !strings.HasPrefix(k, pinOptionsMetaPrefix) { @@ -683,6 +718,12 @@ func (pin *Pin) ProtoMarshal() ([]byte, error) { allocs[i] = bs } + var expireAtProto uint64 + // Only set the protobuf field with non-zero times. + if !(pin.ExpireAt.IsZero() || pin.ExpireAt.Equal(unixZero)) { + expireAtProto = uint64(pin.ExpireAt.Unix()) + } + opts := &pb.PinOptions{ ReplicationFactorMin: int32(pin.ReplicationFactorMin), ReplicationFactorMax: int32(pin.ReplicationFactorMax), @@ -691,6 +732,7 @@ func (pin *Pin) ProtoMarshal() ([]byte, error) { // UserAllocations: pin.UserAllocations, Metadata: pin.Metadata, PinUpdate: pin.PinUpdate.Bytes(), + ExpireAt: expireAtProto, } pbPin := &pb.Pin{ @@ -749,8 +791,11 @@ func (pin *Pin) ProtoUnmarshal(data []byte) error { pin.Name = opts.GetName() pin.ShardSize = opts.GetShardSize() // pin.UserAllocations = opts.GetUserAllocations() + t := opts.GetExpireAt() + if t > 0 { + pin.ExpireAt = time.Unix(int64(t), 0) + } pin.Metadata = opts.GetMetadata() - pinUpdate, err := cid.Cast(opts.GetPinUpdate()) if err == nil { pin.PinUpdate = pinUpdate @@ -823,6 +868,15 @@ func (pin *Pin) IsRemotePin(pid peer.ID) bool { return true } +// ExpiredAt returns whether the pin has expired at the given time. +func (pin *Pin) ExpiredAt(t time.Time) bool { + if pin.ExpireAt.IsZero() || pin.ExpireAt.Equal(unixZero) { + return false + } + + return pin.ExpireAt.Before(t) +} + // NodeWithMeta specifies a block of data and a set of optional metadata fields // carrying information about the encoded ipld node type NodeWithMeta struct { @@ -875,6 +929,18 @@ func (m *Metric) Discard() bool { return !m.Valid || m.Expired() } +// MetricSlice is a sortable Metric array. +type MetricSlice []*Metric + +func (es MetricSlice) Len() int { return len(es) } +func (es MetricSlice) Swap(i, j int) { es[i], es[j] = es[j], es[i] } +func (es MetricSlice) Less(i, j int) bool { + if es[i].Peer == es[j].Peer { + return es[i].Expire < es[j].Expire + } + return es[i].Peer < es[j].Peer +} + // Alert carries alerting information about a peer. WIP. type Alert struct { Peer peer.ID diff --git a/api/types_test.go b/api/types_test.go index 184fe386..fad8c8c9 100644 --- a/api/types_test.go +++ b/api/types_test.go @@ -184,6 +184,7 @@ func TestPinOptionsQuery(t *testing.T) { "QmXZrtE5jQwXNqCJMfHUTQkvhQ4ZAnqMnmzFMJfLewuabc", "QmUZ13osndQ5uL4tPWHXe3iBgBgq9gfewcBMSCAuMBsDJ6", }), + ExpireAt: time.Now().Add(12 * time.Hour), Metadata: map[string]string{ "hello": "bye", "hello2": "bye2", @@ -210,7 +211,10 @@ func TestPinOptionsQuery(t *testing.T) { } for _, tc := range testcases { - queryStr := tc.ToQuery() + queryStr, err := tc.ToQuery() + if err != nil { + t.Fatal("error converting to query", err) + } q, err := url.ParseQuery(queryStr) if err != nil { t.Error("error parsing query", err) diff --git a/cluster.go b/cluster.go index 73fc4685..20edf617 100644 --- a/cluster.go +++ b/cluster.go @@ -877,8 +877,8 @@ func (c *Cluster) Join(ctx context.Context, addr ma.Multiaddr) error { logger.Debugf("Join(%s)", addr) - // Add peer to peerstore so we can talk to it (and connect) - pid, err := c.peerManager.ImportPeer(addr, true, peerstore.PermanentAddrTTL) + // Add peer to peerstore so we can talk to it + pid, err := c.peerManager.ImportPeer(addr, false, peerstore.PermanentAddrTTL) if err != nil { return err } @@ -964,6 +964,7 @@ func (c *Cluster) StateSync(ctx context.Context) error { } logger.Debug("syncing state to tracker") + timeNow := time.Now() clusterPins, err := cState.List(ctx) if err != nil { return err @@ -987,9 +988,29 @@ func (c *Cluster) StateSync(ctx context.Context) error { } } + isClosest := func(cid.Cid) bool { + return false + } + + if !c.config.FollowerMode { + trustedPeers, err := c.getTrustedPeers(ctx) + if err != nil { + return nil + } + checker := distanceChecker{ + local: c.id, + otherPeers: trustedPeers, + cache: make(map[peer.ID]distance, len(trustedPeers)+1), + } + isClosest = func(c cid.Cid) bool { + return checker.isClosest(c) + } + } + // a. Untrack items which should not be tracked - // b. Track items which should not be remote as local - // c. Track items which should not be local as remote + // b. Unpin items which have expired + // c. Track items which should not be remote as local + // d. Track items which should not be local as remote for _, p := range trackedPins { pCid := p.Cid currentPin, err := cState.Get(ctx, pCid) @@ -1003,16 +1024,15 @@ func (c *Cluster) StateSync(ctx context.Context) error { continue } - allocatedHere := containsPeer(currentPin.Allocations, c.id) || currentPin.ReplicationFactorMin == -1 - - switch { - case p.Status == api.TrackerStatusRemote && allocatedHere: - logger.Debugf("StateSync: Tracking %s locally (currently remote)", pCid) - err = c.tracker.Track(ctx, currentPin) - case p.Status == api.TrackerStatusPinned && !allocatedHere: - logger.Debugf("StateSync: Tracking %s as remote (currently local)", pCid) - err = c.tracker.Track(ctx, currentPin) + if currentPin.ExpiredAt(timeNow) && isClosest(pCid) { + logger.Infof("Unpinning %s: pin expired at %s", pCid, currentPin.ExpireAt) + if _, err := c.Unpin(ctx, pCid); err != nil { + logger.Error(err) + } + continue } + + err = c.updateRemotePins(ctx, currentPin, p) if err != nil { return err } @@ -1021,6 +1041,22 @@ func (c *Cluster) StateSync(ctx context.Context) error { return nil } +func (c *Cluster) updateRemotePins(ctx context.Context, pin *api.Pin, p *api.PinInfo) error { + var err error + allocatedHere := pin.ReplicationFactorMin == -1 || containsPeer(pin.Allocations, c.id) + + switch { + case p.Status == api.TrackerStatusRemote && allocatedHere: + logger.Debugf("StateSync: Tracking %s locally (currently remote)", p.Cid) + err = c.tracker.Track(ctx, pin) + case p.Status == api.TrackerStatusPinned && !allocatedHere: + logger.Debugf("StateSync: Tracking %s as remote (currently local)", p.Cid) + err = c.tracker.Track(ctx, pin) + } + + return err +} + // StatusAll returns the GlobalPinInfo for all tracked Cids in all peers. // If an error happens, the slice will contain as much information as // could be fetched from other peers. @@ -1330,6 +1366,10 @@ func (c *Cluster) setupPin(ctx context.Context, pin *api.Pin) error { return err } + if !pin.ExpireAt.IsZero() && pin.ExpireAt.Before(time.Now()) { + return errors.New("pin.ExpireAt set before current time") + } + existing, err := c.PinGet(ctx, pin.Cid) if err != nil && err != state.ErrNotFound { return err @@ -1619,6 +1659,25 @@ func (c *Cluster) Peers(ctx context.Context) []*api.ID { return peers } +// getTrustedPeers gives listed of trusted peers except the current peer. +func (c *Cluster) getTrustedPeers(ctx context.Context) ([]peer.ID, error) { + peers, err := c.consensus.Peers(ctx) + if err != nil { + return nil, err + } + + trustedPeers := make([]peer.ID, 0, len(peers)) + + for _, p := range peers { + if p == c.id || !c.consensus.IsTrustedPeer(ctx, p) { + continue + } + trustedPeers = append(trustedPeers, p) + } + + return trustedPeers, nil +} + func (c *Cluster) globalPinInfoCid(ctx context.Context, comp, method string, h cid.Cid) (*api.GlobalPinInfo, error) { ctx, span := trace.StartSpan(ctx, "cluster/globalPinInfoCid") defer span.End() diff --git a/cluster_config.go b/cluster_config.go index d65be26e..bdb39b4f 100644 --- a/cluster_config.go +++ b/cluster_config.go @@ -13,6 +13,7 @@ import ( "github.com/ipfs/ipfs-cluster/config" + ipfsconfig "github.com/ipfs/go-ipfs-config" pnet "github.com/libp2p/go-libp2p-pnet" ma "github.com/multiformats/go-multiaddr" @@ -21,9 +22,12 @@ import ( const configKey = "cluster" +// DefaultListenAddrs contains TCP and QUIC listen addresses +var DefaultListenAddrs = []string{"/ip4/0.0.0.0/tcp/9096", "/ip4/0.0.0.0/udp/9096/quic"} + // Configuration defaults const ( - DefaultListenAddr = "/ip4/0.0.0.0/tcp/9096" + DefaultEnableRelayHop = true DefaultStateSyncInterval = 600 * time.Second DefaultIPFSSyncInterval = 130 * time.Second DefaultPinRecoverInterval = 1 * time.Hour @@ -73,7 +77,11 @@ type Config struct { // Listen parameters for the Cluster libp2p Host. Used by // the RPC and Consensus components. - ListenAddr ma.Multiaddr + ListenAddr []ma.Multiaddr + + // Enables HOP relay for the node. If this is enabled, the node will act as + // an intermediate (Hop Relay) node in relay circuits for connected peers. + EnableRelayHop bool // ConnMgr holds configuration values for the connection manager for // the libp2p host. @@ -160,7 +168,8 @@ type configJSON struct { PrivateKey string `json:"private_key,omitempty"` Secret string `json:"secret"` LeaveOnShutdown bool `json:"leave_on_shutdown"` - ListenMultiaddress string `json:"listen_multiaddress"` + ListenMultiaddress ipfsconfig.Strings `json:"listen_multiaddress"` + EnableRelayHop bool `json:"enable_relay_hop"` ConnectionManager *connMgrConfigJSON `json:"connection_manager"` StateSyncInterval string `json:"state_sync_interval"` IPFSSyncInterval string `json:"ipfs_sync_interval"` @@ -228,6 +237,10 @@ func (cfg *Config) Validate() error { return errors.New("cluster.listen_multiaddress is undefined") } + if len(cfg.ListenAddr) == 0 { + return errors.New("cluster.listen_multiaddress is empty") + } + if cfg.ConnMgr.LowWater <= 0 { return errors.New("cluster.connection_manager.low_water is invalid") } @@ -334,8 +347,13 @@ func (cfg *Config) setDefaults() { } cfg.Peername = hostname - addr, _ := ma.NewMultiaddr(DefaultListenAddr) - cfg.ListenAddr = addr + listenAddrs := []ma.Multiaddr{} + for _, m := range DefaultListenAddrs { + addr, _ := ma.NewMultiaddr(m) + listenAddrs = append(listenAddrs, addr) + } + cfg.ListenAddr = listenAddrs + cfg.EnableRelayHop = DefaultEnableRelayHop cfg.ConnMgr = ConnMgrConfig{ HighWater: DefaultConnMgrHighWater, LowWater: DefaultConnMgrLowWater, @@ -384,13 +402,18 @@ func (cfg *Config) applyConfigJSON(jcfg *configJSON) error { } cfg.Secret = clusterSecret - clusterAddr, err := ma.NewMultiaddr(jcfg.ListenMultiaddress) - if err != nil { - err = fmt.Errorf("error parsing cluster_listen_multiaddress: %s", err) - return err + var listenAddrs []ma.Multiaddr + for _, addr := range jcfg.ListenMultiaddress { + listenAddr, err := ma.NewMultiaddr(addr) + if err != nil { + err = fmt.Errorf("error parsing a listen_multiaddress: %s", err) + return err + } + listenAddrs = append(listenAddrs, listenAddr) } - cfg.ListenAddr = clusterAddr + cfg.ListenAddr = listenAddrs + cfg.EnableRelayHop = jcfg.EnableRelayHop if conman := jcfg.ConnectionManager; conman != nil { cfg.ConnMgr = ConnMgrConfig{ HighWater: jcfg.ConnectionManager.HighWater, @@ -455,7 +478,12 @@ func (cfg *Config) toConfigJSON() (jcfg *configJSON, err error) { jcfg.ReplicationFactorMin = cfg.ReplicationFactorMin jcfg.ReplicationFactorMax = cfg.ReplicationFactorMax jcfg.LeaveOnShutdown = cfg.LeaveOnShutdown - jcfg.ListenMultiaddress = cfg.ListenAddr.String() + var listenAddrs ipfsconfig.Strings + for _, addr := range cfg.ListenAddr { + listenAddrs = append(listenAddrs, addr.String()) + } + jcfg.ListenMultiaddress = ipfsconfig.Strings(listenAddrs) + jcfg.EnableRelayHop = cfg.EnableRelayHop jcfg.ConnectionManager = &connMgrConfigJSON{ HighWater: cfg.ConnMgr.HighWater, LowWater: cfg.ConnMgr.LowWater, diff --git a/cluster_config_test.go b/cluster_config_test.go index 98d2cca4..3ca3e120 100644 --- a/cluster_config_test.go +++ b/cluster_config_test.go @@ -5,6 +5,8 @@ import ( "os" "testing" "time" + + ipfsconfig "github.com/ipfs/go-ipfs-config" ) var ccfgTestJSON = []byte(` @@ -17,7 +19,10 @@ var ccfgTestJSON = []byte(` "low_water": 500, "grace_period": "100m0s" }, - "listen_multiaddress": "/ip4/127.0.0.1/tcp/10000", + "listen_multiaddress": [ + "/ip4/127.0.0.1/tcp/10000", + "/ip4/127.0.0.1/udp/10000/quic" + ], "state_sync_interval": "1m0s", "ipfs_sync_interval": "2m10s", "pin_recover_interval": "1m", @@ -114,7 +119,7 @@ func TestLoadJSON(t *testing.T) { }) t.Run("bad listen multiaddress", func(t *testing.T) { - _, err := loadJSON2(t, func(j *configJSON) { j.ListenMultiaddress = "abc" }) + _, err := loadJSON2(t, func(j *configJSON) { j.ListenMultiaddress = ipfsconfig.Strings{"abc"} }) if err == nil { t.Error("expected error parsing listen_multiaddress") } @@ -197,7 +202,10 @@ func TestLoadJSON(t *testing.T) { func TestToJSON(t *testing.T) { cfg := &Config{} - cfg.LoadJSON(ccfgTestJSON) + err := cfg.LoadJSON(ccfgTestJSON) + if err != nil { + t.Fatal(err) + } newjson, err := cfg.ToJSON() if err != nil { t.Fatal(err) diff --git a/cluster_test.go b/cluster_test.go index 98370295..6bd2ef5d 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -310,6 +310,21 @@ func TestClusterPin(t *testing.T) { } } +func TestPinExpired(t *testing.T) { + ctx := context.Background() + cl, _, _, _ := testingCluster(t) + defer cleanState() + defer cl.Shutdown(ctx) + + c := test.Cid1 + _, err := cl.Pin(ctx, c, api.PinOptions{ + ExpireAt: time.Now(), + }) + if err == nil { + t.Fatal("pin should have errored") + } +} + func TestClusterPinPath(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) diff --git a/clusterhost.go b/clusterhost.go index 034ef0f0..4c7a891d 100644 --- a/clusterhost.go +++ b/clusterhost.go @@ -6,20 +6,27 @@ import ( "github.com/ipfs/ipfs-cluster/config" libp2p "github.com/libp2p/go-libp2p" + autonat "github.com/libp2p/go-libp2p-autonat-svc" + relay "github.com/libp2p/go-libp2p-circuit" connmgr "github.com/libp2p/go-libp2p-connmgr" + corepnet "github.com/libp2p/go-libp2p-core/pnet" + routing "github.com/libp2p/go-libp2p-core/routing" crypto "github.com/libp2p/go-libp2p-crypto" host "github.com/libp2p/go-libp2p-host" - ipnet "github.com/libp2p/go-libp2p-interface-pnet" dht "github.com/libp2p/go-libp2p-kad-dht" pnet "github.com/libp2p/go-libp2p-pnet" pubsub "github.com/libp2p/go-libp2p-pubsub" + libp2pquic "github.com/libp2p/go-libp2p-quic-transport" + secio "github.com/libp2p/go-libp2p-secio" + libp2ptls "github.com/libp2p/go-libp2p-tls" routedhost "github.com/libp2p/go-libp2p/p2p/host/routed" ) -// NewClusterHost creates a libp2p Host with the options from the provided -// cluster configuration. Using that host, it creates pubsub and a DHT -// instances, for shared use by all cluster components. The returned host uses -// the DHT for routing. The resulting DHT is not bootstrapped. +// NewClusterHost creates a fully-featured libp2p Host with the options from +// the provided cluster configuration. Using that host, it creates pubsub and +// a DHT instances, for shared use by all cluster components. The returned +// host uses the DHT for routing. The resulting DHT is not bootstrapped. Relay +// and AutoNATService are additionally setup for this host. func NewClusterHost( ctx context.Context, ident *config.Identity, @@ -28,13 +35,35 @@ func NewClusterHost( connman := connmgr.NewConnManager(cfg.ConnMgr.LowWater, cfg.ConnMgr.HighWater, cfg.ConnMgr.GracePeriod) - h, err := newHost( - ctx, - cfg.Secret, - ident.PrivateKey, - libp2p.ListenAddrs(cfg.ListenAddr), + relayOpts := []relay.RelayOpt{relay.OptDiscovery} + if cfg.EnableRelayHop { + relayOpts = append(relayOpts, relay.OptHop) + } + + var idht *dht.IpfsDHT + var err error + opts := []libp2p.Option{ + libp2p.ListenAddrs(cfg.ListenAddr...), libp2p.NATPortMap(), libp2p.ConnectionManager(connman), + libp2p.Routing(func(h host.Host) (routing.PeerRouting, error) { + idht, err = newDHT(ctx, h) + return idht, err + }), + libp2p.EnableRelay(relayOpts...), + libp2p.EnableAutoRelay(), + } + + prot, err := newProtector(cfg.Secret) + if err != nil { + return nil, nil, nil, err + } + + h, err := newHost( + ctx, + prot, + ident.PrivateKey, + opts..., ) if err != nil { return nil, nil, nil, err @@ -46,39 +75,55 @@ func NewClusterHost( return nil, nil, nil, err } - idht, err := newDHT(ctx, h) + // needed for auto relay + _, err = autonat.NewAutoNATService(ctx, h, baseOpts(prot)...) if err != nil { h.Close() return nil, nil, nil, err } - return routedHost(h, idht), psub, idht, nil + return h, psub, idht, nil } -func newHost(ctx context.Context, secret []byte, priv crypto.PrivKey, opts ...libp2p.Option) (host.Host, error) { - var prot ipnet.Protector - var err error - - // Create protector if we have a secret. - if secret != nil && len(secret) > 0 { - var key [32]byte - copy(key[:], secret) - prot, err = pnet.NewV1ProtectorFromBytes(&key) - if err != nil { - return nil, err - } - } - +// newHost creates a base cluster host without dht, pubsub, relay or nat etc. +// mostly used for testing. +func newHost(ctx context.Context, prot corepnet.Protector, priv crypto.PrivKey, opts ...libp2p.Option) (host.Host, error) { finalOpts := []libp2p.Option{ libp2p.Identity(priv), - libp2p.PrivateNetwork(prot), } + finalOpts = append(finalOpts, baseOpts(prot)...) finalOpts = append(finalOpts, opts...) - return libp2p.New( + h, err := libp2p.New( ctx, finalOpts..., ) + if err != nil { + return nil, err + } + + return h, nil +} + +func baseOpts(prot corepnet.Protector) []libp2p.Option { + return []libp2p.Option{ + libp2p.PrivateNetwork(prot), + libp2p.Security(libp2ptls.ID, libp2ptls.New), + libp2p.Security(secio.ID, secio.New), + libp2p.Transport(libp2pquic.NewTransport), + libp2p.DefaultTransports, + } +} + +func newProtector(secret []byte) (corepnet.Protector, error) { + // Create protector if we have a secret. + if len(secret) == 0 { + return nil, nil + } + + var key [32]byte + copy(key[:], secret) + return pnet.NewV1ProtectorFromBytes(&key) } func newDHT(ctx context.Context, h host.Host) (*dht.IpfsDHT, error) { diff --git a/cmd/ipfs-cluster-ctl/Makefile b/cmd/ipfs-cluster-ctl/Makefile index bd8fee1c..f1332bd6 100644 --- a/cmd/ipfs-cluster-ctl/Makefile +++ b/cmd/ipfs-cluster-ctl/Makefile @@ -4,7 +4,7 @@ SRC := $(shell find .. -type f -name '*.go') all: ipfs-cluster-ctl ipfs-cluster-ctl: $(SRC) - go build + go build -mod=readonly build: ipfs-cluster-ctl diff --git a/cmd/ipfs-cluster-ctl/formatters.go b/cmd/ipfs-cluster-ctl/formatters.go index 58cec062..317da9fe 100644 --- a/cmd/ipfs-cluster-ctl/formatters.go +++ b/cmd/ipfs-cluster-ctl/formatters.go @@ -5,12 +5,15 @@ import ( "errors" "fmt" "sort" + "strconv" "strings" "time" "github.com/ipfs/ipfs-cluster/api" peer "github.com/libp2p/go-libp2p-core/peer" + + humanize "github.com/dustin/go-humanize" ) type addedOutputQuiet struct { @@ -46,6 +49,8 @@ func textFormatObject(resp interface{}) { switch resp.(type) { case nil: return + case string: + fmt.Println(resp) case *api.ID: textFormatPrintID(resp.(*api.ID)) case *api.GlobalPinInfo: @@ -88,6 +93,10 @@ func textFormatObject(resp interface{}) { } case *api.GlobalRepoGC: textFormatPrintGlobalRepoGC(resp.(*api.GlobalRepoGC)) + case []string: + for _, item := range resp.([]string) { + textFormatObject(item) + } default: checkErr("", errors.New("unsupported type returned")) } @@ -213,8 +222,14 @@ func textFormatPrintAddedOutputQuiet(obj *addedOutputQuiet) { } func textFormatPrintMetric(obj *api.Metric) { - date := time.Unix(0, obj.Expire).UTC().Format(time.RFC3339) - fmt.Printf("%s: %s | Expire: %s\n", peer.IDB58Encode(obj.Peer), obj.Value, date) + if obj.Name == "freespace" { + u, err := strconv.ParseUint(obj.Value, 10, 64) + checkErr("parsing to uint64", err) + fmt.Printf("%s | freespace: %s | Expires in: %s\n", peer.IDB58Encode(obj.Peer), humanize.Bytes(u), humanize.Time(time.Unix(0, obj.Expire))) + return + } + + fmt.Printf("%s | %s | Expires in: %s\n", peer.IDB58Encode(obj.Peer), obj.Name, humanize.Time(time.Unix(0, obj.Expire))) } func textFormatPrintGlobalRepoGC(obj *api.GlobalRepoGC) { diff --git a/cmd/ipfs-cluster-ctl/graph.go b/cmd/ipfs-cluster-ctl/graph.go index d6a76545..fd71fdaf 100644 --- a/cmd/ipfs-cluster-ctl/graph.go +++ b/cmd/ipfs-cluster-ctl/graph.go @@ -6,8 +6,8 @@ import ( "io" "sort" + dot "github.com/kishansagathiya/go-dot" peer "github.com/libp2p/go-libp2p-peer" - dot "github.com/zenground0/go-dot" "github.com/ipfs/ipfs-cluster/api" ) @@ -31,8 +31,10 @@ import ( type nodeType int const ( - tCluster nodeType = iota // The cluster node type - tIpfs // The IPFS node type + tSelfCluster nodeType = iota // cluster self node + tCluster // cluster node + tTrustedCluster // trusted cluster node + tIPFS // IPFS node linked to a Cluster node ) var errUnfinishedWrite = errors.New("could not complete write of line to output") @@ -62,6 +64,9 @@ func makeDot(cg *api.ConnectGraph, w io.Writer, allIpfs bool) error { dW := dotWriter{ w: w, dotGraph: dot.NewGraph("cluster"), + self: peer.IDB58Encode(cg.ClusterID), + trustMap: cg.ClusterTrustLinks, + idToPeername: cg.IDtoPeername, ipfsEdges: ipfsEdges, clusterEdges: cg.ClusterLinks, clusterIpfsEdges: cg.ClustertoIPFS, @@ -78,53 +83,117 @@ type dotWriter struct { w io.Writer dotGraph dot.Graph + self string + idToPeername map[string]string + trustMap map[string]bool ipfsEdges map[string][]peer.ID clusterEdges map[string][]peer.ID clusterIpfsEdges map[string]peer.ID } +func (dW *dotWriter) addSubGraph(sGraph dot.Graph, rank string) { + sGraph.IsSubGraph = true + sGraph.Rank = rank + dW.dotGraph.AddSubGraph(&sGraph) +} + // writes nodes to dot file output and creates and stores an ordering over nodes -func (dW *dotWriter) addNode(id string, nT nodeType) error { - var node dot.VertexDescription - pid, _ := peer.IDB58Decode(id) - node.Label = pid.ShortString() +func (dW *dotWriter) addNode(graph *dot.Graph, id string, nT nodeType) error { + node := dot.NewVertexDescription("") + node.Group = id + node.ColorScheme = "x11" + node.FontName = "Ariel" + node.Shape = "ellipse" + node.Style = "filled" switch nT { - case tCluster: + case tSelfCluster: + node.Shape = "box3d" + node.Label = label(dW.idToPeername[id], shorten(id)) node.ID = fmt.Sprintf("C%d", len(dW.clusterNodes)) - node.Color = "blue2" + node.Color = "orange" + node.Peripheries = 2 + node.FontColor = "black" dW.clusterNodes[id] = &node - case tIpfs: + case tTrustedCluster: + node.Shape = "box3d" + node.Label = label(dW.idToPeername[id], shorten(id)) + node.ID = fmt.Sprintf("T%d", len(dW.clusterNodes)) + node.Color = "orange" + node.FontColor = "black" + dW.clusterNodes[id] = &node + case tCluster: + node.Shape = "box3d" + node.Label = label(dW.idToPeername[id], shorten(id)) + node.ID = fmt.Sprintf("C%d", len(dW.clusterNodes)) + node.Color = "darkorange3" + node.FontColor = "black" + dW.clusterNodes[id] = &node + case tIPFS: node.ID = fmt.Sprintf("I%d", len(dW.ipfsNodes)) - node.Color = "goldenrod" - dW.ipfsNodes[id] = &node + node.Shape = "cylinder" + ipfsID, ok := dW.clusterIpfsEdges[id] + if !ok { + node.Label = label("IPFS", "Errored") + node.ColorScheme = "X11" + node.Color = "firebrick1" + node.FontColor = "black" + dW.ipfsNodes[id] = &node + } else { + ipfsIDStr := peer.IDB58Encode(ipfsID) + node.Label = label("IPFS", shorten(ipfsIDStr)) + node.Color = "turquoise3" + node.FontColor = "black" + dW.ipfsNodes[ipfsIDStr] = &node + } default: return errUnknownNodeType } - dW.dotGraph.AddVertex(&node) + graph.AddVertex(&node) return nil } +func shorten(id string) string { + return id[:2] + "*" + id[len(id)-6:] +} + +func label(peername, id string) string { + return fmt.Sprintf("< %s
%s >", peername, id) +} + func (dW *dotWriter) print() error { dW.dotGraph.AddComment("The nodes of the connectivity graph") dW.dotGraph.AddComment("The cluster-service peers") // Write cluster nodes, use sorted order for consistent labels + sGraphCluster := dot.NewGraph("") + sGraphCluster.IsSubGraph = true for _, k := range sortedKeys(dW.clusterEdges) { - err := dW.addNode(k, tCluster) + var err error + if k == dW.self { + err = dW.addNode(&sGraphCluster, k, tSelfCluster) + } else if dW.trustMap[k] { + err = dW.addNode(&sGraphCluster, k, tTrustedCluster) + } else { + err = dW.addNode(&sGraphCluster, k, tCluster) + } if err != nil { return err } } + dW.addSubGraph(sGraphCluster, "min") dW.dotGraph.AddNewLine() - dW.dotGraph.AddComment("The ipfs peers") + dW.dotGraph.AddComment("The ipfs peers linked to cluster peers") + sGraphIPFS := dot.NewGraph("") + sGraphIPFS.IsSubGraph = true // Write ipfs nodes, use sorted order for consistent labels - for _, k := range sortedKeys(dW.ipfsEdges) { - err := dW.addNode(k, tIpfs) + for _, k := range sortedKeys(dW.clusterEdges) { + err := dW.addNode(&sGraphIPFS, k, tIPFS) if err != nil { return err } } + dW.addSubGraph(sGraphIPFS, "max") dW.dotGraph.AddNewLine() dW.dotGraph.AddComment("Edges representing active connections in the cluster") @@ -134,30 +203,52 @@ func (dW *dotWriter) print() error { for _, id := range v { toNode := dW.clusterNodes[k] fromNode := dW.clusterNodes[peer.IDB58Encode(id)] - dW.dotGraph.AddEdge(toNode, fromNode, true) + dW.dotGraph.AddEdge(toNode, fromNode, true, "") } } dW.dotGraph.AddNewLine() dW.dotGraph.AddComment("The connections between cluster peers and their ipfs daemons") // Write cluster to ipfs edges - for k, id := range dW.clusterIpfsEdges { + for k := range dW.clusterEdges { + var fromNode *dot.VertexDescription toNode := dW.clusterNodes[k] - fromNode := dW.ipfsNodes[peer.IDB58Encode(id)] - dW.dotGraph.AddEdge(toNode, fromNode, true) + ipfsID, ok := dW.clusterIpfsEdges[k] + if !ok { + fromNode, ok2 := dW.ipfsNodes[k] + if !ok2 { + logger.Warning("expected a node at this id") + continue + } + dW.dotGraph.AddEdge(toNode, fromNode, true, "dotted") + continue + } + + fromNode, ok = dW.ipfsNodes[peer.IDB58Encode(ipfsID)] + if !ok { + logger.Warning("expected a node at this id") + continue + } + dW.dotGraph.AddEdge(toNode, fromNode, true, "") } dW.dotGraph.AddNewLine() dW.dotGraph.AddComment("The swarm peer connections among ipfs daemons in the cluster") // Write ipfs edges - for k, v := range dW.ipfsEdges { + for _, k := range sortedKeys(dW.ipfsEdges) { + v := dW.ipfsEdges[k] + toNode := dW.ipfsNodes[k] for _, id := range v { - toNode := dW.ipfsNodes[k] - fromNode := dW.ipfsNodes[peer.IDB58Encode(id)] - dW.dotGraph.AddEdge(toNode, fromNode, true) + idStr := peer.IDB58Encode(id) + fromNode, ok := dW.ipfsNodes[idStr] + if !ok { + logger.Warning("expected a node here") + continue + } + dW.dotGraph.AddEdge(toNode, fromNode, true, "") } } - return dW.dotGraph.WriteDot(dW.w) + return dW.dotGraph.Write(dW.w) } func sortedKeys(dict map[string][]peer.ID) []string { diff --git a/cmd/ipfs-cluster-ctl/graph_test.go b/cmd/ipfs-cluster-ctl/graph_test.go index c32b130e..e38a9814 100644 --- a/cmd/ipfs-cluster-ctl/graph_test.go +++ b/cmd/ipfs-cluster-ctl/graph_test.go @@ -32,17 +32,24 @@ func verifyOutput(t *testing.T, outStr string, trueStr string) { } var simpleIpfs = `digraph cluster { - /* The nodes of the connectivity graph */ /* The cluster-service peers */ -C0 [label="" color="blue2"] -C1 [label="" color="blue2"] -C2 [label="" color="blue2"] +subgraph { +rank="min" +C0 [label=<
Qm*EhD > group="QmUBuxVHoNNjfmNpTad36UeaFQv3gXAtCv9r6KhmeqhEhD" color="11" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="ellipse" peripheries="2" ] +C1 [label=<
Qm*DQJ > group="QmV35LjbEGPfN7KfMAJp43VV2enwXqqQf5esx4vUcgHDQJ" color="9" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="ellipse" ] +C2 [label=<
Qm*mJu > group="QmZ2ckU7G35MYyJgMTwMUnicsGqSy3YUxGBX7qny6MQmJu" color="9" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="ellipse" ] +} -/* The ipfs peers */ -I0 [label="" color="goldenrod"] -I1 [label="" color="goldenrod"] -I2 [label="" color="goldenrod"] +/* The ipfs peers linked to cluster peers */ +subgraph { +rank="max" +I0 [label=< IPFS
Qm*ZDV > group="QmUBuxVHoNNjfmNpTad36UeaFQv3gXAtCv9r6KhmeqhEhD" color="1" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="box" ] +I1 [label=< IPFS
Qm*Ssq > group="QmV35LjbEGPfN7KfMAJp43VV2enwXqqQf5esx4vUcgHDQJ" color="1" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="box" ] +I2 [label=< IPFS
Qm*suL > group="QmZ2ckU7G35MYyJgMTwMUnicsGqSy3YUxGBX7qny6MQmJu" color="1" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="box" ] +} + +/* The ipfs swarm peers */ /* Edges representing active connections in the cluster */ /* The connections among cluster-service peers */ @@ -54,19 +61,18 @@ C2 -> C0 C2 -> C1 /* The connections between cluster peers and their ipfs daemons */ -C0 -> I1 -C1 -> I0 +C0 -> I0 +C1 -> I1 C2 -> I2 /* The swarm peer connections among ipfs daemons in the cluster */ -I0 -> I1 -I0 -> I2 I1 -> I0 I1 -> I2 +I0 -> I1 +I0 -> I2 I2 -> I0 I2 -> I1 - - }` +}` var ( pid1, _ = peer.IDB58Decode("QmUBuxVHoNNjfmNpTad36UeaFQv3gXAtCv9r6KhmeqhEhD") @@ -127,20 +133,27 @@ func TestSimpleIpfsGraphs(t *testing.T) { } var allIpfs = `digraph cluster { - /* The nodes of the connectivity graph */ /* The cluster-service peers */ -C0 [label="" color="blue2"] -C1 [label="" color="blue2"] -C2 [label="" color="blue2"] +subgraph { +rank="min" +C0 [label=<
Qm*EhD > group="QmUBuxVHoNNjfmNpTad36UeaFQv3gXAtCv9r6KhmeqhEhD" color="11" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="ellipse" peripheries="2" ] +C1 [label=<
Qm*DQJ > group="QmV35LjbEGPfN7KfMAJp43VV2enwXqqQf5esx4vUcgHDQJ" color="9" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="ellipse" ] +C2 [label=<
Qm*mJu > group="QmZ2ckU7G35MYyJgMTwMUnicsGqSy3YUxGBX7qny6MQmJu" color="9" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="ellipse" ] +} -/* The ipfs peers */ -I0 [label="" color="goldenrod"] -I1 [label="" color="goldenrod"] -I2 [label="" color="goldenrod"] -I3 [label="" color="goldenrod"] -I4 [label="" color="goldenrod"] -I5 [label="" color="goldenrod"] +/* The ipfs peers linked to cluster peers */ +subgraph { +rank="max" +I0 [label=< IPFS
Qm*ZDV > group="QmUBuxVHoNNjfmNpTad36UeaFQv3gXAtCv9r6KhmeqhEhD" color="1" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="box" ] +I1 [label=< IPFS
Qm*Ssq > group="QmV35LjbEGPfN7KfMAJp43VV2enwXqqQf5esx4vUcgHDQJ" color="1" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="box" ] +I2 [label=< IPFS
Qm*suL > group="QmZ2ckU7G35MYyJgMTwMUnicsGqSy3YUxGBX7qny6MQmJu" color="1" style="filled" colorscheme="brbg11" fontcolor="6" fontname="Ariel" shape="box" ] +} + +/* The ipfs swarm peers */ +I3 [label=< IPFS
Qm*ccb > group="QmQsdAdCHs4PRLi5tcoLfasYppryqQENxgAy4b2aS8xccb" color="5" style="filled" colorscheme="brbg11" fontcolor="1" fontname="Ariel" shape="box" ] +I4 [label=< IPFS
Qm*nM8 > group="QmVV2enwXqqQf5esx4v36UeaFQvFehSPzNfi8aaaaaanM8" color="5" style="filled" colorscheme="brbg11" fontcolor="1" fontname="Ariel" shape="box" ] +I5 [label=< IPFS
Qm*deD > group="QmfCHNQ2vbUmAuJZhE2hEpgiJq4sL1XScWEKnUrVtWZdeD" color="5" style="filled" colorscheme="brbg11" fontcolor="1" fontname="Ariel" shape="box" ] /* Edges representing active connections in the cluster */ /* The connections among cluster-service peers */ @@ -152,28 +165,27 @@ C1 -> C0 C1 -> C2 /* The connections between cluster peers and their ipfs daemons */ -C0 -> I3 -C1 -> I0 -C2 -> I4 +C0 -> I0 +C1 -> I1 +C2 -> I2 /* The swarm peer connections among ipfs daemons in the cluster */ +I1 -> I0 +I1 -> I2 +I1 -> I3 +I1 -> I4 +I1 -> I5 I0 -> I1 I0 -> I2 I0 -> I3 I0 -> I4 I0 -> I5 -I3 -> I0 -I3 -> I1 -I3 -> I2 -I3 -> I4 -I3 -> I5 -I4 -> I0 -I4 -> I1 -I4 -> I2 -I4 -> I3 -I4 -> I5 - - }` +I2 -> I0 +I2 -> I1 +I2 -> I3 +I2 -> I4 +I2 -> I5 +}` func TestIpfsAllGraphs(t *testing.T) { cg := api.ConnectGraph{ diff --git a/cmd/ipfs-cluster-ctl/main.go b/cmd/ipfs-cluster-ctl/main.go index 8d8be50e..f9e86a5f 100644 --- a/cmd/ipfs-cluster-ctl/main.go +++ b/cmd/ipfs-cluster-ctl/main.go @@ -359,6 +359,10 @@ content. Value: defaultAddParams.ReplicationFactorMax, Usage: "Sets the maximum replication factor for pinning this file", }, + cli.StringFlag{ + Name: "expire-in", + Usage: "Duration after which the pin should be unpinned automatically", + }, cli.StringSliceFlag{ Name: "metadata", Usage: "Pin metadata: key=value. Can be added multiple times", @@ -414,6 +418,12 @@ content. p := api.DefaultAddParams() p.ReplicationFactorMin = c.Int("replication-min") p.ReplicationFactorMax = c.Int("replication-max") + if expireIn := c.String("expire-in"); expireIn != "" { + d, err := time.ParseDuration(expireIn) + checkErr("parsing expire-in", err) + p.ExpireAt = time.Now().Add(d) + } + p.Metadata = parseMetadata(c.StringSlice("metadata")) p.Name = name if c.String("allocations") != "" { @@ -540,6 +550,10 @@ would stil be respected. Value: "", Usage: "Sets a name for this pin", }, + cli.StringFlag{ + Name: "expire-in", + Usage: "Duration after which pin should be unpinned automatically", + }, cli.StringSliceFlag{ Name: "metadata", Usage: "Pin metadata: key=value. Can be added multiple times", @@ -579,12 +593,19 @@ would stil be respected. checkErr("decoding allocations", errors.New("some peer IDs could not be decoded")) } } + var expireAt time.Time + if expireIn := c.String("expire-in"); expireIn != "" { + d, err := time.ParseDuration(expireIn) + checkErr("parsing expire-in", err) + expireAt = time.Now().Add(d) + } opts := api.PinOptions{ ReplicationFactorMin: rplMin, ReplicationFactorMax: rplMax, Name: c.String("name"), UserAllocations: userAllocs, + ExpireAt: expireAt, Metadata: parseMetadata(c.StringSlice("metadata")), } @@ -932,6 +953,8 @@ graph of the connections. Output is a dot file encoding the cluster's connectio This commands displays the latest valid metrics of the given type logged by this peer for all current cluster peers. +If no argument is provided, the command retrieves all currently existing metric types. + Currently supported metrics depend on the informer component used, but usually are: @@ -942,7 +965,9 @@ but usually are: Action: func(c *cli.Context) error { metric := c.Args().First() if metric == "" { - checkErr("", errors.New("provide a metric name")) + resp, cerr := globalClient.MetricNames(ctx) + formatResponse(c, resp, cerr) + return nil } resp, cerr := globalClient.Metrics(ctx, metric) diff --git a/cmd/ipfs-cluster-service/Makefile b/cmd/ipfs-cluster-service/Makefile index 046f1194..a522020f 100644 --- a/cmd/ipfs-cluster-service/Makefile +++ b/cmd/ipfs-cluster-service/Makefile @@ -4,7 +4,7 @@ SRC := $(shell find .. -type f -name '*.go') all: ipfs-cluster-service ipfs-cluster-service: $(SRC) - go build -ldflags "-X main.commit=$(shell git rev-parse HEAD)" + go build -mod=readonly -ldflags "-X main.commit=$(shell git rev-parse HEAD)" build: ipfs-cluster-service diff --git a/cmd/ipfs-cluster-service/main.go b/cmd/ipfs-cluster-service/main.go index 16996124..174b258b 100644 --- a/cmd/ipfs-cluster-service/main.go +++ b/cmd/ipfs-cluster-service/main.go @@ -272,6 +272,10 @@ the peer IDs in the given multiaddresses. Name: "force, f", Usage: "overwrite configuration without prompting", }, + cli.BoolFlag{ + Name: "randomports", + Usage: "configure random ports to listen on instead of defaults", + }, }, Action: func(c *cli.Context) error { consensus := c.String("consensus") @@ -319,6 +323,18 @@ the peer IDs in the given multiaddresses. // Generate defaults for all registered components err := cfgHelper.Manager().Default() checkErr("generating default configuration", err) + if c.Bool("randomports") { + cfgs := cfgHelper.Configs() + + for i := range cfgs.Cluster.ListenAddr { + cfgs.Cluster.ListenAddr[i], err = cmdutils.RandomizePorts(cfgs.Cluster.ListenAddr[i]) + } + checkErr("randomizing ports", err) + cfgs.Restapi.HTTPListenAddr, err = cmdutils.RandomizePorts(cfgs.Restapi.HTTPListenAddr) + checkErr("randomizing ports", err) + cfgs.Ipfsproxy.ListenAddr, err = cmdutils.RandomizePorts(cfgs.Ipfsproxy.ListenAddr) + checkErr("randomizing ports", err) + } err = cfgHelper.Manager().ApplyEnvVars() checkErr("applying environment variables to configuration", err) diff --git a/cmd/ipfs-cluster-service/main_test.go b/cmd/ipfs-cluster-service/main_test.go new file mode 100644 index 00000000..4dbad8ee --- /dev/null +++ b/cmd/ipfs-cluster-service/main_test.go @@ -0,0 +1,33 @@ +package main + +import ( + "testing" + + "github.com/ipfs/ipfs-cluster/cmdutils" + + ma "github.com/multiformats/go-multiaddr" +) + +func TestRandomPorts(t *testing.T) { + m1, _ := ma.NewMultiaddr("/ip4/0.0.0.0/tcp/9096") + m2, _ := ma.NewMultiaddr("/ip4/0.0.0.0/tcp/9096") + + m1, err := cmdutils.RandomizePorts(m1) + if err != nil { + t.Fatal(err) + } + + v1, err := m1.ValueForProtocol(ma.P_TCP) + if err != nil { + t.Fatal(err) + } + + v2, err := m2.ValueForProtocol(ma.P_TCP) + if err != nil { + t.Fatal(err) + } + + if v1 == v2 { + t.Error("expected different ports") + } +} diff --git a/cmdutils/cmdutils.go b/cmdutils/cmdutils.go index c75ed81f..11752851 100644 --- a/cmdutils/cmdutils.go +++ b/cmdutils/cmdutils.go @@ -1,3 +1,58 @@ // Package cmdutils contains utilities to facilitate building of command line // applications launching cluster peers. package cmdutils + +import ( + "fmt" + "net" + + ma "github.com/multiformats/go-multiaddr" +) + +// RandomizePorts replaces TCP and UDP ports with random, but valid port +// values. +func RandomizePorts(m ma.Multiaddr) (ma.Multiaddr, error) { + var prev string + + var err error + components := []ma.Multiaddr{} + ma.ForEach(m, func(c ma.Component) bool { + code := c.Protocol().Code + + if code != ma.P_TCP && code != ma.P_UDP { + components = append(components, &c) + prev = c.Value() + return true + } + + var ln net.Listener + ln, err = net.Listen(c.Protocol().Name, prev+":") + if err != nil { + return false + } + defer ln.Close() + + var c1 *ma.Component + c1, err = ma.NewComponent(c.Protocol().Name, fmt.Sprintf("%d", getPort(ln, code))) + if err != nil { + return false + } + + components = append(components, c1) + prev = c.Value() + + return true + }) + + return ma.Join(components...), err +} + +func getPort(ln net.Listener, code int) int { + if code == ma.P_TCP { + return ln.Addr().(*net.TCPAddr).Port + } + if code == ma.P_UDP { + return ln.Addr().(*net.UDPAddr).Port + } + return 0 +} diff --git a/config_test.go b/config_test.go index 85357722..2185bf92 100644 --- a/config_test.go +++ b/config_test.go @@ -64,7 +64,10 @@ var testingCrdtCfg = []byte(`{ }`) var testingBadgerCfg = []byte(`{ - "folder": "badgerFromTests" + "folder": "badgerFromTests", + "badger_options": { + "max_table_size": 1048576 + } }`) var testingAPICfg = []byte(`{ diff --git a/connect_graph.go b/connect_graph.go index 2f0cf4a7..57029a35 100644 --- a/connect_graph.go +++ b/connect_graph.go @@ -10,22 +10,29 @@ import ( ) // ConnectGraph returns a description of which cluster peers and ipfs -// daemons are connected to each other +// daemons are connected to each other. func (c *Cluster) ConnectGraph() (api.ConnectGraph, error) { ctx, span := trace.StartSpan(c.ctx, "cluster/ConnectGraph") defer span.End() cg := api.ConnectGraph{ - ClusterID: c.host.ID(), - IPFSLinks: make(map[string][]peer.ID), - ClusterLinks: make(map[string][]peer.ID), - ClustertoIPFS: make(map[string]peer.ID), + ClusterID: c.host.ID(), + IDtoPeername: make(map[string]string), + IPFSLinks: make(map[string][]peer.ID), + ClusterLinks: make(map[string][]peer.ID), + ClusterTrustLinks: make(map[string]bool), + ClustertoIPFS: make(map[string]peer.ID), } members, err := c.consensus.Peers(ctx) if err != nil { return cg, err } + for _, member := range members { + // one of the entries is for itself, but that shouldn't hurt + cg.ClusterTrustLinks[peer.IDB58Encode(member)] = c.consensus.IsTrustedPeer(ctx, member) + } + peers := make([][]*api.ID, len(members), len(members)) ctxs, cancels := rpcutil.CtxsWithCancel(ctx, len(members)) @@ -49,7 +56,7 @@ func (c *Cluster) ConnectGraph() (api.ConnectGraph, error) { } selfConnection, pID := c.recordClusterLinks(&cg, p, peers[i]) - + cg.IDtoPeername[p] = pID.Peername // IPFS connections if !selfConnection { logger.Warningf("cluster peer %s not its own peer. No ipfs info ", p) diff --git a/consensus/crdt/consensus.go b/consensus/crdt/consensus.go index 6f66a81c..f12b0a91 100644 --- a/consensus/crdt/consensus.go +++ b/consensus/crdt/consensus.go @@ -266,7 +266,8 @@ func (css *Consensus) setup() { css.readyCh <- struct{}{} } -// Shutdown closes this component, cancelling the pubsub subscription. +// Shutdown closes this component, cancelling the pubsub subscription and +// closing the datastore. func (css *Consensus) Shutdown(ctx context.Context) error { css.shutdownLock.Lock() defer css.shutdownLock.Unlock() diff --git a/consensus/crdt/consensus_test.go b/consensus/crdt/consensus_test.go index 23c092cb..5b0422c6 100644 --- a/consensus/crdt/consensus_test.go +++ b/consensus/crdt/consensus_test.go @@ -293,7 +293,7 @@ func TestConsensusDistrustPeer(t *testing.T) { t.Fatal(err) } if len(pins) != 1 || !pins[0].Cid.Equals(test.Cid1) { - t.Error("the added pin should be in the state") + t.Error("only first pin should be in the state") } } diff --git a/consensus/raft/logging.go b/consensus/raft/logging.go index 3273d21d..15c7e98a 100644 --- a/consensus/raft/logging.go +++ b/consensus/raft/logging.go @@ -92,6 +92,10 @@ func (log *hcLogToLogger) IsError() bool { return true } +func (log *hcLogToLogger) Name() string { + return log.name +} + func (log *hcLogToLogger) With(args ...interface{}) hclog.Logger { return &hcLogToLogger{extraArgs: args} } @@ -114,6 +118,10 @@ func (log *hcLogToLogger) StandardWriter(opts *hclog.StandardLoggerOptions) io.W return nil } +func (log *hcLogToLogger) ImpliedArgs() []interface{} { + return nil +} + const repeatPoolSize = 10 const repeatReset = time.Minute diff --git a/datastore/badger/config.go b/datastore/badger/config.go index faf47853..125f4838 100644 --- a/datastore/badger/config.go +++ b/datastore/badger/config.go @@ -169,7 +169,6 @@ func (cfg *Config) LoadJSON(raw []byte) error { if err != nil { return err } - cfg.Default() return cfg.applyJSONConfig(jcfg) diff --git a/go.mod b/go.mod index 23892720..eafc3bb6 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,16 @@ require ( contrib.go.opencensus.io/exporter/jaeger v0.1.0 contrib.go.opencensus.io/exporter/prometheus v0.1.0 github.com/blang/semver v3.5.1+incompatible + github.com/btcsuite/btcd v0.20.0-beta // indirect github.com/dgraph-io/badger v1.6.0 github.com/dustin/go-humanize v1.0.0 - github.com/gogo/protobuf v1.3.0 + github.com/gogo/protobuf v1.3.1 github.com/golang/protobuf v1.3.2 github.com/google/uuid v1.1.1 github.com/gorilla/handlers v1.4.2 github.com/gorilla/mux v1.7.3 - github.com/hashicorp/go-hclog v0.9.2 + github.com/hashicorp/go-hclog v0.10.0 + github.com/hashicorp/go-immutable-radix v1.1.0 // indirect github.com/hashicorp/raft v1.1.1 github.com/hashicorp/raft-boltdb v0.0.0-20190605210249-ef2e128ed477 github.com/hsanjuan/ipfs-lite v0.1.6 @@ -24,6 +26,7 @@ require ( github.com/ipfs/go-fs-lock v0.0.1 github.com/ipfs/go-ipfs-api v0.0.2 github.com/ipfs/go-ipfs-chunker v0.0.1 + github.com/ipfs/go-ipfs-config v0.0.11 github.com/ipfs/go-ipfs-ds-help v0.0.1 github.com/ipfs/go-ipfs-files v0.0.6 github.com/ipfs/go-ipfs-posinfo v0.0.1 @@ -36,30 +39,33 @@ require ( github.com/ipfs/go-path v0.0.7 github.com/ipfs/go-unixfs v0.2.2 github.com/kelseyhightower/envconfig v1.4.0 + github.com/kishansagathiya/go-dot v0.1.0 github.com/lanzafame/go-libp2p-ocgorpc v0.1.1 github.com/libp2p/go-libp2p v0.4.0 + github.com/libp2p/go-libp2p-autonat-svc v0.1.0 + github.com/libp2p/go-libp2p-circuit v0.1.3 github.com/libp2p/go-libp2p-connmgr v0.1.1 github.com/libp2p/go-libp2p-consensus v0.0.1 - github.com/libp2p/go-libp2p-core v0.2.3 + github.com/libp2p/go-libp2p-core v0.2.4 github.com/libp2p/go-libp2p-crypto v0.1.0 github.com/libp2p/go-libp2p-gorpc v0.1.0 github.com/libp2p/go-libp2p-gostream v0.2.0 github.com/libp2p/go-libp2p-host v0.1.0 github.com/libp2p/go-libp2p-http v0.1.4 - github.com/libp2p/go-libp2p-interface-pnet v0.1.0 github.com/libp2p/go-libp2p-kad-dht v0.2.1 github.com/libp2p/go-libp2p-peer v0.2.0 github.com/libp2p/go-libp2p-peerstore v0.1.3 github.com/libp2p/go-libp2p-pnet v0.1.0 github.com/libp2p/go-libp2p-protocol v0.1.0 github.com/libp2p/go-libp2p-pubsub v0.1.1 - github.com/libp2p/go-libp2p-raft v0.1.3 + github.com/libp2p/go-libp2p-quic-transport v0.2.0 + github.com/libp2p/go-libp2p-raft v0.1.4 + github.com/libp2p/go-libp2p-secio v0.2.0 + github.com/libp2p/go-libp2p-tls v0.1.2 github.com/libp2p/go-ws-transport v0.1.2 - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect github.com/multiformats/go-multiaddr v0.1.1 github.com/multiformats/go-multiaddr-dns v0.1.1 - github.com/multiformats/go-multiaddr-net v0.1.0 + github.com/multiformats/go-multiaddr-net v0.1.1 github.com/multiformats/go-multicodec v0.1.6 github.com/multiformats/go-multihash v0.0.8 github.com/pkg/errors v0.8.1 @@ -67,8 +73,9 @@ require ( github.com/rs/cors v1.7.0 github.com/ugorji/go/codec v1.1.7 github.com/urfave/cli v1.22.1 - github.com/zenground0/go-dot v0.0.0-20180912213407-94a425d4984e + github.com/whyrusleeping/go-logging v0.0.1 // indirect go.opencensus.io v0.22.1 + golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 gonum.org/v1/gonum v0.0.0-20190926113837-94b2bbd8ac13 gonum.org/v1/plot v0.0.0-20190615073203-9aa86143727f ) diff --git a/go.sum b/go.sum index 77ab686a..f66cbb5d 100644 --- a/go.sum +++ b/go.sum @@ -1,19 +1,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= contrib.go.opencensus.io/exporter/jaeger v0.1.0 h1:WNc9HbA38xEQmsI40Tjd/MNU/g8byN2Of7lwIjv0Jdc= contrib.go.opencensus.io/exporter/jaeger v0.1.0/go.mod h1:VYianECmuFPwU37O699Vc1GOcy+y8kOsfaxHRImmjbA= contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg= contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 h1:PqzgE6kAMi81xWQA2QIVxjWkFHptGgC547vchpUbtFo= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= -github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMGXfDk= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= @@ -24,10 +19,12 @@ github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af h1:wVe6/Ea46ZMeNkQjjBW6xcqyQA/j5e0D6GytH95g0gQ= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75 h1:3ILjVyslFbc4jl1w5TWuvvslFD/nDfR2H8tVaMVLrEY= +github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75/go.mod h1:uAXEEpARkRhCZfEvy/y0Jcc888f9tHCc1W7/UeEtreE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -51,8 +48,8 @@ github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcug github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 h1:A/EVblehb75cUgXA5njHPn0kLAsykn6mJGz7rnmW5W0= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= -github.com/btcsuite/btcd v0.0.0-20190926002857-ba530c4abb35 h1:o2mPiVrkVzpBg/Q+lSfuf/92pEgsSIJvsQ13DyHs/3A= -github.com/btcsuite/btcd v0.0.0-20190926002857-ba530c4abb35/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= +github.com/btcsuite/btcd v0.20.0-beta h1:DnZGUjFbRkpytojHWwy6nfUSA7vFrzWXDLpFNzt74ZA= +github.com/btcsuite/btcd v0.20.0-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= @@ -63,6 +60,8 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= +github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= @@ -88,8 +87,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC5IlbaIF5Q7JNieBoACT7iW0YTxQHR0in0= github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= -github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f h1:BOaYiTvg8p9vBUXpklC22XSK/mifLF7lG9jtmYYi3Tc= -github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo= @@ -104,17 +101,15 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90 h1:WXb3TSNmHp2vHoCroCIB1foO/yQ36swABL8aOVeDpgg= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-check/check v0.0.0-20190902080502-41f04d3bba15 h1:xJdCV5uP69sUzCIIzmhAw6EKKdVk3Tu48oLzM86+XPI= -github.com/go-check/check v0.0.0-20190902080502-41f04d3bba15/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -124,6 +119,8 @@ github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -131,6 +128,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk= @@ -140,26 +138,17 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000= -github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg= github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= @@ -177,10 +166,12 @@ github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmv github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.1 h1:9PZfAcVEvez4yhLH2TBU64/h/z4xlFI80cWXRrxuKuM= github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.10.0 h1:b86HUuA126IcSHyC55WjPo7KtCOVeTCKIjr+3lBhPxI= +github.com/hashicorp/go-hclog v0.10.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTMEfgEdSj/hr8MPc= +github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= @@ -325,8 +316,6 @@ github.com/ipfs/go-peertaskqueue v0.1.1 h1:+gPjbI+V3NktXZOqJA1kzbms2pYmhjgQQal0M github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-todocounter v0.0.1 h1:kITWA5ZcQZfrUnDNkRn04Xzh0YFaDFXsoO2A81Eb6Lw= github.com/ipfs/go-todocounter v0.0.1/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4= -github.com/ipfs/go-todocounter v0.0.2 h1:9UBngSQhylg2UDcxSAtpkT+rEWFr26hDPXVStE8LFyc= -github.com/ipfs/go-todocounter v0.0.2/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4= github.com/ipfs/go-unixfs v0.1.0 h1:KkjcfqObdNwUN8heMtt5OdrgrRKYTIWEvpGl1bDYIho= github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= github.com/ipfs/go-unixfs v0.2.2 h1:eTkDT9F0dn4qHmBMVRMZbziwyqLRcogjtPYqMgZYmQs= @@ -352,7 +341,6 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -363,6 +351,8 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kishansagathiya/go-dot v0.1.0 h1:XPj/333a6Qn4VPFqF+e2EiyABL7yRObJ7RTAbriKA1s= +github.com/kishansagathiya/go-dot v0.1.0/go.mod h1:U1dCUFzZ+KnBgkaCWPj2JFUQygVepVudkINK9QRsxMs= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= @@ -409,6 +399,8 @@ github.com/libp2p/go-libp2p-autonat v0.0.6 h1:OCStANLLpeyQeWFUuqZJ7aS9+Bx0/uoVb1 github.com/libp2p/go-libp2p-autonat v0.0.6/go.mod h1:uZneLdOkZHro35xIhpbtTzLlgYturpu4J5+0cZK3MqE= github.com/libp2p/go-libp2p-autonat v0.1.0 h1:aCWAu43Ri4nU0ZPO7NyLzUvvfqd0nE3dX0R/ZGYVgOU= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= +github.com/libp2p/go-libp2p-autonat-svc v0.1.0 h1:28IM7iWMDclZeVkpiFQaWVANwXwE7zLlpbnS7yXxrfs= +github.com/libp2p/go-libp2p-autonat-svc v0.1.0/go.mod h1:fqi8Obl/z3R4PFVLm8xFtZ6PBL9MlV/xumymRFkKq5A= github.com/libp2p/go-libp2p-blankhost v0.0.1 h1:/mZuuiwntNR8RywnCFlGHLKrKLYne+qciBpQXWqp5fk= github.com/libp2p/go-libp2p-blankhost v0.0.1/go.mod h1:Ibpbw/7cPPYwFb7PACIWdvxxv0t0XCCI10t7czjAjTc= github.com/libp2p/go-libp2p-blankhost v0.1.1 h1:X919sCh+KLqJcNRApj43xCSiQRYqOSI88Fdf55ngf78= @@ -441,6 +433,8 @@ github.com/libp2p/go-libp2p-core v0.2.2 h1:Sv1ggdoMx9c7v7FOFkR7agraHCnAgqYsXrU1A github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0= github.com/libp2p/go-libp2p-core v0.2.3 h1:zXikZ5pLfebtTMeIYfcwVQ2Pae77O0FIwDquwM6AGNM= github.com/libp2p/go-libp2p-core v0.2.3/go.mod h1:GqhyQqyIAPsxFYXHMjfXgMv03lxsvM0mFzuYA9Ib42A= +github.com/libp2p/go-libp2p-core v0.2.4 h1:Et6ykkTwI6PU44tr8qUF9k43vP0aduMNniShAbUJJw8= +github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= github.com/libp2p/go-libp2p-crypto v0.0.1 h1:JNQd8CmoGTohO/akqrH16ewsqZpci2CbgYH/LmYl8gw= github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= github.com/libp2p/go-libp2p-crypto v0.0.2 h1:TTdJ4y6Uoa6NxQcuEaVkQfFRcQeCE2ReDk8Ok4I0Fyw= @@ -471,8 +465,6 @@ github.com/libp2p/go-libp2p-interface-connmgr v0.0.5 h1:KG/KNYL2tYzXAfMvQN5K1aAG github.com/libp2p/go-libp2p-interface-connmgr v0.0.5/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= github.com/libp2p/go-libp2p-interface-pnet v0.0.1 h1:7GnzRrBTJHEsofi1ahFdPN9Si6skwXQE9UqR2S+Pkh8= github.com/libp2p/go-libp2p-interface-pnet v0.0.1/go.mod h1:el9jHpQAXK5dnTpKA4yfCNBZXvrzdOU75zz+C6ryp3k= -github.com/libp2p/go-libp2p-interface-pnet v0.1.0 h1:PaofJtuDcrGBukgTymiGyuI313nxARRQFmE/oxZXlog= -github.com/libp2p/go-libp2p-interface-pnet v0.1.0/go.mod h1:8+FQ08+xMxR6BjG0tUZoQzKxPAV2W7ck6IxjCWqZ6ek= github.com/libp2p/go-libp2p-kad-dht v0.2.1 h1:+pb1DCkV/6oNQjTZVXl+Y++eV0rnelx/L8y1t4J+Rnw= github.com/libp2p/go-libp2p-kad-dht v0.2.1/go.mod h1:k7ONOlup7HKzQ68dE6lSnp07cdxdkmnRa+6B4Fh9/w0= github.com/libp2p/go-libp2p-kbucket v0.2.1 h1:q9Jfwww9XnXc1K9dyYuARJxJvIvhgYVaQCuziO/dF3c= @@ -520,8 +512,10 @@ github.com/libp2p/go-libp2p-protocol v0.1.0 h1:HdqhEyhg0ToCaxgMhnOmUO8snQtt/kQlc github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.1.1 h1:phDnQvO3H3hAgaEEQi6yt3LILqIYVXaw05bxzezrEwQ= github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= -github.com/libp2p/go-libp2p-raft v0.1.3 h1:XcrMI3XZqZSHQWnsG8eqM1pQGoiztAj1VG1om6os1Vs= -github.com/libp2p/go-libp2p-raft v0.1.3/go.mod h1:JzLdxSiY4UR7WcVSFWcfPyDKf+9wlXFyNS5Z59eHoBA= +github.com/libp2p/go-libp2p-quic-transport v0.2.0 h1:KCAK6OKM1LG7u4Bot2mDIfFSs5QWFxCLDsIh2zQff6o= +github.com/libp2p/go-libp2p-quic-transport v0.2.0/go.mod h1:yADxsG+nm+mTTp0uqmANBmih6TEqH6SGsmbe/2K2sLY= +github.com/libp2p/go-libp2p-raft v0.1.4 h1:mE/RH6Q/QjwXXl1eWkbSpF6EjuD4pt2E9f94rkMalUE= +github.com/libp2p/go-libp2p-raft v0.1.4/go.mod h1:+JGEXVP5ziDLtdDDRqvFjWN3Vsa6ahdLZNvFmTiN9gc= github.com/libp2p/go-libp2p-record v0.0.1 h1:zN7AS3X46qmwsw5JLxdDuI43cH5UYwovKxHPjKBYQxw= github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q= github.com/libp2p/go-libp2p-record v0.1.0 h1:wHwBGbFzymoIl69BpgwIu0O6ta3TXGcMPvHUAcodzRc= @@ -554,6 +548,10 @@ github.com/libp2p/go-libp2p-testing v0.0.4 h1:Qev57UR47GcLPXWjrunv5aLIQGO4n9mhI/ github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.1.0 h1:WaFRj/t3HdMZGNZqnU2pS7pDRBmMeoDx7/HDNpeyT9U= github.com/libp2p/go-libp2p-testing v0.1.0/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= +github.com/libp2p/go-libp2p-tls v0.1.1 h1:tjW7njTM8JX8FbEvqr8/VSKBdZYZ7CtGtv3i6NiFf10= +github.com/libp2p/go-libp2p-tls v0.1.1/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= +github.com/libp2p/go-libp2p-tls v0.1.2 h1:H96DWm11fO3+LF6fgLUfK/AtD8350RuMluFeeYXS01k= +github.com/libp2p/go-libp2p-tls v0.1.2/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= github.com/libp2p/go-libp2p-transport v0.0.1/go.mod h1:UzbUs9X+PHOSw7S3ZmeOxfnwaQY5vGDzZmKPod3N3tk= github.com/libp2p/go-libp2p-transport v0.0.5 h1:pV6+UlRxyDpASSGD+60vMvdifSCby6JkJDfi+yUMHac= github.com/libp2p/go-libp2p-transport v0.0.5/go.mod h1:StoY3sx6IqsP6XKoabsPnHCwqKXWUMWU7Rfcsubee/A= @@ -588,6 +586,8 @@ github.com/libp2p/go-nat v0.0.3 h1:l6fKV+p0Xa354EqQOQP+d8CivdLM4kl5GxC1hSc/UeI= github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI= github.com/libp2p/go-openssl v0.0.2 h1:9pP2d3Ubaxkv7ZisLjx9BFwgOGnQdQYnfcH29HNY3ls= github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= +github.com/libp2p/go-openssl v0.0.3 h1:wjlG7HvQkt4Fq4cfH33Ivpwp0omaElYEi9z26qaIkIk= +github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FWpFLw= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4= @@ -619,27 +619,35 @@ github.com/libp2p/go-yamux v1.2.1/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZ github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.2.3 h1:xX8A36vpXb59frIzWFdEgptLMsOANMFq2K7fPRlunYI= github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= +github.com/lucas-clemente/quic-go v0.13.0 h1:oY83lV1tmpYcju2xzcB3tHVHX8j+pkCp4uv2VcxNV+U= +github.com/lucas-clemente/quic-go v0.13.0/go.mod h1:Vn3/Fb0/77b02SGhQk36KzOUmXgVpFfizUfW5WMaqyU= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/marten-seemann/chacha20 v0.2.0 h1:f40vqzzx+3GdOmzQoItkLX5WLvHgPgyYqFFIO5Gh4hQ= +github.com/marten-seemann/chacha20 v0.2.0/go.mod h1:HSdjFau7GzYRj+ahFNwsO3ouVJr1HFkWoEwNDb4TMtE= +github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI= +github.com/marten-seemann/qtls v0.4.1 h1:YlT8QP3WCCvvok7MGEZkMldXbyqgr8oFg5/n8Gtbkks= +github.com/marten-seemann/qtls v0.4.1/go.mod h1:pxVXcHHw1pNIt8Qo0pwSYQEoZ8yYOOPXTCZLQQunvRc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.12 h1:WMhc1ik4LNkTg8U9l3hI1LvxKmIL+f1+WV/SZtCbDDA= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.18 h1:RP3QrPVgq8fLiw0mjYwVGNaIsCc9MxNFlRHu8PlAeiY= -github.com/miekg/dns v1.1.18/go.mod h1:WgzbA6oji13JREwiNsRDNfl7jYdPnmz+VEuLrA+/48M= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16 h1:5W7KhL8HVF3XCFOweFD3BNESdnO8ewyYTFT2R+/b8FQ= @@ -690,6 +698,8 @@ github.com/multiformats/go-multiaddr-net v0.0.1 h1:76O59E3FavvHqNg7jvzWzsPSW5JSi github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU= github.com/multiformats/go-multiaddr-net v0.1.0 h1:ZepO8Ezwovd+7b5XPPDhQhayk1yt0AJpzQBpq9fejx4= github.com/multiformats/go-multiaddr-net v0.1.0/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= +github.com/multiformats/go-multiaddr-net v0.1.1 h1:jFFKUuXTXv+3ARyHZi3XUqQO+YWMKgBdhEvuGRfnL6s= +github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multicodec v0.1.6 h1:4u6lcjbE4VVVoigU4QJSSVYsGVP4j2jtDkR8lPwOrLE= @@ -712,14 +722,10 @@ github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= @@ -738,8 +744,6 @@ github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 h1:bzMe+2coZJYHnhGg github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14 h1:2m16U/rLwVaRdz7ANkHtHTodP3zTP3N451MADg64x5k= github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= -github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 h1:CskT+S6Ay54OwxBGB0R3Rsx4Muto6UnEYTyKJbyRIAI= -github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 h1:D+CiwcpGTW6pL6bv6KI3KbyEyCKyS+1JWS2h8PNDnGA= @@ -753,8 +757,6 @@ github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f h1:BVwpUVJ github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -762,8 +764,6 @@ github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg= @@ -772,8 +772,6 @@ github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNG github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= @@ -783,20 +781,16 @@ github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= -github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= -github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa h1:E+gaaifzi2xF65PbDmuKI3PhLWY6G5opMLniFq8vmXA= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= -github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a h1:/eS3yfGjQKG+9kayBkj0ip1BGhq6zJ3eaVksphxAaek= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= @@ -810,14 +804,13 @@ github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tL github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/src-d/envconfig v1.0.0 h1:/AJi6DtjFhZKNx3OB2qMsq7y4yT5//AeSZIe7rk+PX8= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= @@ -829,6 +822,10 @@ github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vishvananda/netlink v1.0.0 h1:bqNY2lgheFIu1meHUFSH3d7vG93AFyqg3oGbJCOJgSM= +github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f h1:nBX3nTcmxEtHSERBJaIo1Qa26VwRaopnZmfDQUXsF4I= +github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436 h1:qOpVTI+BrstcjTZLm2Yz/3sOnqkzj3FQoh0g+E5s3Gc= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0= @@ -841,6 +838,8 @@ github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdz github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= +github.com/whyrusleeping/go-logging v0.0.1 h1:fwpzlmT0kRC/Fmd0MdmGgJG/CXIZ6gFq46FQZjprUcc= +github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f h1:M/lL30eFZTKnomXY6huvM6G0+gVquFNf6mxghaWlFUg= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8= github.com/whyrusleeping/mafmt v1.2.8 h1:TCghSl5kkwEE0j+sU/gudyhVMRlpBin8fMBBHg59EbA= @@ -855,10 +854,9 @@ github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c h1:GGsyl0d github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/zenground0/go-dot v0.0.0-20180912213407-94a425d4984e h1:GN1PUQ/MNDdtiZZhCAnZ4PwTcslUM8qWVz8q2bLkDeM= -github.com/zenground0/go-dot v0.0.0-20180912213407-94a425d4984e/go.mod h1:T00FaxHq4SlnicuZFole4yRAgcjWtqbMcUXBfXAYvaI= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -879,7 +877,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90Pveol golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -889,29 +886,19 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392 h1:ACG4HJsFiNMf47Y4PeRoebLNy/2lXT9EtprMuTFWt1M= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad h1:5E5raQxcv+6CZ11RrBYQe5WRbUIWpScjh0kvHZkZIrQ= -golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2 h1:y102fOLFqhV41b+4GPiJoa0k/x+pJcEi2/HB1Y5T6fU= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190925190815-26a69ce95baf h1:V3Pwu2Rbgu9M+TwZjWt4811gVksfbQRU7/rX1dUq8rI= -golang.org/x/exp v0.0.0-20190925190815-26a69ce95baf/go.mod h1:/XYaSSBNneWTe8VNa9AjSawUDfGmbpmIqqzQT33BSc0= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81 h1:00VmoueYNlNz/aHIilyyQz/MHSqGoWJzpFv/HW8xpzI= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a h1:gHevYm0pO4QUbwy8Dmdr01R5r1BuKtfYqRqF0h/Cbh0= -golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -923,11 +910,11 @@ golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7 h1:C2F/nMkR/9sfUTpvR3QrjBuTdvMUC/cFajkphs1YLQo= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -937,12 +924,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190926025831-c00fd9afed17 h1:qPnAdmjNA41t3QBTx2mFGf/SD1IoslhYu7AmdsVzCcs= -golang.org/x/net v0.0.0-20190926025831-c00fd9afed17/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= @@ -951,8 +934,6 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -965,33 +946,28 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190302025703-b6889370fb10 h1:xQJI9OEiErEQ++DoXOHqEpzsGMrAv2Q2jyCpi7DmfpQ= golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190524122548-abf6ff778158 h1:v73Zw0Y1htnV0qaOAYSNiuIAviPSBkNtdy1tPi1+zpY= golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190524152521-dbbf3f1254d4 h1:VSJ45BzqrVgR4clSx415y1rHH7QAGhGt71J0ZmhLYrc= golang.org/x/sys v0.0.0-20190524152521-dbbf3f1254d4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190927073244-c990c680b611 h1:q9u40nxWT5zRClI/uU9dHCiYGottAg6Nzz4YUQyHxdA= -golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1005,10 +981,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190925164712-ae58c0ff6b32/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= @@ -1023,22 +996,14 @@ google.golang.org/api v0.3.1 h1:oJra/lMfmtm13/rgY/8i3MzjFWYXvQIAKjQ3HqofMk8= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw= google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.10.0 h1:7tmAxx3oKE98VMZ+SBZzvYYWRQ9HODBxmC8mXUsraSQ= -google.golang.org/api v0.10.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19 h1:Lj2SnHtxkRGJDqnGaSjo+CCdIieEnwVazbOXILwQemk= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb h1:i1Ppqkc3WQXikh8bXiwHqAN5Rv3/qDCcRk0/Otx73BY= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190926190326-7ee9db18f195 h1:dWzgMaXfaHsnkRKZ1l3iJLDmTEB40JMl/dqRbJX4D/o= -google.golang.org/genproto v0.0.0-20190926190326-7ee9db18f195/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8= @@ -1046,17 +1011,15 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.22.0 h1:J0UbZOIrCAl+fpTOf8YLs4dJo8L/owV4LYVtAXQoPkw= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= +gopkg.in/src-d/go-log.v1 v1.0.1 h1:heWvX7J6qbGWbeFS/aRmiy1eYaT+QMV6wNvHDyMjQV4= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -1065,8 +1028,6 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/ipfscluster.go b/ipfscluster.go index 782204bc..fa01b3fc 100644 --- a/ipfscluster.go +++ b/ipfscluster.go @@ -178,6 +178,8 @@ type PeerMonitor interface { // LatestMetrics returns a map with the latest metrics of matching name // for the current cluster peers. LatestMetrics(ctx context.Context, name string) []*api.Metric + // MetricNames returns a list of metric names. + MetricNames(ctx context.Context) []string // Alerts delivers alerts generated when this peer monitor detects // a problem (i.e. metrics not arriving as expected). Alerts can be used // to trigger self-healing measures or re-pinnings of content. diff --git a/ipfscluster_test.go b/ipfscluster_test.go index e21b15b1..942fed05 100644 --- a/ipfscluster_test.go +++ b/ipfscluster_test.go @@ -127,12 +127,6 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func checkErr(t *testing.T, err error) { - if err != nil { - t.Fatal(err) - } -} - func randomBytes() []byte { bs := make([]byte, 64, 64) for i := 0; i < len(bs); i++ { @@ -195,17 +189,27 @@ func createComponents( badgerCfg.Folder = filepath.Join(testsFolder, host.ID().Pretty(), "badger") api, err := rest.NewAPI(ctx, apiCfg) - checkErr(t, err) + if err != nil { + t.Fatal(err) + } + ipfsProxy, err := rest.NewAPI(ctx, apiCfg) - checkErr(t, err) + if err != nil { + t.Fatal(err) + } ipfs, err := ipfshttp.NewConnector(ipfshttpCfg) - checkErr(t, err) + if err != nil { + t.Fatal(err) + } + tracker := makePinTracker(t, ident.ID, maptrackerCfg, statelesstrackerCfg, clusterCfg.Peername) alloc := descendalloc.NewAllocator() inf, err := disk.NewInformer(diskInfCfg) - checkErr(t, err) + if err != nil { + t.Fatal(err) + } store := makeStore(t, badgerCfg) cons := makeConsensus(t, store, host, pubsub, dht, raftCfg, staging, crdtCfg) @@ -215,11 +219,14 @@ func createComponents( peersF = cons.Peers } mon, err := pubsubmon.New(ctx, psmonCfg, pubsub, peersF) - checkErr(t, err) - + if err != nil { + t.Fatal(err) + } tracingCfg.ServiceName = peername tracer, err := observations.SetupTracing(tracingCfg) - checkErr(t, err) + if err != nil { + t.Fatal(err) + } return clusterCfg, store, cons, []API{api, ipfsProxy}, ipfs, tracker, mon, alloc, inf, tracer, mock } @@ -228,7 +235,9 @@ func makeStore(t *testing.T, badgerCfg *badger.Config) ds.Datastore { switch consensus { case "crdt": dstr, err := badger.New(badgerCfg) - checkErr(t, err) + if err != nil { + t.Fatal(err) + } return dstr default: return inmem.New() @@ -239,11 +248,15 @@ func makeConsensus(t *testing.T, store ds.Datastore, h host.Host, psub *pubsub.P switch consensus { case "raft": raftCon, err := raft.NewConsensus(h, raftCfg, store, staging) - checkErr(t, err) + if err != nil { + t.Fatal(err) + } return raftCon case "crdt": crdtCon, err := crdt.New(h, dht, psub, crdtCfg, store) - checkErr(t, err) + if err != nil { + t.Fatal(err) + } return crdtCon default: panic("bad consensus") @@ -265,7 +278,9 @@ func makePinTracker(t *testing.T, pid peer.ID, mptCfg *maptracker.Config, sptCfg func createCluster(t *testing.T, host host.Host, dht *dht.IpfsDHT, clusterCfg *Config, store ds.Datastore, consensus Consensus, apis []API, ipfs IPFSConnector, tracker PinTracker, mon PeerMonitor, alloc PinAllocator, inf Informer, tracer Tracer) *Cluster { cl, err := NewCluster(context.Background(), host, dht, clusterCfg, store, consensus, apis, ipfs, tracker, mon, alloc, inf, tracer) - checkErr(t, err) + if err != nil { + t.Fatal(err) + } return cl } @@ -282,12 +297,15 @@ func createHosts(t *testing.T, clusterSecret []byte, nClusters int) ([]host.Host pubsubs := make([]*pubsub.PubSub, nClusters, nClusters) dhts := make([]*dht.IpfsDHT, nClusters, nClusters) - listen, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0") + tcpaddr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0") + quicAddr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/udp/0/quic") for i := range hosts { priv, _, err := crypto.GenerateKeyPair(crypto.RSA, 2048) - checkErr(t, err) + if err != nil { + t.Fatal(err) + } - h, p, d := createHost(t, priv, clusterSecret, listen) + h, p, d := createHost(t, priv, clusterSecret, []ma.Multiaddr{quicAddr, tcpaddr}) hosts[i] = h dhts[i] = d pubsubs[i] = p @@ -296,23 +314,43 @@ func createHosts(t *testing.T, clusterSecret []byte, nClusters int) ([]host.Host return hosts, pubsubs, dhts } -func createHost(t *testing.T, priv crypto.PrivKey, clusterSecret []byte, listen ma.Multiaddr) (host.Host, *pubsub.PubSub, *dht.IpfsDHT) { +func createHost(t *testing.T, priv crypto.PrivKey, clusterSecret []byte, listen []ma.Multiaddr) (host.Host, *pubsub.PubSub, *dht.IpfsDHT) { ctx := context.Background() - h, err := newHost(ctx, clusterSecret, priv, libp2p.ListenAddrs(listen)) - checkErr(t, err) + prot, err := newProtector(clusterSecret) + if err != nil { + t.Fatal(err) + } + + h, err := newHost(ctx, prot, priv, libp2p.ListenAddrs(listen...)) + if err != nil { + t.Fatal(err) + } // DHT needs to be created BEFORE connecting the peers, but // bootstrapped AFTER - d, err := newDHT(ctx, h) - checkErr(t, err) + d, err := newTestDHT(ctx, h) + if err != nil { + t.Fatal(err) + } // Pubsub needs to be created BEFORE connecting the peers, // otherwise they are not picked up. psub, err := newPubSub(ctx, h) - checkErr(t, err) + if err != nil { + t.Fatal(err) + } return routedHost(h, d), psub, d } +func newTestDHT(ctx context.Context, h host.Host) (*dht.IpfsDHT, error) { + return newDHT(ctx, h) + // TODO: when new dht options are released + // return dht.New(ctx, h, dhtopts.Bootstrap(dhtopts.BootstrapConfig{ + // Timeout: 300 * time.Millisecond, + // SelfQueryInterval: 300 * time.Millisecond, + // })) +} + func createClusters(t *testing.T) ([]*Cluster, []*test.IpfsMock) { ctx := context.Background() os.RemoveAll(testsFolder) @@ -344,7 +382,6 @@ func createClusters(t *testing.T) ([]*Cluster, []*test.IpfsMock) { clusters[0] = createCluster(t, hosts[0], dhts[0], cfgs[0], stores[0], cons[0], apis[0], ipfss[0], trackers[0], mons[0], allocs[0], infs[0], tracers[0]) <-clusters[0].Ready() bootstrapAddr := clusterAddr(clusters[0]) - // Start the rest and join for i := 1; i < nClusters; i++ { clusters[i] = createCluster(t, hosts[i], dhts[i], cfgs[i], stores[i], cons[i], apis[i], ipfss[i], trackers[i], mons[i], allocs[i], infs[i], tracers[i]) @@ -611,7 +648,9 @@ func TestClustersPin(t *testing.T) { for i := 0; i < nPins; i++ { j := rand.Intn(nClusters) // choose a random cluster peer h, err := prefix.Sum(randomBytes()) // create random cid - checkErr(t, err) + if err != nil { + t.Fatal(err) + } _, err = clusters[j].Pin(ctx, h, api.PinOptions{}) if err != nil { t.Errorf("error pinning %s: %s", h, err) @@ -659,7 +698,14 @@ func TestClustersPin(t *testing.T) { t.Errorf("error unpinning %s: %s", pinList[i].Cid, err) } } - delay() + + switch consensus { + case "crdt": + time.Sleep(20 * time.Second) + default: + delay() + } + for i := 0; i < len(pinList); i++ { j := rand.Intn(nClusters) // choose a random cluster peer _, err := clusters[j].Unpin(ctx, pinList[i].Cid) @@ -669,6 +715,7 @@ func TestClustersPin(t *testing.T) { } delay() + funpinned := func(t *testing.T, c *Cluster) { status := c.tracker.StatusAll(ctx) for _, v := range status { @@ -1205,7 +1252,9 @@ func TestClustersReplicationOverall(t *testing.T) { // Pick a random cluster and hash j := rand.Intn(nClusters) // choose a random cluster peer h, err := prefix.Sum(randomBytes()) // create random cid - checkErr(t, err) + if err != nil { + t.Fatal(err) + } _, err = clusters[j].Pin(ctx, h, api.PinOptions{}) if err != nil { t.Error(err) @@ -1532,7 +1581,8 @@ func TestClustersReplicationMinMaxNoRealloc(t *testing.T) { // This test checks that repinning something that has becomed // underpinned does re-allocations when it's not sufficiently -// pinned anymore +// pinned anymore. +// FIXME: The manual repin only works if the pin options changed. func TestClustersReplicationMinMaxRealloc(t *testing.T) { ctx := context.Background() if nClusters < 5 { @@ -1549,7 +1599,9 @@ func TestClustersReplicationMinMaxRealloc(t *testing.T) { ttlDelay() // make sure metrics are in h := test.Cid1 - _, err := clusters[0].Pin(ctx, h, api.PinOptions{}) + _, err := clusters[0].Pin(ctx, h, api.PinOptions{ + Name: "a", + }) if err != nil { t.Fatal(err) } @@ -1569,17 +1621,23 @@ func TestClustersReplicationMinMaxRealloc(t *testing.T) { } // kill two of the allocations - alloc1 := peerIDMap[firstAllocations[0]] - alloc2 := peerIDMap[firstAllocations[1]] - safePeer := peerIDMap[firstAllocations[2]] + // Only the first allocated peer (or the second if the first is + // alerting) will automatically repin. + alloc1 := peerIDMap[firstAllocations[1]] + alloc2 := peerIDMap[firstAllocations[2]] + safePeer := peerIDMap[firstAllocations[0]] alloc1.Shutdown(ctx) alloc2.Shutdown(ctx) waitForLeaderAndMetrics(t, clusters) - // Repin - (although this might have been taken of if there was an alert - _, err = safePeer.Pin(ctx, h, api.PinOptions{}) + // Repin - (although this should have been taken of as alerts + // happen for the shutdown nodes. We force re-allocation by + // changing the name. + _, err = safePeer.Pin(ctx, h, api.PinOptions{ + Name: "b", + }) if err != nil { t.Fatal(err) } @@ -1819,9 +1877,9 @@ func TestClustersRebalanceOnPeerDown(t *testing.T) { } } -// Helper function for verifying cluster graph. Will only pass if exactly the +// Helper function for verifying cluster graph. Will only pass if exactly the // peers in clusterIDs are fully connected to each other and the expected ipfs -// mock connectivity exists. Cluster peers not in clusterIDs are assumed to +// mock connectivity exists. Cluster peers not in clusterIDs are assumed to // be disconnected and the graph should reflect this func validateClusterGraph(t *testing.T, graph api.ConnectGraph, clusterIDs map[string]struct{}, peerNum int) { // Check that all cluster peers see each other as peers @@ -1858,6 +1916,10 @@ func validateClusterGraph(t *testing.T, graph api.ConnectGraph, clusterIDs map[s } } + if len(graph.ClusterTrustLinks) != peerNum { + t.Errorf("Unexpected number of trust links in graph") + } + // Check that the mocked ipfs swarm is recorded if len(graph.IPFSLinks) != 1 { t.Error("Expected exactly one ipfs peer for all cluster nodes, the mocked peer") @@ -2132,3 +2194,57 @@ func TestClustersFollowerMode(t *testing.T) { } }) } + +func TestClusterPinsWithExpiration(t *testing.T) { + ctx := context.Background() + + clusters, mock := createClusters(t) + defer shutdownClusters(t, clusters, mock) + + ttlDelay() + + cl := clusters[rand.Intn(nClusters)] // choose a random cluster peer to query + + c := test.Cid1 + expireIn := 1 * time.Second + opts := api.PinOptions{ + ExpireAt: time.Now().Add(expireIn), + } + _, err := cl.Pin(ctx, c, opts) + if err != nil { + t.Fatal("pin should have worked:", err) + } + + pinDelay() + + pins, err := cl.Pins(ctx) + if err != nil { + t.Fatal(err) + } + if len(pins) != 1 { + t.Error("pin should be part of the state") + } + + // wait till expiry time + time.Sleep(expireIn) + + // manually call state sync on all peers, so we don't have to wait till + // state sync interval + for _, c := range clusters { + err = c.StateSync(ctx) + if err != nil { + t.Error(err) + } + } + + pinDelay() + + // state sync should have unpinned expired pin + pins, err = cl.Pins(ctx) + if err != nil { + t.Fatal(err) + } + if len(pins) != 0 { + t.Error("pin should not be part of the state") + } +} diff --git a/monitor/metrics/store.go b/monitor/metrics/store.go index 4f70dcd6..bb1b78b7 100644 --- a/monitor/metrics/store.go +++ b/monitor/metrics/store.go @@ -1,6 +1,7 @@ package metrics import ( + "sort" "sync" "github.com/ipfs/ipfs-cluster/api" @@ -84,7 +85,10 @@ func (mtrs *Store) LatestValid(name string) []*api.Metric { } metrics = append(metrics, m) } - return metrics + + sortedMetrics := api.MetricSlice(metrics) + sort.Stable(sortedMetrics) + return sortedMetrics } // AllMetrics returns the latest metrics for all peers and metrics types. It diff --git a/monitor/pubsubmon/pubsubmon.go b/monitor/pubsubmon/pubsubmon.go index d6919e85..a52d24d9 100644 --- a/monitor/pubsubmon/pubsubmon.go +++ b/monitor/pubsubmon/pubsubmon.go @@ -245,3 +245,11 @@ func (mon *Monitor) LatestMetrics(ctx context.Context, name string) []*api.Metri func (mon *Monitor) Alerts() <-chan *api.Alert { return mon.checker.Alerts() } + +// MetricNames lists all metric names. +func (mon *Monitor) MetricNames(ctx context.Context) []string { + ctx, span := trace.StartSpan(ctx, "monitor/pubsub/MetricNames") + defer span.End() + + return mon.metrics.MetricNames() +} diff --git a/peer_manager_test.go b/peer_manager_test.go index d8bb7fac..881c02d5 100644 --- a/peer_manager_test.go +++ b/peer_manager_test.go @@ -15,6 +15,7 @@ import ( cid "github.com/ipfs/go-cid" host "github.com/libp2p/go-libp2p-core/host" peer "github.com/libp2p/go-libp2p-core/peer" + dht "github.com/libp2p/go-libp2p-kad-dht" ma "github.com/multiformats/go-multiaddr" ) @@ -42,31 +43,33 @@ func peerManagerClusters(t *testing.T) ([]*Cluster, []*test.IpfsMock, host.Host) cfg := &Config{} cfg.Default() listen, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0") - cfg.ListenAddr = listen + cfg.ListenAddr = []ma.Multiaddr{listen} cfg.Secret = testingClusterSecret - // Create a bootstrapping libp2p host - h, _, dht, err := NewClusterHost(context.Background(), ident, cfg) - if err != nil { - t.Fatal(err) - } + h, _, idht := createHost(t, ident.PrivateKey, testingClusterSecret, cfg.ListenAddr) - // Connect all peers to that host. This will allow that they - // can discover each others via DHT. + // Connect host to all peers. This will allow that they can discover + // each others via DHT. for i := 0; i < nClusters; i++ { - err := cls[i].host.Connect( + err := h.Connect( context.Background(), peer.AddrInfo{ - ID: h.ID(), - Addrs: h.Addrs(), + ID: cls[i].host.ID(), + Addrs: cls[i].host.Addrs(), }, ) if err != nil { t.Fatal(err) } } - dht.Bootstrap(context.Background()) + ctx := context.Background() + dhtCfg := dht.BootstrapConfig{ + Queries: 1, + Period: 600 * time.Millisecond, + Timeout: 300 * time.Millisecond, + } + idht.BootstrapWithConfig(ctx, dhtCfg) return cls, mocks, h } @@ -164,23 +167,24 @@ func TestClustersPeerAdd(t *testing.T) { func TestClustersJoinBadPeer(t *testing.T) { ctx := context.Background() clusters, mocks, boot := peerManagerClusters(t) - defer shutdownClusters(t, clusters, mocks) + defer shutdownClusters(t, clusters[0:1], mocks[0:1]) defer boot.Close() + addr := clusterAddr(clusters[1]) + if len(clusters) < 2 { t.Skip("need at least 2 nodes for this test") } - addr := clusterAddr(clusters[1]) + for _, c := range clusters[1:] { + c.Shutdown(ctx) + } // We add a cluster that has been shutdown // (closed transports) - clusters[1].Shutdown(ctx) - // Let the OS actually close the ports. // Sometimes we hang otherwise. delay() - err := clusters[0].Join(ctx, addr) if err == nil { t.Error("expected an error") @@ -448,9 +452,13 @@ func TestClustersPeerRemoveReallocsPins(t *testing.T) { // pin the same number of Cids. for i := 0; i < nClusters; i++ { h, err := prefix.Sum(randomBytes()) - checkErr(t, err) + if err != nil { + t.Fatal(err) + } _, err = chosen.Pin(ctx, h, api.PinOptions{}) - checkErr(t, err) + if err != nil { + t.Fatal(err) + } ttlDelay() } diff --git a/pstoremgr/pstoremgr.go b/pstoremgr/pstoremgr.go index e0f77fb3..2d41ea59 100644 --- a/pstoremgr/pstoremgr.go +++ b/pstoremgr/pstoremgr.go @@ -15,7 +15,6 @@ import ( "time" logging "github.com/ipfs/go-log" - utils "github.com/ipfs/ipfs-cluster/utils" host "github.com/libp2p/go-libp2p-core/host" net "github.com/libp2p/go-libp2p-core/network" peer "github.com/libp2p/go-libp2p-core/peer" @@ -143,7 +142,7 @@ func (pm *Manager) filteredPeerAddrs(p peer.ID) []ma.Multiaddr { return peerDNSAddrs } - sort.Sort(utils.ByString(peerAddrs)) + sort.Sort(byString(peerAddrs)) return peerAddrs } @@ -397,3 +396,10 @@ func (ps *peerSort) Swap(i, j int) { ps.pinfos[i] = pinfo2 ps.pinfos[j] = pinfo1 } + +// byString can sort multiaddresses by its string +type byString []ma.Multiaddr + +func (m byString) Len() int { return len(m) } +func (m byString) Swap(i, j int) { m[i], m[j] = m[j], m[i] } +func (m byString) Less(i, j int) bool { return m[i].String() < m[j].String() } diff --git a/rpc_api.go b/rpc_api.go index b9815922..0f52a252 100644 --- a/rpc_api.go +++ b/rpc_api.go @@ -646,3 +646,9 @@ func (rpcapi *PeerMonitorRPCAPI) LatestMetrics(ctx context.Context, in string, o *out = rpcapi.mon.LatestMetrics(ctx, in) return nil } + +// MetricNames runs PeerMonitor.MetricNames(). +func (rpcapi *PeerMonitorRPCAPI) MetricNames(ctx context.Context, in struct{}, out *[]string) error { + *out = rpcapi.mon.MetricNames(ctx) + return nil +} diff --git a/rpc_policy.go b/rpc_policy.go index b0bd0942..a9109211 100644 --- a/rpc_policy.go +++ b/rpc_policy.go @@ -66,4 +66,5 @@ var DefaultRPCPolicy = map[string]RPCEndpointType{ // PeerMonitor methods "PeerMonitor.LatestMetrics": RPCClosed, + "PeerMonitor.MetricNames": RPCClosed, } diff --git a/sharness/t0030-ctl-pin.sh b/sharness/t0030-ctl-pin.sh index e642a19e..23c1f6dc 100755 --- a/sharness/t0030-ctl-pin.sh +++ b/sharness/t0030-ctl-pin.sh @@ -66,13 +66,13 @@ test_expect_success IPFS,CLUSTER "unpin data to cluster with ctl using ipfs path test_expect_success IPFS,CLUSTER "pin data to cluster with ctl using ipns paths" ' name=`docker exec ipfs sh -c "ipfs name publish -Q ${cid[0]}"` - ipfs-cluster-ctl pin add "/ipns/$name" && + ipfs-cluster-ctl pin add --wait --wait-timeout 2s "/ipns/$name" && ipfs-cluster-ctl pin ls "${cid[0]}" | grep -q "${cid[0]}" && ipfs-cluster-ctl status "${cid[0]}" | grep -q -i "PINNED" ' test_expect_success IPFS,CLUSTER "unpin data to cluster with ctl using ipns paths" ' - removed=(`ipfs-cluster-ctl pin rm "/ipns/$name"`) && + removed=(`ipfs-cluster-ctl pin rm --wait --wait-timeout 2s "/ipns/$name"`) && echo "${removed[0]}" | grep -q "${cid[0]}" && !(ipfs-cluster-ctl pin ls "${cid[0]}" | grep -q "${cid[0]}") && ipfs-cluster-ctl status "${cid[0]}" | grep -q -i "UNPINNED" diff --git a/sharness/t0032-ctl-health.sh b/sharness/t0032-ctl-health.sh index f35f058c..7dc5b1c1 100755 --- a/sharness/t0032-ctl-health.sh +++ b/sharness/t0032-ctl-health.sh @@ -16,13 +16,13 @@ test_expect_success IPFS,CLUSTER "health metrics with metric name must succeed" ipfs-cluster-ctl health metrics freespace ' -test_expect_success IPFS,CLUSTER "health metrics without metric name fails" ' - test_must_fail ipfs-cluster-ctl health metrics +test_expect_success IPFS,CLUSTER "health metrics without metric name doesn't fail" ' + ipfs-cluster-ctl health metrics ' test_expect_success IPFS,CLUSTER "list latest metrics logged by this peer" ' pid=`ipfs-cluster-ctl --enc=json id | jq -r ".id"` - ipfs-cluster-ctl health metrics freespace | grep -q -E "^$pid: [0-9]+ | Expire: .+T.+Z" + ipfs-cluster-ctl health metrics freespace | grep -q -E "(^$pid \| freespace: [0-9]+ (G|M|K)B \| Expires in: [0-9]+ seconds from now)" ' test_clean_ipfs diff --git a/state/interface.go b/state/interface.go index 17c56d1c..db8aa1c2 100644 --- a/state/interface.go +++ b/state/interface.go @@ -2,7 +2,7 @@ // IPFS Cluster must satisfy. package state -// State represents the shared state of the cluster and it +// State represents the shared state of the cluster import ( "context" "errors" @@ -17,7 +17,7 @@ import ( var ErrNotFound = errors.New("pin is not part of the pinset") // State is a wrapper to the Cluster shared state so that Pin objects can -// easily read, written and queried. The state can be marshaled and +// be easily read, written and queried. The state can be marshaled and // unmarshaled. Implementation should be thread-safe. type State interface { ReadOnly @@ -29,10 +29,9 @@ type State interface { Marshal(io.Writer) error // Unmarshal deserializes the state from marshaled bytes. Unmarshal(io.Reader) error - // Commit writes any batched operations. } -// ReadOnly represents a the read side of a State. +// ReadOnly represents the read side of a State. type ReadOnly interface { // List lists all the pins in the state. List(context.Context) ([]*api.Pin, error) diff --git a/test/ipfs_mock.go b/test/ipfs_mock.go index c8da7794..10090920 100644 --- a/test/ipfs_mock.go +++ b/test/ipfs_mock.go @@ -475,8 +475,8 @@ func (m *IpfsMock) Close() { defer m.closeMux.Unlock() if !m.closed { m.closed = true - close(m.reqCounter) m.server.Close() + close(m.reqCounter) } } diff --git a/test/rpc_api_mock.go b/test/rpc_api_mock.go index 70dd09a8..1cc20a7e 100644 --- a/test/rpc_api_mock.go +++ b/test/rpc_api_mock.go @@ -422,6 +422,13 @@ func (mock *mockPeerMonitor) LatestMetrics(ctx context.Context, in string, out * return nil } +// MetricNames runs PeerMonitor.MetricNames(). +func (mock *mockPeerMonitor) MetricNames(ctx context.Context, in struct{}, out *[]string) error { + k := []string{"ping", "freespace"} + *out = k + return nil +} + /* IPFSConnector methods */ func (mock *mockIPFSConnector) Pin(ctx context.Context, in *api.Pin, out *struct{}) error { diff --git a/util.go b/util.go index 389a575d..77d919aa 100644 --- a/util.go +++ b/util.go @@ -1,9 +1,12 @@ package ipfscluster import ( + "bytes" "errors" "fmt" + blake2b "golang.org/x/crypto/blake2b" + cid "github.com/ipfs/go-cid" peer "github.com/libp2p/go-libp2p-core/peer" ma "github.com/multiformats/go-multiaddr" @@ -93,3 +96,53 @@ func minInt(x, y int) int { // pin.Parents.Add(c) // } // } + +type distance [blake2b.Size256]byte + +type distanceChecker struct { + local peer.ID + otherPeers []peer.ID + cache map[peer.ID]distance +} + +func (dc distanceChecker) isClosest(ci cid.Cid) bool { + ciHash := convertKey(ci.KeyString()) + localPeerHash := dc.convertPeerID(dc.local) + myDistance := xor(ciHash, localPeerHash) + + for _, p := range dc.otherPeers { + peerHash := dc.convertPeerID(p) + distance := xor(peerHash, ciHash) + + // if myDistance is larger than for other peers... + if bytes.Compare(myDistance[:], distance[:]) > 0 { + return false + } + } + return true +} + +// convertPeerID hashes a Peer ID (Multihash). +func (dc distanceChecker) convertPeerID(id peer.ID) distance { + hash, ok := dc.cache[id] + if ok { + return hash + } + + hashBytes := convertKey(string(id)) + dc.cache[id] = hashBytes + return hashBytes +} + +// convertKey hashes a key. +func convertKey(id string) distance { + return blake2b.Sum256([]byte(id)) +} + +func xor(a, b distance) distance { + var c distance + for i := 0; i < len(c); i++ { + c[i] = a[i] ^ b[i] + } + return c +} diff --git a/utils/utils.go b/utils/utils.go deleted file mode 100644 index d0872402..00000000 --- a/utils/utils.go +++ /dev/null @@ -1,12 +0,0 @@ -package utils - -import ( - ma "github.com/multiformats/go-multiaddr" -) - -// ByString can sort multiaddresses by its string -type ByString []ma.Multiaddr - -func (m ByString) Len() int { return len(m) } -func (m ByString) Swap(i, j int) { m[i], m[j] = m[j], m[i] } -func (m ByString) Less(i, j int) bool { return m[i].String() < m[j].String() }