diff --git a/api/common/api.go b/api/common/api.go index 32189ca9..094c439c 100644 --- a/api/common/api.go +++ b/api/common/api.go @@ -54,6 +54,9 @@ func init() { rand.Seed(time.Now().UnixNano()) } +// StreamChannelSize is used to define buffer sizes for channels. +const StreamChannelSize = 1024 + // Common errors var ( // ErrNoEndpointEnabled is returned when the API is created but @@ -583,19 +586,23 @@ func (api *API) SendResponse( w.WriteHeader(status) } -// Iterator is a function that returns the next item. -type Iterator func() (interface{}, bool, error) +// StreamIterator is a function that returns the next item. It is used in +// StreamResponse. +type StreamIterator func() (interface{}, bool, error) // StreamResponse reads from an iterator and sends the response. -func (api *API) StreamResponse(w http.ResponseWriter, next Iterator) { +func (api *API) StreamResponse(w http.ResponseWriter, next StreamIterator, errCh chan error) { api.SetHeaders(w) enc := json.NewEncoder(w) flusher, flush := w.(http.Flusher) w.Header().Set("Trailer", "X-Stream-Error") total := 0 + var err error + var ok bool + var item interface{} for { - item, ok, err := next() + item, ok, err = next() if total == 0 { if err != nil { st := http.StatusInternalServerError @@ -612,16 +619,15 @@ func (api *API) StreamResponse(w http.ResponseWriter, next Iterator) { w.WriteHeader(http.StatusNoContent) return } + w.WriteHeader(http.StatusOK) } if err != nil { - w.Header().Set("X-Stream-Error", err.Error()) - // trailer error - return + break } // finish just fine if !ok { - return + break } // we have an item @@ -635,9 +641,19 @@ func (api *API) StreamResponse(w http.ResponseWriter, next Iterator) { flusher.Flush() } } + + if err != nil { + w.Header().Set("X-Stream-Error", err.Error()) + } + // check for function errors + for funcErr := range errCh { + if funcErr != nil { + w.Header().Add("X-Stream-Error", funcErr.Error()) + } + } } -// SetsHeaders sets all the headers that are common to all responses +// SetHeaders sets all the headers that are common to all responses // from this API. Called automatically from SendResponse(). func (api *API) SetHeaders(w http.ResponseWriter) { for header, values := range api.config.Headers { diff --git a/api/common/test/helpers.go b/api/common/test/helpers.go index d0624141..833fa09c 100644 --- a/api/common/test/helpers.go +++ b/api/common/test/helpers.go @@ -54,7 +54,7 @@ func ProcessResp(t *testing.T, httpResp *http.Response, err error, resp interfac // ProcessStreamingResp decodes a streaming response into the given type // and fails the test on error. -func ProcessStreamingResp(t *testing.T, httpResp *http.Response, err error, resp interface{}) { +func ProcessStreamingResp(t *testing.T, httpResp *http.Response, err error, resp interface{}, trailerError bool) { if err != nil { t.Fatal("error making streaming request: ", err) } @@ -97,6 +97,13 @@ func ProcessStreamingResp(t *testing.T, httpResp *http.Response, err error, resp } } } + trailerMsg := httpResp.Trailer.Get("X-Stream-Error") + if trailerError && trailerMsg == "" { + t.Error("expected trailer error") + } + if !trailerError && trailerMsg != "" { + t.Error("got trailer error: ", trailerMsg) + } } // CheckHeaders checks that all the headers are set to what is expected. @@ -246,19 +253,19 @@ func MakeStreamingPost(t *testing.T, api API, url string, body io.Reader, conten req.Header.Set("Content-Type", contentType) req.Header.Set("Origin", ClientOrigin) httpResp, err := c.Do(req) - ProcessStreamingResp(t, httpResp, err, resp) + ProcessStreamingResp(t, httpResp, err, resp, false) CheckHeaders(t, api.Headers(), url, httpResp.Header) } // MakeStreamingGet performs a GET request and uses ProcessStreamingResp -func MakeStreamingGet(t *testing.T, api API, url string, resp interface{}) { +func MakeStreamingGet(t *testing.T, api API, url string, resp interface{}, trailerError bool) { h := MakeHost(t, api) defer h.Close() c := HTTPClient(t, h, IsHTTPS(url)) req, _ := http.NewRequest(http.MethodGet, url, nil) req.Header.Set("Origin", ClientOrigin) httpResp, err := c.Do(req) - ProcessStreamingResp(t, httpResp, err, resp) + ProcessStreamingResp(t, httpResp, err, resp, trailerError) CheckHeaders(t, api.Headers(), url, httpResp.Header) } diff --git a/api/ipfsproxy/ipfsproxy.go b/api/ipfsproxy/ipfsproxy.go index 0382af80..5b1a5832 100644 --- a/api/ipfsproxy/ipfsproxy.go +++ b/api/ipfsproxy/ipfsproxy.go @@ -386,10 +386,15 @@ func (proxy *Server) unpinHandler(w http.ResponseWriter, r *http.Request) { func (proxy *Server) pinLsHandler(w http.ResponseWriter, r *http.Request) { proxy.setHeaders(w.Header(), r) - pinLs := ipfsPinLsResp{} - pinLs.Keys = make(map[string]ipfsPinType) - arg := r.URL.Query().Get("arg") + + stream := false + streamArg := r.URL.Query().Get("stream") + streamArg2 := r.URL.Query().Get("s") + if streamArg == "true" || streamArg2 == "true" { + stream = true + } + if arg != "" { c, err := cid.Decode(arg) if err != nil { @@ -409,8 +414,23 @@ func (proxy *Server) pinLsHandler(w http.ResponseWriter, r *http.Request) { ipfsErrorResponder(w, fmt.Sprintf("Error: path '%s' is not pinned", arg), -1) return } - pinLs.Keys[pin.Cid.String()] = ipfsPinType{ - Type: "recursive", + if stream { + ipinfo := api.IPFSPinInfo{ + Cid: api.Cid(pin.Cid), + Type: pin.Mode.ToIPFSPinStatus(), + } + resBytes, _ := json.Marshal(ipinfo) + w.WriteHeader(http.StatusOK) + w.Write(resBytes) + } else { + pinLs := ipfsPinLsResp{} + pinLs.Keys = make(map[string]ipfsPinType) + pinLs.Keys[pin.Cid.String()] = ipfsPinType{ + Type: "recursive", + } + resBytes, _ := json.Marshal(pinLs) + w.WriteHeader(http.StatusOK) + w.Write(resBytes) } } else { in := make(chan struct{}) @@ -432,22 +452,42 @@ func (proxy *Server) pinLsHandler(w http.ResponseWriter, r *http.Request) { ) }() - for pin := range pins { - pinLs.Keys[pin.Cid.String()] = ipfsPinType{ - Type: "recursive", + if stream { + w.Header().Set("Trailer", "X-Stream-Error") + w.WriteHeader(http.StatusOK) + for pin := range pins { + ipinfo := api.IPFSPinInfo{ + Cid: api.Cid(pin.Cid), + Type: pin.Mode.ToIPFSPinStatus(), + } + resBytes, _ := json.Marshal(ipinfo) + w.Write(resBytes) } - } + wg.Wait() + if err != nil { + w.Header().Add("X-Stream-Error", err.Error()) + return + } + } else { + pinLs := ipfsPinLsResp{} + pinLs.Keys = make(map[string]ipfsPinType) - wg.Wait() - if err != nil { - ipfsErrorResponder(w, err.Error(), -1) - return + for pin := range pins { + pinLs.Keys[pin.Cid.String()] = ipfsPinType{ + Type: "recursive", + } + } + + wg.Wait() + if err != nil { + ipfsErrorResponder(w, err.Error(), -1) + return + } + resBytes, _ := json.Marshal(pinLs) + w.WriteHeader(http.StatusOK) + w.Write(resBytes) } } - - resBytes, _ := json.Marshal(pinLs) - w.WriteHeader(http.StatusOK) - w.Write(resBytes) } func (proxy *Server) pinUpdateHandler(w http.ResponseWriter, r *http.Request) { diff --git a/api/pinsvcapi/pinsvcapi.go b/api/pinsvcapi/pinsvcapi.go index bd95b1f1..81431c67 100644 --- a/api/pinsvcapi/pinsvcapi.go +++ b/api/pinsvcapi/pinsvcapi.go @@ -346,20 +346,27 @@ func (api *API) listPins(w http.ResponseWriter, r *http.Request) { return } } else { - var globalPinInfos []types.GlobalPinInfo - err := api.rpcClient.CallContext( - r.Context(), - "", - "Cluster", - "StatusAll", - tst, - &globalPinInfos, - ) - if err != nil { - api.SendResponse(w, common.SetStatusAutomatically, err, nil) - return - } - for i, gpi := range globalPinInfos { + in := make(chan types.TrackerStatus, 1) + in <- tst + close(in) + out := make(chan types.GlobalPinInfo, common.StreamChannelSize) + errCh := make(chan error, 1) + + go func() { + defer close(errCh) + + errCh <- api.rpcClient.Stream( + r.Context(), + "", + "Cluster", + "StatusAll", + in, + out, + ) + }() + + i := 0 + for gpi := range out { st := globalPinInfoToSvcPinStatus(gpi.Cid.String(), gpi) if st.Status == pinsvc.StatusUndefined { // i.e things unpinning @@ -380,10 +387,17 @@ func (api *API) listPins(w http.ResponseWriter, r *http.Request) { continue } pinList.Results = append(pinList.Results, st) - if i+1 == opts.Limit { + i++ + if i == opts.Limit { break } } + + err := <-errCh + if err != nil { + api.SendResponse(w, common.SetStatusAutomatically, err, nil) + return + } } pinList.Count = len(pinList.Results) diff --git a/api/rest/client/client.go b/api/rest/client/client.go index 5819a5a8..0f433ded 100644 --- a/api/rest/client/client.go +++ b/api/rest/client/client.go @@ -85,9 +85,9 @@ type Client interface { // is fetched from all cluster peers. Status(ctx context.Context, ci cid.Cid, local bool) (api.GlobalPinInfo, error) // StatusCids status information for the requested CIDs. - StatusCids(ctx context.Context, cids []cid.Cid, local bool) ([]api.GlobalPinInfo, error) + StatusCids(ctx context.Context, cids []cid.Cid, local bool, out chan<- api.GlobalPinInfo) error // StatusAll gathers Status() for all tracked items. - StatusAll(ctx context.Context, filter api.TrackerStatus, local bool) ([]api.GlobalPinInfo, error) + StatusAll(ctx context.Context, filter api.TrackerStatus, local bool, out chan<- api.GlobalPinInfo) error // Recover retriggers pin or unpin ipfs operations for a Cid in error // state. If local is true, the operation is limited to the current @@ -96,7 +96,7 @@ type Client interface { // RecoverAll triggers Recover() operations on all tracked items. If // local is true, the operation is limited to the current peer. // Otherwise, it happens everywhere. - RecoverAll(ctx context.Context, local bool) ([]api.GlobalPinInfo, error) + RecoverAll(ctx context.Context, local bool, out chan<- api.GlobalPinInfo) error // Alerts returns information health events in the cluster (expired // metrics etc.). diff --git a/api/rest/client/lbclient.go b/api/rest/client/lbclient.go index d3be1295..d223cda3 100644 --- a/api/rest/client/lbclient.go +++ b/api/rest/client/lbclient.go @@ -253,16 +253,13 @@ func (lc *loadBalancingClient) Status(ctx context.Context, ci cid.Cid, local boo // StatusCids returns Status() information for the given Cids. If local is // true, the information affects only the current peer, otherwise the // information is fetched from all cluster peers. -func (lc *loadBalancingClient) StatusCids(ctx context.Context, cids []cid.Cid, local bool) ([]api.GlobalPinInfo, error) { - var pinInfos []api.GlobalPinInfo +func (lc *loadBalancingClient) StatusCids(ctx context.Context, cids []cid.Cid, local bool, out chan<- api.GlobalPinInfo) error { call := func(c Client) error { - var err error - pinInfos, err = c.StatusCids(ctx, cids, local) - return err + return c.StatusCids(ctx, cids, local, out) } err := lc.retry(0, call) - return pinInfos, err + return err } // StatusAll gathers Status() for all tracked items. If a filter is @@ -270,16 +267,13 @@ func (lc *loadBalancingClient) StatusCids(ctx context.Context, cids []cid.Cid, l // will be returned. A filter can be built by merging TrackerStatuses with // a bitwise OR operation (st1 | st2 | ...). A "0" filter value (or // api.TrackerStatusUndefined), means all. -func (lc *loadBalancingClient) StatusAll(ctx context.Context, filter api.TrackerStatus, local bool) ([]api.GlobalPinInfo, error) { - var pinInfos []api.GlobalPinInfo +func (lc *loadBalancingClient) StatusAll(ctx context.Context, filter api.TrackerStatus, local bool, out chan<- api.GlobalPinInfo) error { call := func(c Client) error { - var err error - pinInfos, err = c.StatusAll(ctx, filter, local) - return err + return c.StatusAll(ctx, filter, local, out) } err := lc.retry(0, call) - return pinInfos, err + return err } // Recover retriggers pin or unpin ipfs operations for a Cid in error state. @@ -300,16 +294,13 @@ func (lc *loadBalancingClient) Recover(ctx context.Context, ci cid.Cid, local bo // RecoverAll triggers Recover() operations on all tracked items. If local is // true, the operation is limited to the current peer. Otherwise, it happens // everywhere. -func (lc *loadBalancingClient) RecoverAll(ctx context.Context, local bool) ([]api.GlobalPinInfo, error) { - var pinInfos []api.GlobalPinInfo +func (lc *loadBalancingClient) RecoverAll(ctx context.Context, local bool, out chan<- api.GlobalPinInfo) error { call := func(c Client) error { - var err error - pinInfos, err = c.RecoverAll(ctx, local) - return err + return c.RecoverAll(ctx, local, out) } err := lc.retry(0, call) - return pinInfos, err + return err } // Alerts returns things that are wrong with cluster. diff --git a/api/rest/client/methods.go b/api/rest/client/methods.go index 88eb2bd7..4a898e08 100644 --- a/api/rest/client/methods.go +++ b/api/rest/client/methods.go @@ -156,11 +156,11 @@ func (c *defaultClient) UnpinPath(ctx context.Context, p string) (api.Pin, error // Allocations returns the consensus state listing all tracked items and // the peers that should be pinning them. func (c *defaultClient) Allocations(ctx context.Context, filter api.PinType, out chan<- api.Pin) error { + defer close(out) + ctx, span := trace.StartSpan(ctx, "client/Allocations") defer span.End() - defer close(out) - types := []api.PinType{ api.DataType, api.MetaType, @@ -191,14 +191,13 @@ func (c *defaultClient) Allocations(ctx context.Context, filter api.PinType, out } f := url.QueryEscape(strings.Join(strFilter, ",")) - err := c.doStream( + return c.doStream( ctx, "GET", fmt.Sprintf("/allocations?filter=%s", f), nil, nil, handler) - return err } // Allocation returns the current allocations for a given Cid. @@ -233,8 +232,8 @@ func (c *defaultClient) Status(ctx context.Context, ci cid.Cid, local bool) (api // StatusCids returns Status() information for the given Cids. If local is // true, the information affects only the current peer, otherwise the // information is fetched from all cluster peers. -func (c *defaultClient) StatusCids(ctx context.Context, cids []cid.Cid, local bool) ([]api.GlobalPinInfo, error) { - return c.statusAllWithCids(ctx, api.TrackerStatusUndefined, cids, local) +func (c *defaultClient) StatusCids(ctx context.Context, cids []cid.Cid, local bool, out chan<- api.GlobalPinInfo) error { + return c.statusAllWithCids(ctx, api.TrackerStatusUndefined, cids, local, out) } // StatusAll gathers Status() for all tracked items. If a filter is @@ -242,21 +241,20 @@ func (c *defaultClient) StatusCids(ctx context.Context, cids []cid.Cid, local bo // will be returned. A filter can be built by merging TrackerStatuses with // a bitwise OR operation (st1 | st2 | ...). A "0" filter value (or // api.TrackerStatusUndefined), means all. -func (c *defaultClient) StatusAll(ctx context.Context, filter api.TrackerStatus, local bool) ([]api.GlobalPinInfo, error) { - return c.statusAllWithCids(ctx, filter, nil, local) +func (c *defaultClient) StatusAll(ctx context.Context, filter api.TrackerStatus, local bool, out chan<- api.GlobalPinInfo) error { + return c.statusAllWithCids(ctx, filter, nil, local, out) } -func (c *defaultClient) statusAllWithCids(ctx context.Context, filter api.TrackerStatus, cids []cid.Cid, local bool) ([]api.GlobalPinInfo, error) { +func (c *defaultClient) statusAllWithCids(ctx context.Context, filter api.TrackerStatus, cids []cid.Cid, local bool, out chan<- api.GlobalPinInfo) error { + defer close(out) ctx, span := trace.StartSpan(ctx, "client/StatusAll") defer span.End() - var gpis []api.GlobalPinInfo - filterStr := "" if filter != api.TrackerStatusUndefined { // undefined filter means "all" filterStr = filter.String() if filterStr == "" { - return nil, errors.New("invalid filter value") + return errors.New("invalid filter value") } } @@ -265,16 +263,25 @@ func (c *defaultClient) statusAllWithCids(ctx context.Context, filter api.Tracke cidsStr[i] = c.String() } - err := c.do( + handler := func(dec *json.Decoder) error { + var obj api.GlobalPinInfo + err := dec.Decode(&obj) + if err != nil { + return err + } + out <- obj + return nil + } + + return c.doStream( ctx, "GET", fmt.Sprintf("/pins?local=%t&filter=%s&cids=%s", local, url.QueryEscape(filterStr), strings.Join(cidsStr, ",")), nil, nil, - &gpis, + handler, ) - return gpis, err } // Recover retriggers pin or unpin ipfs operations for a Cid in error state. @@ -292,13 +299,29 @@ func (c *defaultClient) Recover(ctx context.Context, ci cid.Cid, local bool) (ap // RecoverAll triggers Recover() operations on all tracked items. If local is // true, the operation is limited to the current peer. Otherwise, it happens // everywhere. -func (c *defaultClient) RecoverAll(ctx context.Context, local bool) ([]api.GlobalPinInfo, error) { +func (c *defaultClient) RecoverAll(ctx context.Context, local bool, out chan<- api.GlobalPinInfo) error { + defer close(out) + ctx, span := trace.StartSpan(ctx, "client/RecoverAll") defer span.End() - var gpis []api.GlobalPinInfo - err := c.do(ctx, "POST", fmt.Sprintf("/pins/recover?local=%t", local), nil, nil, &gpis) - return gpis, err + handler := func(dec *json.Decoder) error { + var obj api.GlobalPinInfo + err := dec.Decode(&obj) + if err != nil { + return err + } + out <- obj + return nil + } + + return c.doStream( + ctx, + "POST", + fmt.Sprintf("/pins/recover?local=%t", local), + nil, + nil, + handler) } // Alerts returns information health events in the cluster (expired metrics diff --git a/api/rest/client/methods_test.go b/api/rest/client/methods_test.go index d4a2026e..71c9e8d9 100644 --- a/api/rest/client/methods_test.go +++ b/api/rest/client/methods_test.go @@ -346,10 +346,16 @@ func TestStatusCids(t *testing.T) { defer shutdown(api) testF := func(t *testing.T, c Client) { - pins, err := c.StatusCids(ctx, []cid.Cid{test.Cid1}, false) - if err != nil { - t.Fatal(err) - } + out := make(chan types.GlobalPinInfo) + + go func() { + err := c.StatusCids(ctx, []cid.Cid{test.Cid1}, false, out) + if err != nil { + t.Error(err) + } + }() + + pins := collectGlobalPinInfos(t, out) if len(pins) != 1 { t.Fatal("wrong number of pins returned") } @@ -361,48 +367,87 @@ func TestStatusCids(t *testing.T) { testClients(t, api, testF) } +func collectGlobalPinInfos(t *testing.T, out <-chan types.GlobalPinInfo) []types.GlobalPinInfo { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var gpis []types.GlobalPinInfo + for { + select { + case <-ctx.Done(): + t.Error(ctx.Err()) + return gpis + case gpi, ok := <-out: + if !ok { + return gpis + } + gpis = append(gpis, gpi) + } + } +} + func TestStatusAll(t *testing.T) { ctx := context.Background() api := testAPI(t) defer shutdown(api) testF := func(t *testing.T, c Client) { - pins, err := c.StatusAll(ctx, 0, false) - if err != nil { - t.Fatal(err) - } + out := make(chan types.GlobalPinInfo) + go func() { + err := c.StatusAll(ctx, 0, false, out) + if err != nil { + t.Error(err) + } + }() + pins := collectGlobalPinInfos(t, out) if len(pins) == 0 { t.Error("there should be some pins") } - // With local true - pins, err = c.StatusAll(ctx, 0, true) - if err != nil { - t.Fatal(err) - } + out2 := make(chan types.GlobalPinInfo) + go func() { + err := c.StatusAll(ctx, 0, true, out2) + if err != nil { + t.Error(err) + } + }() + pins = collectGlobalPinInfos(t, out2) + if len(pins) != 2 { t.Error("there should be two pins") } - // With filter option - pins, err = c.StatusAll(ctx, types.TrackerStatusPinning, false) - if err != nil { - t.Fatal(err) - } + out3 := make(chan types.GlobalPinInfo) + go func() { + err := c.StatusAll(ctx, types.TrackerStatusPinning, false, out3) + if err != nil { + t.Error(err) + } + }() + pins = collectGlobalPinInfos(t, out3) + if len(pins) != 1 { t.Error("there should be one pin") } - pins, err = c.StatusAll(ctx, types.TrackerStatusPinned|types.TrackerStatusError, false) - if err != nil { - t.Fatal(err) - } + out4 := make(chan types.GlobalPinInfo) + go func() { + err := c.StatusAll(ctx, types.TrackerStatusPinned|types.TrackerStatusError, false, out4) + if err != nil { + t.Error(err) + } + }() + pins = collectGlobalPinInfos(t, out4) + if len(pins) != 2 { t.Error("there should be two pins") } - _, err = c.StatusAll(ctx, 1<<25, false) + out5 := make(chan types.GlobalPinInfo, 1) + err := c.StatusAll(ctx, 1<<25, false, out5) if err == nil { t.Error("expected an error") } @@ -435,12 +480,14 @@ func TestRecoverAll(t *testing.T) { defer shutdown(api) testF := func(t *testing.T, c Client) { - _, err := c.RecoverAll(ctx, true) + out := make(chan types.GlobalPinInfo, 10) + err := c.RecoverAll(ctx, true, out) if err != nil { t.Fatal(err) } - _, err = c.RecoverAll(ctx, false) + out2 := make(chan types.GlobalPinInfo, 10) + err = c.RecoverAll(ctx, false, out2) if err != nil { t.Fatal(err) } diff --git a/api/rest/restapi.go b/api/rest/restapi.go index 89e969fa..9445050c 100644 --- a/api/rest/restapi.go +++ b/api/rest/restapi.go @@ -21,7 +21,6 @@ import ( "github.com/ipfs/ipfs-cluster/adder/adderutils" types "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/api/common" - "go.uber.org/multierr" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/host" @@ -457,12 +456,15 @@ func (api *API) allocationsHandler(w http.ResponseWriter, r *http.Request) { close(in) pins := make(chan types.Pin) + errCh := make(chan error, 1) ctx, cancel := context.WithCancel(r.Context()) defer cancel() go func() { - err := api.rpcClient.Stream( + defer close(errCh) + + errCh <- api.rpcClient.Stream( r.Context(), "", "Cluster", @@ -470,10 +472,6 @@ func (api *API) allocationsHandler(w http.ResponseWriter, r *http.Request) { in, pins, ) - if err != nil { - logger.Error(err) - cancel() - } }() iter := func() (interface{}, bool, error) { @@ -481,6 +479,7 @@ func (api *API) allocationsHandler(w http.ResponseWriter, r *http.Request) { var ok bool iterloop: for { + select { case <-ctx.Done(): break iterloop @@ -498,7 +497,7 @@ func (api *API) allocationsHandler(w http.ResponseWriter, r *http.Request) { return p, ok, ctx.Err() } - api.StreamResponse(w, iter) + api.StreamResponse(w, iter, errCh) } func (api *API) allocationHandler(w http.ResponseWriter, r *http.Request) { @@ -517,6 +516,9 @@ func (api *API) allocationHandler(w http.ResponseWriter, r *http.Request) { } func (api *API) statusAllHandler(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + queryValues := r.URL.Query() if queryValues.Get("cids") != "" { api.statusCidsHandler(w, r) @@ -525,8 +527,6 @@ func (api *API) statusAllHandler(w http.ResponseWriter, r *http.Request) { local := queryValues.Get("local") - var globalPinInfos []types.GlobalPinInfo - filterStr := queryValues.Get("filter") filter := types.TrackerStatusFromString(filterStr) // FIXME: This is a bit lazy, as "invalidxx,pinned" would result in a @@ -536,42 +536,68 @@ func (api *API) statusAllHandler(w http.ResponseWriter, r *http.Request) { return } - if local == "true" { - var pinInfos []types.PinInfo + var iter common.StreamIterator + in := make(chan types.TrackerStatus, 1) + in <- filter + close(in) + errCh := make(chan error, 1) - err := api.rpcClient.CallContext( - r.Context(), - "", - "Cluster", - "StatusAllLocal", - filter, - &pinInfos, - ) - if err != nil { - api.SendResponse(w, common.SetStatusAutomatically, err, nil) - return + if local == "true" { + out := make(chan types.PinInfo, common.StreamChannelSize) + iter = func() (interface{}, bool, error) { + select { + case <-ctx.Done(): + return nil, false, ctx.Err() + case p, ok := <-out: + return p.ToGlobal(), ok, nil + } } - globalPinInfos = pinInfosToGlobal(pinInfos) + + go func() { + defer close(errCh) + + errCh <- api.rpcClient.Stream( + r.Context(), + "", + "Cluster", + "StatusAllLocal", + in, + out, + ) + }() + } else { - err := api.rpcClient.CallContext( - r.Context(), - "", - "Cluster", - "StatusAll", - filter, - &globalPinInfos, - ) - if err != nil { - api.SendResponse(w, common.SetStatusAutomatically, err, nil) - return + out := make(chan types.GlobalPinInfo, common.StreamChannelSize) + iter = func() (interface{}, bool, error) { + select { + case <-ctx.Done(): + return nil, false, ctx.Err() + case p, ok := <-out: + return p, ok, nil + } } + go func() { + defer close(errCh) + + errCh <- api.rpcClient.Stream( + r.Context(), + "", + "Cluster", + "StatusAll", + in, + out, + ) + }() } - api.SendResponse(w, common.SetStatusAutomatically, nil, globalPinInfos) + api.StreamResponse(w, iter, errCh) } // request statuses for multiple CIDs in parallel. func (api *API) statusCidsHandler(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + queryValues := r.URL.Query() filterCidsStr := strings.Split(queryValues.Get("cids"), ",") var cids []cid.Cid @@ -587,17 +613,15 @@ func (api *API) statusCidsHandler(w http.ResponseWriter, r *http.Request) { local := queryValues.Get("local") - type gpiResult struct { - gpi types.GlobalPinInfo - err error - } - gpiCh := make(chan gpiResult, len(cids)) + gpiCh := make(chan types.GlobalPinInfo, len(cids)) + errCh := make(chan error, len(cids)) var wg sync.WaitGroup wg.Add(len(cids)) // Close channel when done go func() { wg.Wait() + close(errCh) close(gpiCh) }() @@ -607,14 +631,18 @@ func (api *API) statusCidsHandler(w http.ResponseWriter, r *http.Request) { defer wg.Done() var pinInfo types.PinInfo err := api.rpcClient.CallContext( - r.Context(), + ctx, "", "Cluster", "StatusLocal", c, &pinInfo, ) - gpiCh <- gpiResult{gpi: pinInfo.ToGlobal(), err: err} + if err != nil { + errCh <- err + return + } + gpiCh <- pinInfo.ToGlobal() }(ci) } } else { @@ -623,25 +651,28 @@ func (api *API) statusCidsHandler(w http.ResponseWriter, r *http.Request) { defer wg.Done() var pinInfo types.GlobalPinInfo err := api.rpcClient.CallContext( - r.Context(), + ctx, "", "Cluster", "Status", c, &pinInfo, ) - gpiCh <- gpiResult{gpi: pinInfo, err: err} + if err != nil { + errCh <- err + return + } + gpiCh <- pinInfo }(ci) } } - var gpis []types.GlobalPinInfo - var err error - for gpiResult := range gpiCh { - gpis = append(gpis, gpiResult.gpi) - err = multierr.Append(err, gpiResult.err) + iter := func() (interface{}, bool, error) { + gpi, ok := <-gpiCh + return gpi, ok, nil } - api.SendResponse(w, common.SetStatusAutomatically, err, gpis) + + api.StreamResponse(w, iter, errCh) } func (api *API) statusHandler(w http.ResponseWriter, r *http.Request) { @@ -676,31 +707,66 @@ func (api *API) statusHandler(w http.ResponseWriter, r *http.Request) { } func (api *API) recoverAllHandler(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + queryValues := r.URL.Query() local := queryValues.Get("local") + + var iter common.StreamIterator + in := make(chan struct{}) + close(in) + errCh := make(chan error, 1) + if local == "true" { - var pinInfos []types.PinInfo - err := api.rpcClient.CallContext( - r.Context(), - "", - "Cluster", - "RecoverAllLocal", - struct{}{}, - &pinInfos, - ) - api.SendResponse(w, common.SetStatusAutomatically, err, pinInfosToGlobal(pinInfos)) + out := make(chan types.PinInfo, common.StreamChannelSize) + iter = func() (interface{}, bool, error) { + select { + case <-ctx.Done(): + return nil, false, ctx.Err() + case p, ok := <-out: + return p.ToGlobal(), ok, nil + } + } + + go func() { + defer close(errCh) + + errCh <- api.rpcClient.Stream( + r.Context(), + "", + "Cluster", + "RecoverAllLocal", + in, + out, + ) + }() + } else { - var globalPinInfos []types.GlobalPinInfo - err := api.rpcClient.CallContext( - r.Context(), - "", - "Cluster", - "RecoverAll", - struct{}{}, - &globalPinInfos, - ) - api.SendResponse(w, common.SetStatusAutomatically, err, globalPinInfos) + out := make(chan types.GlobalPinInfo, common.StreamChannelSize) + iter = func() (interface{}, bool, error) { + select { + case <-ctx.Done(): + return nil, false, ctx.Err() + case p, ok := <-out: + return p, ok, nil + } + } + go func() { + defer close(errCh) + + errCh <- api.rpcClient.Stream( + r.Context(), + "", + "Cluster", + "RecoverAll", + in, + out, + ) + }() } + + api.StreamResponse(w, iter, errCh) } func (api *API) recoverHandler(w http.ResponseWriter, r *http.Request) { @@ -772,12 +838,3 @@ func repoGCToGlobal(r types.RepoGC) types.GlobalRepoGC { }, } } - -func pinInfosToGlobal(pInfos []types.PinInfo) []types.GlobalPinInfo { - gPInfos := make([]types.GlobalPinInfo, len(pInfos)) - for i, p := range pInfos { - gpi := p.ToGlobal() - gPInfos[i] = gpi - } - return gPInfos -} diff --git a/api/rest/restapi_test.go b/api/rest/restapi_test.go index 4af3cfde..c348ccc4 100644 --- a/api/rest/restapi_test.go +++ b/api/rest/restapi_test.go @@ -222,7 +222,7 @@ func TestAPIAddFileEndpointShard(t *testing.T) { defer closer.Close() mpContentType := "multipart/form-data; boundary=" + body.Boundary() resp := api.AddedOutput{} - fmtStr1 := "/add?shard=true&repl_min=-1&repl_max=-1&stream-channels=true" + fmtStr1 := "/add?shard=true&repl_min=-1&repl_max=-1&stream-channels=true&shard-size=1000000" shardURL := url(rest) + fmtStr1 test.MakeStreamingPost(t, rest, shardURL, body, mpContentType, &resp) } @@ -507,14 +507,14 @@ func TestAPIAllocationsEndpoint(t *testing.T) { tf := func(t *testing.T, url test.URLFunc) { var resp []api.Pin - test.MakeStreamingGet(t, rest, url(rest)+"/allocations?filter=pin,meta-pin", &resp) + test.MakeStreamingGet(t, rest, url(rest)+"/allocations?filter=pin,meta-pin", &resp, false) if len(resp) != 3 || !resp[0].Cid.Equals(clustertest.Cid1) || !resp[1].Cid.Equals(clustertest.Cid2) || !resp[2].Cid.Equals(clustertest.Cid3) { t.Error("unexpected pin list: ", resp) } - test.MakeStreamingGet(t, rest, url(rest)+"/allocations", &resp) + test.MakeStreamingGet(t, rest, url(rest)+"/allocations", &resp, false) if len(resp) != 3 || !resp[0].Cid.Equals(clustertest.Cid1) || !resp[1].Cid.Equals(clustertest.Cid2) || !resp[2].Cid.Equals(clustertest.Cid3) { @@ -522,7 +522,7 @@ func TestAPIAllocationsEndpoint(t *testing.T) { } errResp := api.Error{} - test.MakeStreamingGet(t, rest, url(rest)+"/allocations?filter=invalid", &errResp) + test.MakeStreamingGet(t, rest, url(rest)+"/allocations?filter=invalid", &errResp, false) if errResp.Code != http.StatusBadRequest { t.Error("an invalid filter value should 400") } @@ -615,8 +615,9 @@ func TestAPIStatusAllEndpoint(t *testing.T) { defer rest.Shutdown(ctx) tf := func(t *testing.T, url test.URLFunc) { - var resp []*api.GlobalPinInfo - test.MakeGet(t, rest, url(rest)+"/pins", &resp) + var resp []api.GlobalPinInfo + + test.MakeStreamingGet(t, rest, url(rest)+"/pins", &resp, false) // mockPinTracker returns 3 items for Cluster.StatusAll if len(resp) != 3 || @@ -626,8 +627,8 @@ func TestAPIStatusAllEndpoint(t *testing.T) { } // Test local=true - var resp2 []*api.GlobalPinInfo - test.MakeGet(t, rest, url(rest)+"/pins?local=true", &resp2) + var resp2 []api.GlobalPinInfo + test.MakeStreamingGet(t, rest, url(rest)+"/pins?local=true", &resp2, false) // mockPinTracker calls pintracker.StatusAll which returns 2 // items. if len(resp2) != 2 { @@ -635,38 +636,38 @@ func TestAPIStatusAllEndpoint(t *testing.T) { } // Test with filter - var resp3 []*api.GlobalPinInfo - test.MakeGet(t, rest, url(rest)+"/pins?filter=queued", &resp3) + var resp3 []api.GlobalPinInfo + test.MakeStreamingGet(t, rest, url(rest)+"/pins?filter=queued", &resp3, false) if len(resp3) != 0 { t.Errorf("unexpected statusAll+filter=queued resp:\n %+v", resp3) } - var resp4 []*api.GlobalPinInfo - test.MakeGet(t, rest, url(rest)+"/pins?filter=pinned", &resp4) + var resp4 []api.GlobalPinInfo + test.MakeStreamingGet(t, rest, url(rest)+"/pins?filter=pinned", &resp4, false) if len(resp4) != 1 { t.Errorf("unexpected statusAll+filter=pinned resp:\n %+v", resp4) } - var resp5 []*api.GlobalPinInfo - test.MakeGet(t, rest, url(rest)+"/pins?filter=pin_error", &resp5) + var resp5 []api.GlobalPinInfo + test.MakeStreamingGet(t, rest, url(rest)+"/pins?filter=pin_error", &resp5, false) if len(resp5) != 1 { t.Errorf("unexpected statusAll+filter=pin_error resp:\n %+v", resp5) } - var resp6 []*api.GlobalPinInfo - test.MakeGet(t, rest, url(rest)+"/pins?filter=error", &resp6) + var resp6 []api.GlobalPinInfo + test.MakeStreamingGet(t, rest, url(rest)+"/pins?filter=error", &resp6, false) if len(resp6) != 1 { t.Errorf("unexpected statusAll+filter=error resp:\n %+v", resp6) } - var resp7 []*api.GlobalPinInfo - test.MakeGet(t, rest, url(rest)+"/pins?filter=error,pinned", &resp7) + var resp7 []api.GlobalPinInfo + test.MakeStreamingGet(t, rest, url(rest)+"/pins?filter=error,pinned", &resp7, false) if len(resp7) != 2 { t.Errorf("unexpected statusAll+filter=error,pinned resp:\n %+v", resp7) } var errorResp api.Error - test.MakeGet(t, rest, url(rest)+"/pins?filter=invalid", &errorResp) + test.MakeStreamingGet(t, rest, url(rest)+"/pins?filter=invalid", &errorResp, false) if errorResp.Code != http.StatusBadRequest { t.Error("an invalid filter value should 400") } @@ -681,32 +682,32 @@ func TestAPIStatusAllWithCidsEndpoint(t *testing.T) { defer rest.Shutdown(ctx) tf := func(t *testing.T, url test.URLFunc) { - var resp []*api.GlobalPinInfo + var resp []api.GlobalPinInfo cids := []string{ clustertest.Cid1.String(), clustertest.Cid2.String(), clustertest.Cid3.String(), clustertest.Cid4.String(), } - test.MakeGet(t, rest, url(rest)+"/pins/?cids="+strings.Join(cids, ","), &resp) + test.MakeStreamingGet(t, rest, url(rest)+"/pins/?cids="+strings.Join(cids, ","), &resp, false) if len(resp) != 4 { t.Error("wrong number of responses") } // Test local=true - var resp2 []*api.GlobalPinInfo - test.MakeGet(t, rest, url(rest)+"/pins/?local=true&cids="+strings.Join(cids, ","), &resp2) + var resp2 []api.GlobalPinInfo + test.MakeStreamingGet(t, rest, url(rest)+"/pins/?local=true&cids="+strings.Join(cids, ","), &resp2, false) if len(resp2) != 4 { t.Error("wrong number of responses") } - // Test with an error + // Test with an error. This should produce a trailer error. cids = append(cids, clustertest.ErrorCid.String()) - var errorResp api.Error - test.MakeGet(t, rest, url(rest)+"/pins/?local=true&cids="+strings.Join(cids, ","), &errorResp) - if errorResp.Message != clustertest.ErrBadCid.Error() { - t.Error("expected an error") + var resp3 []api.GlobalPinInfo + test.MakeStreamingGet(t, rest, url(rest)+"/pins/?local=true&cids="+strings.Join(cids, ","), &resp3, true) + if len(resp3) != 4 { + t.Error("wrong number of responses") } } @@ -782,14 +783,14 @@ func TestAPIRecoverAllEndpoint(t *testing.T) { defer rest.Shutdown(ctx) tf := func(t *testing.T, url test.URLFunc) { - var resp []*api.GlobalPinInfo - test.MakePost(t, rest, url(rest)+"/pins/recover?local=true", []byte{}, &resp) + var resp []api.GlobalPinInfo + test.MakeStreamingPost(t, rest, url(rest)+"/pins/recover?local=true", nil, "", &resp) if len(resp) != 0 { t.Fatal("bad response length") } - var resp1 []*api.GlobalPinInfo - test.MakePost(t, rest, url(rest)+"/pins/recover", []byte{}, &resp1) + var resp1 []api.GlobalPinInfo + test.MakeStreamingPost(t, rest, url(rest)+"/pins/recover", nil, "", &resp1) if len(resp1) == 0 { t.Fatal("bad response length") } diff --git a/api/types.go b/api/types.go index 3e8defb0..4766b93a 100644 --- a/api/types.go +++ b/api/types.go @@ -217,6 +217,36 @@ func IPFSPinStatusFromString(t string) IPFSPinStatus { } } +// String returns the string form of the status as written by IPFS. +func (ips IPFSPinStatus) String() string { + switch ips { + case IPFSPinStatusDirect: + return "direct" + case IPFSPinStatusRecursive: + return "recursive" + case IPFSPinStatusIndirect: + return "indirect" + default: + return "" + } +} + +// UnmarshalJSON parses a status from JSON +func (ips *IPFSPinStatus) UnmarshalJSON(b []byte) error { + var str string + err := json.Unmarshal(b, &str) + if err != nil { + return err + } + *ips = IPFSPinStatusFromString(str) + return nil +} + +// MarshalJSON converts a status to JSON. +func (ips IPFSPinStatus) MarshalJSON() ([]byte, error) { + return json.Marshal(ips.String()) +} + // IsPinned returns true if the item is pinned as expected by the // maxDepth parameter. func (ips IPFSPinStatus) IsPinned(maxDepth PinDepth) bool { @@ -247,6 +277,40 @@ var ipfsPinStatus2TrackerStatusMap = map[IPFSPinStatus]TrackerStatus{ IPFSPinStatusError: TrackerStatusClusterError, //TODO(ajl): check suitability } +// Cid is a CID with the MarshalJSON/UnmarshalJSON methods overwritten. +type Cid cid.Cid + +func (c Cid) String() string { + return cid.Cid(c).String() +} + +// MarshalJSON marshals a CID as JSON as a normal CID string. +func (c Cid) MarshalJSON() ([]byte, error) { + return json.Marshal(c.String()) +} + +// UnmarshalJSON reads a CID from its representation as JSON string. +func (c *Cid) UnmarshalJSON(b []byte) error { + var cidStr string + err := json.Unmarshal(b, &cidStr) + if err != nil { + return err + } + cc, err := cid.Decode(cidStr) + if err != nil { + return err + } + *c = Cid(cc) + return nil +} + +// IPFSPinInfo represents an IPFS Pin, which only has a CID and type. +// Its JSON form is what IPFS returns when querying a pinset. +type IPFSPinInfo struct { + Cid Cid `json:"Cid" codec:"c"` + Type IPFSPinStatus `json:"Type" codec:"t"` +} + // GlobalPinInfo contains cluster-wide status information about a tracked Cid, // indexed by cluster peer. type GlobalPinInfo struct { @@ -320,6 +384,19 @@ type PinInfoShort struct { PriorityPin bool `json:"priority_pin" codec:"y,omitempty"` } +// String provides a string representation of PinInfoShort. +func (pis PinInfoShort) String() string { + var b strings.Builder + fmt.Fprintf(&b, "status: %s\n", pis.Status) + fmt.Fprintf(&b, "peername: %s\n", pis.PeerName) + fmt.Fprintf(&b, "ipfs: %s\n", pis.IPFS) + fmt.Fprintf(&b, "ipfsAddresses: %v\n", pis.IPFSAddresses) + fmt.Fprintf(&b, "error: %s\n", pis.Error) + fmt.Fprintf(&b, "attemptCount: %d\n", pis.AttemptCount) + fmt.Fprintf(&b, "priority: %t\n", pis.PriorityPin) + return b.String() +} + // PinInfo holds information about local pins. This is used by the Pin // Trackers. type PinInfo struct { @@ -347,6 +424,17 @@ func (pi PinInfo) Defined() bool { return pi.Cid.Defined() } +// String provides a string representation of PinInfo. +func (pi PinInfo) String() string { + var b strings.Builder + fmt.Fprintf(&b, "cid: %s\n", pi.Cid) + fmt.Fprintf(&b, "name: %s\n", pi.Name) + fmt.Fprintf(&b, "peer: %s\n", pi.Peer) + fmt.Fprintf(&b, "allocations: %v\n", pi.Allocations) + fmt.Fprintf(&b, "%s\n", pi.PinInfoShort) + return b.String() +} + // Version holds version information type Version struct { Version string `json:"version" codec:"v"` @@ -571,6 +659,17 @@ func (pm PinMode) String() string { } } +// ToIPFSPinStatus converts a PinMode to IPFSPinStatus. +func (pm PinMode) ToIPFSPinStatus() IPFSPinStatus { + if pm == PinModeDirect { + return IPFSPinStatusDirect + } + if pm == PinModeRecursive { + return IPFSPinStatusRecursive + } + return IPFSPinStatusBug +} + // MarshalJSON converts the PinMode into a readable string in JSON. func (pm PinMode) MarshalJSON() ([]byte, error) { return json.Marshal(pm.String()) diff --git a/cluster.go b/cluster.go index 98b55035..0b335fdb 100644 --- a/cluster.go +++ b/cluster.go @@ -271,7 +271,16 @@ func (c *Cluster) watchPinset() { stateSyncTimer.Reset(c.config.StateSyncInterval) case <-recoverTimer.C: logger.Debug("auto-triggering RecoverAllLocal()") - c.RecoverAllLocal(ctx) + + out := make(chan api.PinInfo, 1024) + go func() { + for range out { + } + }() + err := c.RecoverAllLocal(ctx, out) + if err != nil { + logger.Error(err) + } recoverTimer.Reset(c.config.PinRecoverInterval) case <-c.ctx.Done(): if !stateSyncTimer.Stop() { @@ -436,6 +445,12 @@ func (c *Cluster) pushPingMetrics(ctx context.Context) { ticker := time.NewTicker(c.config.MonitorPingInterval) for { + select { + case <-ctx.Done(): + return + default: + } + c.sendPingMetric(ctx) select { @@ -507,11 +522,13 @@ func (c *Cluster) alertsHandler() { return } - pinCh, err := cState.List(c.ctx) - if err != nil { - logger.Warn(err) - return - } + pinCh := make(chan api.Pin, 1024) + go func() { + err = cState.List(c.ctx, pinCh) + if err != nil { + logger.Warn(err) + } + }() for pin := range pinCh { if containsPeer(pin.Allocations, alrt.Peer) && distance.isClosest(pin.Cid) { @@ -529,11 +546,17 @@ func (c *Cluster) watchPeers() { defer ticker.Stop() for { + select { + case <-c.ctx.Done(): + return + default: + } + select { case <-c.ctx.Done(): return case <-ticker.C: - // logger.Debugf("%s watching peers", c.id) + //logger.Debugf("%s watching peers", c.id) hasMe := false peers, err := c.consensus.Peers(c.ctx) if err != nil { @@ -594,11 +617,14 @@ func (c *Cluster) vacatePeer(ctx context.Context, p peer.ID) { logger.Warn(err) return } - pinCh, err := cState.List(ctx) - if err != nil { - logger.Warn(err) - return - } + + pinCh := make(chan api.Pin, 1024) + go func() { + err = cState.List(ctx, pinCh) + if err != nil { + logger.Warn(err) + } + }() for pin := range pinCh { if containsPeer(pin.Allocations, p) { @@ -1070,7 +1096,13 @@ func (c *Cluster) Join(ctx context.Context, addr ma.Multiaddr) error { } // Start pinning items in the state that are not on IPFS yet. - c.RecoverAllLocal(ctx) + out := make(chan api.PinInfo, 1024) + // discard outputs + go func() { + for range out { + } + }() + go c.RecoverAllLocal(ctx, out) logger.Infof("%s: joined %s's cluster", c.id.Pretty(), pid.Pretty()) return nil @@ -1100,6 +1132,8 @@ func (c *Cluster) distances(ctx context.Context, exclude peer.ID) (*distanceChec func (c *Cluster) StateSync(ctx context.Context) error { _, span := trace.StartSpan(ctx, "cluster/StateSync") defer span.End() + logger.Debug("StateSync") + ctx = trace.NewContext(c.ctx, span) if c.config.FollowerMode { @@ -1122,10 +1156,13 @@ func (c *Cluster) StateSync(ctx context.Context) error { return err // could not list peers } - clusterPins, err := cState.List(ctx) - if err != nil { - return err - } + clusterPins := make(chan api.Pin, 1024) + go func() { + err = cState.List(ctx, clusterPins) + if err != nil { + logger.Error(err) + } + }() // Unpin expired items when we are the closest peer to them. for p := range clusterPins { @@ -1140,24 +1177,29 @@ func (c *Cluster) StateSync(ctx context.Context) error { return nil } -// StatusAll returns the GlobalPinInfo for all tracked Cids in all peers. -// If an error happens, the slice will contain as much information as -// could be fetched from other peers. -func (c *Cluster) StatusAll(ctx context.Context, filter api.TrackerStatus) ([]api.GlobalPinInfo, error) { +// StatusAll returns the GlobalPinInfo for all tracked Cids in all peers on +// the out channel. This is done by broacasting a StatusAll to all peers. If +// an error happens, it is returned. This method blocks until it finishes. The +// operation can be aborted by cancelling the context. +func (c *Cluster) StatusAll(ctx context.Context, filter api.TrackerStatus, out chan<- api.GlobalPinInfo) error { _, span := trace.StartSpan(ctx, "cluster/StatusAll") defer span.End() ctx = trace.NewContext(c.ctx, span) - return c.globalPinInfoSlice(ctx, "PinTracker", "StatusAll", filter) + in := make(chan api.TrackerStatus, 1) + in <- filter + close(in) + return c.globalPinInfoStream(ctx, "PinTracker", "StatusAll", in, out) } -// StatusAllLocal returns the PinInfo for all the tracked Cids in this peer. -func (c *Cluster) StatusAllLocal(ctx context.Context, filter api.TrackerStatus) []api.PinInfo { +// StatusAllLocal returns the PinInfo for all the tracked Cids in this peer on +// the out channel. It blocks until finished. +func (c *Cluster) StatusAllLocal(ctx context.Context, filter api.TrackerStatus, out chan<- api.PinInfo) error { _, span := trace.StartSpan(ctx, "cluster/StatusAllLocal") defer span.End() ctx = trace.NewContext(c.ctx, span) - return c.tracker.StatusAll(ctx, filter) + return c.tracker.StatusAll(ctx, filter, out) } // Status returns the GlobalPinInfo for a given Cid as fetched from all @@ -1206,13 +1248,15 @@ func (c *Cluster) localPinInfoOp( return pInfo, err } -// RecoverAll triggers a RecoverAllLocal operation on all peers. -func (c *Cluster) RecoverAll(ctx context.Context) ([]api.GlobalPinInfo, error) { +// RecoverAll triggers a RecoverAllLocal operation on all peers and returns +// GlobalPinInfo objets for all recovered items. This method blocks until +// finished. Operation can be aborted by cancelling the context. +func (c *Cluster) RecoverAll(ctx context.Context, out chan<- api.GlobalPinInfo) error { _, span := trace.StartSpan(ctx, "cluster/RecoverAll") defer span.End() ctx = trace.NewContext(c.ctx, span) - return c.globalPinInfoSlice(ctx, "Cluster", "RecoverAllLocal", nil) + return c.globalPinInfoStream(ctx, "Cluster", "RecoverAllLocal", nil, out) } // RecoverAllLocal triggers a RecoverLocal operation for all Cids tracked @@ -1222,15 +1266,16 @@ func (c *Cluster) RecoverAll(ctx context.Context) ([]api.GlobalPinInfo, error) { // is faster than calling Pin on the same CID as it avoids committing an // identical pin to the consensus layer. // -// It returns the list of pins that were re-queued for pinning. +// It returns the list of pins that were re-queued for pinning on the out +// channel. It blocks until done. // // RecoverAllLocal is called automatically every PinRecoverInterval. -func (c *Cluster) RecoverAllLocal(ctx context.Context) ([]api.PinInfo, error) { +func (c *Cluster) RecoverAllLocal(ctx context.Context, out chan<- api.PinInfo) error { _, span := trace.StartSpan(ctx, "cluster/RecoverAllLocal") defer span.End() ctx = trace.NewContext(c.ctx, span) - return c.tracker.RecoverAll(ctx) + return c.tracker.RecoverAll(ctx, out) } // Recover triggers a recover operation for a given Cid in all @@ -1261,48 +1306,45 @@ func (c *Cluster) RecoverLocal(ctx context.Context, h cid.Cid) (api.PinInfo, err return c.localPinInfoOp(ctx, h, c.tracker.Recover) } -// PinsChannel returns a channel from which to read all the pins in the -// pinset, which are part of the current global state. This is the source of -// truth as to which pins are managed and their allocation, but does not -// indicate if the item is successfully pinned. For that, use the Status*() -// methods. +// Pins sends pins on the given out channel as it iterates the full +// pinset (current global state). This is the source of truth as to which pins +// are managed and their allocation, but does not indicate if the item is +// successfully pinned. For that, use the Status*() methods. // -// The channel can be aborted by cancelling the context. -func (c *Cluster) PinsChannel(ctx context.Context) (<-chan api.Pin, error) { - _, span := trace.StartSpan(ctx, "cluster/PinsChannel") +// The operation can be aborted by cancelling the context. This methods blocks +// until the operation has completed. +func (c *Cluster) Pins(ctx context.Context, out chan<- api.Pin) error { + _, span := trace.StartSpan(ctx, "cluster/Pins") defer span.End() ctx = trace.NewContext(c.ctx, span) cState, err := c.consensus.State(ctx) if err != nil { logger.Error(err) - return nil, err + return err } - return cState.List(ctx) + return cState.List(ctx, out) } -// Pins returns the list of Cids managed by Cluster and which are part +// pinsSlice returns the list of Cids managed by Cluster and which are part // of the current global state. This is the source of truth as to which // pins are managed and their allocation, but does not indicate if // the item is successfully pinned. For that, use StatusAll(). // // It is recommended to use PinsChannel(), as this method is equivalent to // loading the full pinset in memory! -func (c *Cluster) Pins(ctx context.Context) ([]api.Pin, error) { - _, span := trace.StartSpan(ctx, "cluster/Pins") - defer span.End() - ctx = trace.NewContext(c.ctx, span) - - ch, err := c.PinsChannel(ctx) - if err != nil { - return nil, err - } +func (c *Cluster) pinsSlice(ctx context.Context) ([]api.Pin, error) { + out := make(chan api.Pin, 1024) + var err error + go func() { + err = c.Pins(ctx, out) + }() var pins []api.Pin - for pin := range ch { + for pin := range out { pins = append(pins, pin) } - return pins, ctx.Err() + return pins, err } // PinGet returns information for a single Cid managed by Cluster. @@ -1751,14 +1793,12 @@ func (c *Cluster) peersWithFilter(ctx context.Context, peers []peer.ID) []api.ID if rpc.IsAuthorizationError(err) { continue } - ids[i] = api.ID{} ids[i].ID = peers[i] ids[i].Error = err.Error() } return ids - } // getTrustedPeers gives listed of trusted peers except the current peer and @@ -1935,15 +1975,18 @@ func (c *Cluster) globalPinInfoCid(ctx context.Context, comp, method string, h c return gpin, nil } -func (c *Cluster) globalPinInfoSlice(ctx context.Context, comp, method string, arg interface{}) ([]api.GlobalPinInfo, error) { - ctx, span := trace.StartSpan(ctx, "cluster/globalPinInfoSlice") +func (c *Cluster) globalPinInfoStream(ctx context.Context, comp, method string, inChan interface{}, out chan<- api.GlobalPinInfo) error { + defer close(out) + + ctx, span := trace.StartSpan(ctx, "cluster/globalPinInfoStream") defer span.End() - if arg == nil { - arg = struct{}{} + if inChan == nil { + emptyChan := make(chan struct{}) + close(emptyChan) + inChan = emptyChan } - infos := make([]api.GlobalPinInfo, 0) fullMap := make(map[cid.Cid]api.GlobalPinInfo) var members []peer.ID @@ -1954,27 +1997,31 @@ func (c *Cluster) globalPinInfoSlice(ctx context.Context, comp, method string, a members, err = c.consensus.Peers(ctx) if err != nil { logger.Error(err) - return nil, err + return err } } - lenMembers := len(members) - replies := make([][]api.PinInfo, lenMembers) + msOut := make(chan api.PinInfo) // We don't have a good timeout proposal for this. Depending on the // size of the state and the peformance of IPFS and the network, this // may take moderately long. - ctxs, cancels := rpcutil.CtxsWithCancel(ctx, lenMembers) - defer rpcutil.MultiCancel(cancels) + // If we did, this is the place to put it. + ctx, cancel := context.WithCancel(ctx) + defer cancel() - errs := c.rpcClient.MultiCall( - ctxs, - members, - comp, - method, - arg, - rpcutil.CopyPinInfoSliceToIfaces(replies), - ) + errsCh := make(chan []error, 1) + go func() { + defer close(errsCh) + errsCh <- c.rpcClient.MultiStream( + ctx, + members, + comp, + method, + inChan, + msOut, + ) + }() setPinInfo := func(p api.PinInfo) { if !p.Defined() { @@ -1989,20 +2036,25 @@ func (c *Cluster) globalPinInfoSlice(ctx context.Context, comp, method string, a fullMap[p.Cid] = info } + // make the big collection. + for pin := range msOut { + setPinInfo(pin) + } + + // This WAITs until MultiStream is DONE. erroredPeers := make(map[peer.ID]string) - for i, r := range replies { - if e := errs[i]; e != nil { // This error must come from not being able to contact that cluster member - if rpc.IsAuthorizationError(e) { - logger.Debug("rpc auth error", e) + errs, ok := <-errsCh + if ok { + for i, err := range errs { + if err == nil { continue } - logger.Errorf("%s: error in broadcast response from %s: %s ", c.id, members[i], e) - erroredPeers[members[i]] = e.Error() - continue - } - - for _, pin := range r { - setPinInfo(pin) + if rpc.IsAuthorizationError(err) { + logger.Debug("rpc auth error", err) + continue + } + logger.Errorf("%s: error in broadcast response from %s: %s ", c.id, members[i], err) + erroredPeers[members[i]] = err.Error() } } @@ -2031,10 +2083,16 @@ func (c *Cluster) globalPinInfoSlice(ctx context.Context, comp, method string, a } for _, v := range fullMap { - infos = append(infos, v) + select { + case <-ctx.Done(): + err := fmt.Errorf("%s.%s aborted: %w", comp, method, ctx.Err()) + logger.Error(err) + return err + case out <- v: + } } - return infos, nil + return nil } func (c *Cluster) getIDForPeer(ctx context.Context, pid peer.ID) (*api.ID, error) { diff --git a/cluster_test.go b/cluster_test.go index d9ac358d..3513aeba 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -64,17 +64,17 @@ func (ipfs *mockConnector) Pin(ctx context.Context, pin api.Pin) error { if pin.Cid == test.ErrorCid { return errors.New("trying to pin ErrorCid") } - ipfs.pins.Store(pin.Cid.String(), pin.MaxDepth) + ipfs.pins.Store(pin.Cid, pin.MaxDepth) return nil } func (ipfs *mockConnector) Unpin(ctx context.Context, c cid.Cid) error { - ipfs.pins.Delete(c.String()) + ipfs.pins.Delete(c) return nil } func (ipfs *mockConnector) PinLsCid(ctx context.Context, pin api.Pin) (api.IPFSPinStatus, error) { - dI, ok := ipfs.pins.Load(pin.Cid.String()) + dI, ok := ipfs.pins.Load(pin.Cid) if !ok { return api.IPFSPinStatusUnpinned, nil } @@ -85,8 +85,9 @@ func (ipfs *mockConnector) PinLsCid(ctx context.Context, pin api.Pin) (api.IPFSP return api.IPFSPinStatusRecursive, nil } -func (ipfs *mockConnector) PinLs(ctx context.Context, filter string) (map[string]api.IPFSPinStatus, error) { - m := make(map[string]api.IPFSPinStatus) +func (ipfs *mockConnector) PinLs(ctx context.Context, in []string, out chan<- api.IPFSPinInfo) error { + defer close(out) + var st api.IPFSPinStatus ipfs.pins.Range(func(k, v interface{}) bool { switch v.(api.PinDepth) { @@ -95,12 +96,13 @@ func (ipfs *mockConnector) PinLs(ctx context.Context, filter string) (map[string default: st = api.IPFSPinStatusRecursive } + c := k.(cid.Cid) - m[k.(string)] = st + out <- api.IPFSPinInfo{Cid: api.Cid(c), Type: st} return true }) - return m, nil + return nil } func (ipfs *mockConnector) SwarmPeers(ctx context.Context) ([]peer.ID, error) { @@ -795,7 +797,7 @@ func TestClusterPins(t *testing.T) { pinDelay() - pins, err := cl.Pins(ctx) + pins, err := cl.pinsSlice(ctx) if err != nil { t.Fatal(err) } @@ -942,10 +944,16 @@ func TestClusterRecoverAllLocal(t *testing.T) { pinDelay() - recov, err := cl.RecoverAllLocal(ctx) - if err != nil { - t.Error("did not expect an error") - } + out := make(chan api.PinInfo, 10) + go func() { + err := cl.RecoverAllLocal(ctx, out) + if err != nil { + t.Error("did not expect an error") + } + }() + + recov := collectPinInfos(t, out) + if len(recov) != 1 { t.Fatalf("there should be one pin recovered, got = %d", len(recov)) } diff --git a/cmd/ipfs-cluster-ctl/formatters.go b/cmd/ipfs-cluster-ctl/formatters.go index 2c7efe19..33309549 100644 --- a/cmd/ipfs-cluster-ctl/formatters.go +++ b/cmd/ipfs-cluster-ctl/formatters.go @@ -39,17 +39,23 @@ func jsonFormatObject(resp interface{}) { } func jsonFormatPrint(obj interface{}) { + print := func(o interface{}) { + j, err := json.MarshalIndent(o, "", " ") + checkErr("generating json output", err) + fmt.Printf("%s\n", j) + } + switch r := obj.(type) { case chan api.Pin: for o := range r { - j, err := json.MarshalIndent(o, "", " ") - checkErr("generating json output", err) - fmt.Printf("%s\n", j) + print(o) + } + case chan api.GlobalPinInfo: + for o := range r { + print(o) } default: - j, err := json.MarshalIndent(obj, "", " ") - checkErr("generating json output", err) - fmt.Printf("%s\n", j) + print(obj) } } @@ -82,8 +88,8 @@ func textFormatObject(resp interface{}) { for _, item := range r { textFormatObject(item) } - case []api.GlobalPinInfo: - for _, item := range r { + case chan api.GlobalPinInfo: + for item := range r { textFormatObject(item) } case chan api.Pin: diff --git a/cmd/ipfs-cluster-ctl/main.go b/cmd/ipfs-cluster-ctl/main.go index 039c540f..51ac80fb 100644 --- a/cmd/ipfs-cluster-ctl/main.go +++ b/cmd/ipfs-cluster-ctl/main.go @@ -888,21 +888,31 @@ separated list). The following are valid status values: checkErr("parsing cid", err) cids[i] = ci } - if len(cids) == 1 { - resp, cerr := globalClient.Status(ctx, cids[0], c.Bool("local")) - formatResponse(c, resp, cerr) - } else if len(cids) > 1 { - resp, cerr := globalClient.StatusCids(ctx, cids, c.Bool("local")) - formatResponse(c, resp, cerr) - } else { - filterFlag := c.String("filter") - filter := api.TrackerStatusFromString(c.String("filter")) - if filter == api.TrackerStatusUndefined && filterFlag != "" { - checkErr("parsing filter flag", errors.New("invalid filter name")) + out := make(chan api.GlobalPinInfo, 1024) + chErr := make(chan error, 1) + go func() { + defer close(chErr) + + if len(cids) == 1 { + resp, cerr := globalClient.Status(ctx, cids[0], c.Bool("local")) + out <- resp + chErr <- cerr + close(out) + } else if len(cids) > 1 { + chErr <- globalClient.StatusCids(ctx, cids, c.Bool("local"), out) + } else { + filterFlag := c.String("filter") + filter := api.TrackerStatusFromString(c.String("filter")) + if filter == api.TrackerStatusUndefined && filterFlag != "" { + checkErr("parsing filter flag", errors.New("invalid filter name")) + } + chErr <- globalClient.StatusAll(ctx, filter, c.Bool("local"), out) } - resp, cerr := globalClient.StatusAll(ctx, filter, c.Bool("local")) - formatResponse(c, resp, cerr) - } + }() + + formatResponse(c, out, nil) + err := <-chErr + formatResponse(c, nil, err) return nil }, }, @@ -932,8 +942,15 @@ operations on the contacted peer (as opposed to on every peer). resp, cerr := globalClient.Recover(ctx, ci, c.Bool("local")) formatResponse(c, resp, cerr) } else { - resp, cerr := globalClient.RecoverAll(ctx, c.Bool("local")) - formatResponse(c, resp, cerr) + out := make(chan api.GlobalPinInfo, 1024) + errCh := make(chan error, 1) + go func() { + defer close(errCh) + errCh <- globalClient.RecoverAll(ctx, c.Bool("local"), out) + }() + formatResponse(c, out, nil) + err := <-errCh + formatResponse(c, nil, err) } return nil }, diff --git a/cmd/ipfs-cluster-follow/commands.go b/cmd/ipfs-cluster-follow/commands.go index b3077114..6d1cf13c 100644 --- a/cmd/ipfs-cluster-follow/commands.go +++ b/cmd/ipfs-cluster-follow/commands.go @@ -493,14 +493,17 @@ func printStatusOnline(absPath, clusterName string) error { if err != nil { return cli.Exit(errors.Wrap(err, "error creating client"), 1) } - gpis, err := client.StatusAll(ctx, 0, true) - if err != nil { - return err - } - // do not return errors after this. + out := make(chan api.GlobalPinInfo, 1024) + errCh := make(chan error, 1) + + go func() { + defer close(errCh) + errCh <- client.StatusAll(ctx, 0, true, out) + }() + var pid string - for _, gpi := range gpis { + for gpi := range out { if pid == "" { // do this once // PeerMap will only have one key for k := range gpi.PeerMap { @@ -511,7 +514,8 @@ func printStatusOnline(absPath, clusterName string) error { pinInfo := gpi.PeerMap[pid] printPin(gpi.Cid, pinInfo.Status.String(), gpi.Name, pinInfo.Error) } - return nil + err = <-errCh + return err } func printStatusOffline(cfgHelper *cmdutils.ConfigHelper) error { @@ -528,14 +532,20 @@ func printStatusOffline(cfgHelper *cmdutils.ConfigHelper) error { if err != nil { return err } - pins, err := st.List(context.Background()) - if err != nil { - return err - } - for pin := range pins { + + out := make(chan api.Pin, 1024) + errCh := make(chan error, 1) + go func() { + defer close(errCh) + errCh <- st.List(context.Background(), out) + }() + + for pin := range out { printPin(pin.Cid, "offline", pin.Name, "") } - return nil + + err = <-errCh + return err } func printPin(c cid.Cid, status, name, err string) { diff --git a/cmdutils/state.go b/cmdutils/state.go index 8feb7a7d..cb68167a 100644 --- a/cmdutils/state.go +++ b/cmdutils/state.go @@ -222,16 +222,22 @@ func importState(r io.Reader, st state.State, opts api.PinOptions) error { // ExportState saves a json representation of a state func exportState(w io.Writer, st state.State) error { - pins, err := st.List(context.Background()) + out := make(chan api.Pin, 10000) + errCh := make(chan error, 1) + go func() { + defer close(errCh) + errCh <- st.List(context.Background(), out) + }() + var err error + enc := json.NewEncoder(w) + for pin := range out { + if err == nil { + err = enc.Encode(pin) + } + } if err != nil { return err } - enc := json.NewEncoder(w) - for pin := range pins { - err := enc.Encode(pin) - if err != nil { - return err - } - } - return nil + err = <-errCh + return err } diff --git a/consensus/crdt/consensus_test.go b/consensus/crdt/consensus_test.go index 24cf6b70..083f85c6 100644 --- a/consensus/crdt/consensus_test.go +++ b/consensus/crdt/consensus_test.go @@ -125,14 +125,14 @@ func TestConsensusPin(t *testing.T) { t.Fatal("error getting state:", err) } - ch, err := st.List(ctx) + out := make(chan api.Pin, 10) + err = st.List(ctx, out) if err != nil { t.Fatal(err) } var pins []api.Pin - - for p := range ch { + for p := range out { pins = append(pins, p) } @@ -186,14 +186,16 @@ func TestConsensusUpdate(t *testing.T) { t.Fatal("error getting state:", err) } - ch, err := st.List(ctx) + // Channel will not block sending because plenty of space + out := make(chan api.Pin, 100) + err = st.List(ctx, out) if err != nil { t.Fatal(err) } var pins []api.Pin - for p := range ch { + for p := range out { pins = append(pins, p) } @@ -243,14 +245,15 @@ func TestConsensusAddRmPeer(t *testing.T) { t.Fatal("error getting state:", err) } - ch, err := st.List(ctx) + out := make(chan api.Pin, 100) + err = st.List(ctx, out) if err != nil { t.Fatal(err) } var pins []api.Pin - for p := range ch { + for p := range out { pins = append(pins, p) } @@ -310,14 +313,15 @@ func TestConsensusDistrustPeer(t *testing.T) { t.Fatal("error getting state:", err) } - ch, err := st.List(ctx) + out := make(chan api.Pin, 10) + err = st.List(ctx, out) if err != nil { t.Fatal(err) } var pins []api.Pin - for p := range ch { + for p := range out { pins = append(pins, p) } @@ -372,14 +376,15 @@ func TestOfflineState(t *testing.T) { t.Fatal(err) } - ch, err := offlineState.List(ctx) + out := make(chan api.Pin, 100) + err = offlineState.List(ctx, out) if err != nil { t.Fatal(err) } var pins []api.Pin - for p := range ch { + for p := range out { pins = append(pins, p) } @@ -412,14 +417,15 @@ func TestBatching(t *testing.T) { time.Sleep(250 * time.Millisecond) - ch, err := st.List(ctx) + out := make(chan api.Pin, 100) + err = st.List(ctx, out) if err != nil { t.Fatal(err) } var pins []api.Pin - for p := range ch { + for p := range out { pins = append(pins, p) } @@ -430,14 +436,15 @@ func TestBatching(t *testing.T) { // Trigger batch auto-commit by time time.Sleep(time.Second) - ch, err = st.List(ctx) + out = make(chan api.Pin, 100) + err = st.List(ctx, out) if err != nil { t.Fatal(err) } pins = nil - for p := range ch { + for p := range out { pins = append(pins, p) } @@ -456,13 +463,14 @@ func TestBatching(t *testing.T) { // Give a chance for things to persist time.Sleep(250 * time.Millisecond) - ch, err = st.List(ctx) + out = make(chan api.Pin, 100) + err = st.List(ctx, out) if err != nil { t.Fatal(err) } pins = nil - for p := range ch { + for p := range out { pins = append(pins, p) } @@ -472,12 +480,14 @@ func TestBatching(t *testing.T) { // wait for the last pin time.Sleep(time.Second) - ch, err = st.List(ctx) + + out = make(chan api.Pin, 100) + err = st.List(ctx, out) if err != nil { t.Fatal(err) } pins = nil - for p := range ch { + for p := range out { pins = append(pins, p) } diff --git a/consensus/raft/consensus_test.go b/consensus/raft/consensus_test.go index 24df8a2f..d158901d 100644 --- a/consensus/raft/consensus_test.go +++ b/consensus/raft/consensus_test.go @@ -99,13 +99,14 @@ func TestConsensusPin(t *testing.T) { t.Fatal("error getting state:", err) } - ch, err := st.List(ctx) + out := make(chan api.Pin, 10) + err = st.List(ctx, out) if err != nil { t.Fatal(err) } var pins []api.Pin - for p := range ch { + for p := range out { pins = append(pins, p) } @@ -154,13 +155,14 @@ func TestConsensusUpdate(t *testing.T) { t.Fatal("error getting state:", err) } - ch, err := st.List(ctx) + out := make(chan api.Pin, 10) + err = st.List(ctx, out) if err != nil { t.Fatal(err) } var pins []api.Pin - for p := range ch { + for p := range out { pins = append(pins, p) } @@ -330,13 +332,15 @@ func TestRaftLatestSnapshot(t *testing.T) { if err != nil { t.Fatal("Snapshot bytes returned could not restore to state: ", err) } - ch, err := snapState.List(ctx) + + out := make(chan api.Pin, 100) + err = snapState.List(ctx, out) if err != nil { t.Fatal(err) } var pins []api.Pin - for p := range ch { + for p := range out { pins = append(pins, p) } diff --git a/consensus/raft/log_op_test.go b/consensus/raft/log_op_test.go index 3e51df84..94deceed 100644 --- a/consensus/raft/log_op_test.go +++ b/consensus/raft/log_op_test.go @@ -27,13 +27,14 @@ func TestApplyToPin(t *testing.T) { } op.ApplyTo(st) - ch, err := st.List(ctx) + out := make(chan api.Pin, 100) + err = st.List(ctx, out) if err != nil { t.Fatal(err) } var pins []api.Pin - for p := range ch { + for p := range out { pins = append(pins, p) } @@ -59,11 +60,13 @@ func TestApplyToUnpin(t *testing.T) { } st.Add(ctx, testPin(test.Cid1)) op.ApplyTo(st) - pins, err := st.List(ctx) + + out := make(chan api.Pin, 100) + err = st.List(ctx, out) if err != nil { t.Fatal(err) } - if len(pins) != 0 { + if len(out) != 0 { t.Error("the state was not modified correctly") } } diff --git a/go.mod b/go.mod index f8ec3ce2..410fbaa1 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/ipfs/go-cid v0.1.0 github.com/ipfs/go-datastore v0.5.1 github.com/ipfs/go-ds-badger v0.3.0 - github.com/ipfs/go-ds-crdt v0.3.3 + github.com/ipfs/go-ds-crdt v0.3.4 github.com/ipfs/go-ds-leveldb v0.5.0 github.com/ipfs/go-fs-lock v0.0.7 github.com/ipfs/go-ipfs-api v0.3.0 @@ -30,10 +30,10 @@ require ( github.com/ipfs/go-ipfs-pinner v0.2.1 github.com/ipfs/go-ipfs-posinfo v0.0.1 github.com/ipfs/go-ipld-cbor v0.0.6 - github.com/ipfs/go-ipld-format v0.2.0 + github.com/ipfs/go-ipld-format v0.3.0 github.com/ipfs/go-ipns v0.1.2 github.com/ipfs/go-log/v2 v2.5.0 - github.com/ipfs/go-merkledag v0.5.1 + github.com/ipfs/go-merkledag v0.6.0 github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164 github.com/ipfs/go-path v0.2.2 github.com/ipfs/go-unixfs v0.3.1 @@ -45,7 +45,7 @@ require ( github.com/libp2p/go-libp2p-connmgr v0.3.1 github.com/libp2p/go-libp2p-consensus v0.0.1 github.com/libp2p/go-libp2p-core v0.13.0 - github.com/libp2p/go-libp2p-gorpc v0.3.0 + github.com/libp2p/go-libp2p-gorpc v0.3.1 github.com/libp2p/go-libp2p-gostream v0.3.1 github.com/libp2p/go-libp2p-http v0.2.1 github.com/libp2p/go-libp2p-kad-dht v0.15.0 @@ -119,14 +119,14 @@ require ( github.com/huin/goupnp v1.0.2 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.0.0 // indirect - github.com/ipfs/go-bitswap v0.5.1 // indirect - github.com/ipfs/go-blockservice v0.2.1 // indirect + github.com/ipfs/go-bitswap v0.6.0 // indirect + github.com/ipfs/go-blockservice v0.3.0 // indirect github.com/ipfs/go-cidutil v0.0.2 // indirect github.com/ipfs/go-fetcher v1.6.1 // indirect - github.com/ipfs/go-ipfs-blockstore v1.1.2 // indirect + github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-exchange-interface v0.1.0 // indirect - github.com/ipfs/go-ipfs-exchange-offline v0.1.1 // indirect + github.com/ipfs/go-ipfs-exchange-offline v0.2.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.2 // indirect github.com/ipfs/go-ipfs-provider v0.7.1 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect diff --git a/go.sum b/go.sum index 9389ffee..ae506355 100644 --- a/go.sum +++ b/go.sum @@ -424,8 +424,9 @@ github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiL github.com/ipfs/go-bitswap v0.1.3/go.mod h1:YEQlFy0kkxops5Vy+OxWdRSEZIoS7I7KDIwoa5Chkps= github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM= github.com/ipfs/go-bitswap v0.3.4/go.mod h1:4T7fvNv/LmOys+21tnLzGKncMeeXUYUd1nUiJ2teMvI= -github.com/ipfs/go-bitswap v0.5.1 h1:721YAEDBnLIrvcIMkCHCdqp34hA8jwL9yKMkyJpSpco= github.com/ipfs/go-bitswap v0.5.1/go.mod h1:P+ckC87ri1xFLvk74NlXdP0Kj9RmWAh4+H78sC6Qopo= +github.com/ipfs/go-bitswap v0.6.0 h1:f2rc6GZtoSFhEIzQmddgGiel9xntj02Dg0ZNf2hSC+w= +github.com/ipfs/go-bitswap v0.6.0/go.mod h1:Hj3ZXdOC5wBJvENtdqsixmzzRukqd8EHLxZLZc3mzRA= github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.0.3 h1:r8t66QstRp/pd/or4dpnbVfXT5Gt7lOqRvC+/dDTpMc= @@ -434,8 +435,9 @@ github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbR github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.1/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= github.com/ipfs/go-blockservice v0.1.4/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU= -github.com/ipfs/go-blockservice v0.2.1 h1:NJ4j/cwEfIg60rzAWcCIxRtOwbf6ZPK49MewNxObCPQ= github.com/ipfs/go-blockservice v0.2.1/go.mod h1:k6SiwmgyYgs4M/qt+ww6amPeUH9EISLRBnvUurKJhi8= +github.com/ipfs/go-blockservice v0.3.0 h1:cDgcZ+0P0Ih3sl8+qjFr2sVaMdysg/YZpLj5WJ8kiiw= +github.com/ipfs/go-blockservice v0.3.0/go.mod h1:P5ppi8IHDC7O+pA0AlGTF09jruB2h+oP3wVVaZl8sfk= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= @@ -469,8 +471,8 @@ github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBR github.com/ipfs/go-ds-badger v0.2.7/go.mod h1:02rnztVKA4aZwDuaRPTf8mpqcKmXP7mLl6JPxd14JHA= github.com/ipfs/go-ds-badger v0.3.0 h1:xREL3V0EH9S219kFFueOYJJTcjgNSZ2HY1iSvN7U1Ro= github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek= -github.com/ipfs/go-ds-crdt v0.3.3 h1:Q7fj+bm/gCfHte3axLQuCEzK1Uhsxgf065WLRvfeb0w= -github.com/ipfs/go-ds-crdt v0.3.3/go.mod h1:rcfJixHEd+hIWcu/8SecC/lVlNcAkhE6DNgRKPd1xgU= +github.com/ipfs/go-ds-crdt v0.3.4 h1:O/dFBkxxXxNO9cjfQwFQHTsoehfJtV1GNAhuRmLh2Dg= +github.com/ipfs/go-ds-crdt v0.3.4/go.mod h1:bFHBkP56kWufO55QxAKT7qZqz23thrh7FN5l+hYTHa4= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= @@ -488,8 +490,9 @@ github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE= -github.com/ipfs/go-ipfs-blockstore v1.1.2 h1:WCXoZcMYnvOTmlpX+RSSnhVN0uCmbWTeepTGX5lgiXw= github.com/ipfs/go-ipfs-blockstore v1.1.2/go.mod h1:w51tNR9y5+QXB0wkNcHt4O2aSZjTdqaEWaQdSxEyUOY= +github.com/ipfs/go-ipfs-blockstore v1.2.0 h1:n3WTeJ4LdICWs/0VSfjHrlqpPpl6MZ+ySd3j8qz0ykw= +github.com/ipfs/go-ipfs-blockstore v1.2.0/go.mod h1:eh8eTFLiINYNSNawfZOC7HOxNTxpB1PFuA5E1m/7exE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= @@ -510,8 +513,9 @@ github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFq github.com/ipfs/go-ipfs-exchange-interface v0.1.0 h1:TiMekCrOGQuWYtZO3mf4YJXDIdNgnKWZ9IE3fGlnWfo= github.com/ipfs/go-ipfs-exchange-interface v0.1.0/go.mod h1:ych7WPlyHqFvCi/uQI48zLZuAWVP5iTQPXEfVaw5WEI= github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0= -github.com/ipfs/go-ipfs-exchange-offline v0.1.1 h1:mEiXWdbMN6C7vtDG21Fphx8TGCbZPpQnz/496w/PL4g= github.com/ipfs/go-ipfs-exchange-offline v0.1.1/go.mod h1:vTiBRIbzSwDD0OWm+i3xeT0mO7jG2cbJYatp3HPk5XY= +github.com/ipfs/go-ipfs-exchange-offline v0.2.0 h1:2PF4o4A7W656rC0RxuhUace997FTcDTcIQ6NoEtyjAI= +github.com/ipfs/go-ipfs-exchange-offline v0.2.0/go.mod h1:HjwBeW0dvZvfOMwDP0TSKXIHf2s+ksdP4E3MLDRtLKY= github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs= github.com/ipfs/go-ipfs-files v0.0.9/go.mod h1:aFv2uQ/qxWpL/6lidWvnSQmaVqCrf0TBGoUr+C1Fo84= @@ -541,8 +545,9 @@ github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eB github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= -github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA= github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= +github.com/ipfs/go-ipld-format v0.3.0 h1:Mwm2oRLzIuUwEPewWAWyMuuBQUsn3awfFEYVb8akMOQ= +github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= github.com/ipfs/go-ipld-legacy v0.1.0 h1:wxkkc4k8cnvIGIjPO0waJCe7SHEyFgl+yQdafdjGrpA= github.com/ipfs/go-ipld-legacy v0.1.0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= github.com/ipfs/go-ipns v0.1.2 h1:O/s/0ht+4Jl9+VoxoUo0zaHjnZUS+aBQIKTuzdZ/ucI= @@ -565,8 +570,9 @@ github.com/ipfs/go-log/v2 v2.5.0/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOL github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.3.2/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M= -github.com/ipfs/go-merkledag v0.5.1 h1:tr17GPP5XtPhvPPiWtu20tSGZiZDuTaJRXBLcr79Umk= github.com/ipfs/go-merkledag v0.5.1/go.mod h1:cLMZXx8J08idkp5+id62iVftUQV+HlYJ3PIhDfZsjA4= +github.com/ipfs/go-merkledag v0.6.0 h1:oV5WT2321tS4YQVOPgIrWHvJ0lJobRTerU+i9nmUCuA= +github.com/ipfs/go-merkledag v0.6.0/go.mod h1:9HSEwRd5sV+lbykiYP+2NC/3o6MZbKNaa4hfNcH5iH0= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164 h1:0ATu9s5KktHhm8aYRSe1ysOJPik3dRwU/uag1Bcz+tg= @@ -774,8 +780,8 @@ github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKB github.com/libp2p/go-libp2p-discovery v0.6.0 h1:1XdPmhMJr8Tmj/yUfkJMIi8mgwWrLUsCB3bMxdT+DSo= github.com/libp2p/go-libp2p-discovery v0.6.0/go.mod h1:/u1voHt0tKIe5oIA1RHBKQLVCWPna2dXmPNHc2zR9S8= github.com/libp2p/go-libp2p-gorpc v0.1.0/go.mod h1:DrswTLnu7qjLgbqe4fekX4ISoPiHUqtA45thTsJdE1w= -github.com/libp2p/go-libp2p-gorpc v0.3.0 h1:1ww39zPEclHh8p1Exk882Xhy3CK2gW+JZYd+6NZp+q0= -github.com/libp2p/go-libp2p-gorpc v0.3.0/go.mod h1:sRz9ybP9rlOkJB1v65SMLr+NUEPB/ioLZn26MWIV4DU= +github.com/libp2p/go-libp2p-gorpc v0.3.1 h1:ZmqQIgHccgh/Ff1kS3ZlwATZRLvtuRUd633/MLWAx20= +github.com/libp2p/go-libp2p-gorpc v0.3.1/go.mod h1:sRz9ybP9rlOkJB1v65SMLr+NUEPB/ioLZn26MWIV4DU= github.com/libp2p/go-libp2p-gostream v0.3.0/go.mod h1:pLBQu8db7vBMNINGsAwLL/ZCE8wng5V1FThoaE5rNjc= github.com/libp2p/go-libp2p-gostream v0.3.1 h1:XlwohsPn6uopGluEWs1Csv1QCEjrTXf2ZQagzZ5paAg= github.com/libp2p/go-libp2p-gostream v0.3.1/go.mod h1:1V3b+u4Zhaq407UUY9JLCpboaeufAeVQbnvAt12LRsI= diff --git a/informer/numpin/numpin.go b/informer/numpin/numpin.go index 22ed55ad..becddcb8 100644 --- a/informer/numpin/numpin.go +++ b/informer/numpin/numpin.go @@ -5,6 +5,7 @@ package numpin import ( "context" "fmt" + "sync" "github.com/ipfs/ipfs-cluster/api" @@ -19,7 +20,9 @@ var MetricName = "numpin" // Informer is a simple object to implement the ipfscluster.Informer // and Component interfaces type Informer struct { - config *Config + config *Config + + mu sync.Mutex rpcClient *rpc.Client } @@ -38,7 +41,9 @@ func NewInformer(cfg *Config) (*Informer, error) { // SetClient provides us with an rpc.Client which allows // contacting other components in the cluster. func (npi *Informer) SetClient(c *rpc.Client) { + npi.mu.Lock() npi.rpcClient = c + npi.mu.Unlock() } // Shutdown is called on cluster shutdown. We just invalidate @@ -47,7 +52,9 @@ func (npi *Informer) Shutdown(ctx context.Context) error { _, span := trace.StartSpan(ctx, "informer/numpin/Shutdown") defer span.End() + npi.mu.Lock() npi.rpcClient = nil + npi.mu.Unlock() return nil } @@ -63,7 +70,11 @@ func (npi *Informer) GetMetrics(ctx context.Context) []api.Metric { ctx, span := trace.StartSpan(ctx, "informer/numpin/GetMetric") defer span.End() - if npi.rpcClient == nil { + npi.mu.Lock() + rpcClient := npi.rpcClient + npi.mu.Unlock() + + if rpcClient == nil { return []api.Metric{ { Valid: false, @@ -71,24 +82,39 @@ func (npi *Informer) GetMetrics(ctx context.Context) []api.Metric { } } - pinMap := make(map[string]api.IPFSPinStatus) - // make use of the RPC API to obtain information // about the number of pins in IPFS. See RPCAPI docs. - err := npi.rpcClient.CallContext( - ctx, - "", // Local call - "IPFSConnector", // Service name - "PinLs", // Method name - "recursive", // in arg - &pinMap, // out arg - ) + in := make(chan []string, 1) + in <- []string{"recursive", "direct"} + close(in) + out := make(chan api.IPFSPinInfo, 1024) + + errCh := make(chan error, 1) + go func() { + defer close(errCh) + err := rpcClient.Stream( + ctx, + "", // Local call + "IPFSConnector", // Service name + "PinLs", // Method name + in, + out, + ) + errCh <- err + }() + + n := 0 + for range out { + n++ + } + + err := <-errCh valid := err == nil m := api.Metric{ Name: MetricName, - Value: fmt.Sprintf("%d", len(pinMap)), + Value: fmt.Sprintf("%d", n), Valid: valid, Partitionable: false, } diff --git a/informer/numpin/numpin_test.go b/informer/numpin/numpin_test.go index 63e7e950..de18330a 100644 --- a/informer/numpin/numpin_test.go +++ b/informer/numpin/numpin_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/ipfs/ipfs-cluster/api" + "github.com/ipfs/ipfs-cluster/test" rpc "github.com/libp2p/go-libp2p-gorpc" ) @@ -21,11 +22,10 @@ func mockRPCClient(t *testing.T) *rpc.Client { return c } -func (mock *mockService) PinLs(ctx context.Context, in string, out *map[string]api.IPFSPinStatus) error { - *out = map[string]api.IPFSPinStatus{ - "QmPGDFvBkgWhvzEK9qaTWrWurSwqXNmhnK3hgELPdZZNPa": api.IPFSPinStatusRecursive, - "QmUZ13osndQ5uL4tPWHXe3iBgBgq9gfewcBMSCAuMBsDJ6": api.IPFSPinStatusRecursive, - } +func (mock *mockService) PinLs(ctx context.Context, in <-chan []string, out chan<- api.IPFSPinInfo) error { + out <- api.IPFSPinInfo{Cid: api.Cid(test.Cid1), Type: api.IPFSPinStatusRecursive} + out <- api.IPFSPinInfo{Cid: api.Cid(test.Cid2), Type: api.IPFSPinStatusRecursive} + close(out) return nil } diff --git a/ipfscluster.go b/ipfscluster.go index 7dd4eb55..b23fd5be 100644 --- a/ipfscluster.go +++ b/ipfscluster.go @@ -78,7 +78,8 @@ type IPFSConnector interface { Pin(context.Context, api.Pin) error Unpin(context.Context, cid.Cid) error PinLsCid(context.Context, api.Pin) (api.IPFSPinStatus, error) - PinLs(ctx context.Context, typeFilter string) (map[string]api.IPFSPinStatus, error) + // PinLs returns pins in the pinset of the given types (recursive, direct...) + PinLs(ctx context.Context, typeFilters []string, out chan<- api.IPFSPinInfo) error // ConnectSwarms make sure this peer's IPFS daemon is connected to // other peers IPFS daemons. ConnectSwarms(context.Context) error @@ -121,12 +122,11 @@ type PinTracker interface { Untrack(context.Context, cid.Cid) error // StatusAll returns the list of pins with their local status. Takes a // filter to specify which statuses to report. - StatusAll(context.Context, api.TrackerStatus) []api.PinInfo + StatusAll(context.Context, api.TrackerStatus, chan<- api.PinInfo) error // Status returns the local status of a given Cid. Status(context.Context, cid.Cid) api.PinInfo - // RecoverAll calls Recover() for all pins tracked. Returns only - // informations for retriggered pins. - RecoverAll(context.Context) ([]api.PinInfo, error) + // RecoverAll calls Recover() for all pins tracked. + RecoverAll(context.Context, chan<- api.PinInfo) error // Recover retriggers a Pin/Unpin operation in a Cids with error status. Recover(context.Context, cid.Cid) (api.PinInfo, error) } diff --git a/ipfscluster_test.go b/ipfscluster_test.go index 3d77a864..7a26d1da 100644 --- a/ipfscluster_test.go +++ b/ipfscluster_test.go @@ -430,6 +430,48 @@ func shutdownCluster(t *testing.T, c *Cluster, m *test.IpfsMock) { m.Close() } +func collectGlobalPinInfos(t *testing.T, out <-chan api.GlobalPinInfo, timeout time.Duration) []api.GlobalPinInfo { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + var gpis []api.GlobalPinInfo + for { + select { + case <-ctx.Done(): + t.Error(ctx.Err()) + return gpis + case gpi, ok := <-out: + if !ok { + return gpis + } + gpis = append(gpis, gpi) + } + } +} + +func collectPinInfos(t *testing.T, out <-chan api.PinInfo) []api.PinInfo { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var pis []api.PinInfo + for { + select { + case <-ctx.Done(): + t.Error(ctx.Err()) + return pis + case pi, ok := <-out: + if !ok { + return pis + } + pis = append(pis, pi) + } + } +} + func runF(t *testing.T, clusters []*Cluster, f func(*testing.T, *Cluster)) { t.Helper() var wg sync.WaitGroup @@ -654,12 +696,22 @@ func TestClustersPin(t *testing.T) { } switch consensus { case "crdt": - time.Sleep(20 * time.Second) + time.Sleep(10 * time.Second) default: delay() } fpinned := func(t *testing.T, c *Cluster) { - status := c.tracker.StatusAll(ctx, api.TrackerStatusUndefined) + out := make(chan api.PinInfo, 10) + + go func() { + err := c.tracker.StatusAll(ctx, api.TrackerStatusUndefined, out) + if err != nil { + t.Error(err) + } + }() + + status := collectPinInfos(t, out) + for _, v := range status { if v.Status != api.TrackerStatusPinned { t.Errorf("%s should have been pinned but it is %s", v.Cid, v.Status) @@ -672,7 +724,7 @@ func TestClustersPin(t *testing.T) { runF(t, clusters, fpinned) // Unpin everything - pinList, err := clusters[0].Pins(ctx) + pinList, err := clusters[0].pinsSlice(ctx) if err != nil { t.Fatal(err) } @@ -692,7 +744,7 @@ func TestClustersPin(t *testing.T) { switch consensus { case "crdt": - time.Sleep(20 * time.Second) + time.Sleep(10 * time.Second) default: delay() } @@ -708,7 +760,15 @@ func TestClustersPin(t *testing.T) { delay() funpinned := func(t *testing.T, c *Cluster) { - status := c.tracker.StatusAll(ctx, api.TrackerStatusUndefined) + out := make(chan api.PinInfo) + go func() { + err := c.tracker.StatusAll(ctx, api.TrackerStatusUndefined, out) + if err != nil { + t.Error(err) + } + }() + + status := collectPinInfos(t, out) for _, v := range status { t.Errorf("%s should have been unpinned but it is %s", v.Cid, v.Status) } @@ -852,10 +912,15 @@ func TestClustersStatusAll(t *testing.T) { pinDelay() // Global status f := func(t *testing.T, c *Cluster) { - statuses, err := c.StatusAll(ctx, api.TrackerStatusUndefined) - if err != nil { - t.Error(err) - } + out := make(chan api.GlobalPinInfo, 10) + go func() { + err := c.StatusAll(ctx, api.TrackerStatusUndefined, out) + if err != nil { + t.Error(err) + } + }() + + statuses := collectGlobalPinInfos(t, out, 5*time.Second) if len(statuses) != 1 { t.Fatal("bad status. Expected one item") } @@ -920,10 +985,16 @@ func TestClustersStatusAllWithErrors(t *testing.T) { return } - statuses, err := c.StatusAll(ctx, api.TrackerStatusUndefined) - if err != nil { - t.Error(err) - } + out := make(chan api.GlobalPinInfo, 10) + go func() { + err := c.StatusAll(ctx, api.TrackerStatusUndefined, out) + if err != nil { + t.Error(err) + } + }() + + statuses := collectGlobalPinInfos(t, out, 5*time.Second) + if len(statuses) != 1 { t.Fatal("bad status. Expected one item") } @@ -1124,11 +1195,15 @@ func TestClustersRecoverAll(t *testing.T) { pinDelay() - gInfos, err := clusters[rand.Intn(nClusters)].RecoverAll(ctx) - if err != nil { - t.Fatal(err) - } - delay() + out := make(chan api.GlobalPinInfo) + go func() { + err := clusters[rand.Intn(nClusters)].RecoverAll(ctx, out) + if err != nil { + t.Error(err) + } + }() + + gInfos := collectGlobalPinInfos(t, out, 5*time.Second) if len(gInfos) != 1 { t.Error("expected one items") @@ -1219,7 +1294,15 @@ func TestClustersReplicationOverall(t *testing.T) { f := func(t *testing.T, c *Cluster) { // confirm that the pintracker state matches the current global state - pinfos := c.tracker.StatusAll(ctx, api.TrackerStatusUndefined) + out := make(chan api.PinInfo, 100) + + go func() { + err := c.tracker.StatusAll(ctx, api.TrackerStatusUndefined, out) + if err != nil { + t.Error(err) + } + }() + pinfos := collectPinInfos(t, out) if len(pinfos) != nClusters { t.Error("Pinfos does not have the expected pins") } @@ -1243,11 +1326,14 @@ func TestClustersReplicationOverall(t *testing.T) { t.Errorf("%s: Expected 1 remote pin but got %d", c.id.String(), numRemote) } - pins, err := c.Pins(ctx) - if err != nil { - t.Fatal(err) - } - for _, pin := range pins { + outPins := make(chan api.Pin) + go func() { + err := c.Pins(ctx, outPins) + if err != nil { + t.Error(err) + } + }() + for pin := range outPins { allocs := pin.Allocations if len(allocs) != nClusters-1 { t.Errorf("Allocations are [%s]", allocs) @@ -1623,7 +1709,7 @@ func TestClustersReplicationRealloc(t *testing.T) { // Let the pin arrive pinDelay() - pinList, err := clusters[j].Pins(ctx) + pinList, err := clusters[j].pinsSlice(ctx) if err != nil { t.Fatal(err) } @@ -1641,7 +1727,7 @@ func TestClustersReplicationRealloc(t *testing.T) { pinDelay() - pinList2, err := clusters[j].Pins(ctx) + pinList2, err := clusters[j].pinsSlice(ctx) if err != nil { t.Fatal(err) } @@ -2131,7 +2217,7 @@ func TestClusterPinsWithExpiration(t *testing.T) { pinDelay() - pins, err := cl.Pins(ctx) + pins, err := cl.pinsSlice(ctx) if err != nil { t.Fatal(err) } @@ -2154,7 +2240,7 @@ func TestClusterPinsWithExpiration(t *testing.T) { pinDelay() // state sync should have unpinned expired pin - pins, err = cl.Pins(ctx) + pins, err = cl.pinsSlice(ctx) if err != nil { t.Fatal(err) } diff --git a/ipfsconn/ipfshttp/ipfshttp.go b/ipfsconn/ipfshttp/ipfshttp.go index 0b361d1a..4e0fb5b8 100644 --- a/ipfsconn/ipfshttp/ipfshttp.go +++ b/ipfsconn/ipfshttp/ipfshttp.go @@ -73,19 +73,25 @@ type ipfsError struct { func (ie ipfsError) Error() string { return fmt.Sprintf( - "IPFS request unsuccessful (%s). Code: %d. Message: %s", + "IPFS error (%s). Code: %d. Message: %s", ie.path, ie.code, ie.Message, ) } -type ipfsPinType struct { - Type string +type ipfsUnpinnedError ipfsError + +func (unpinned ipfsUnpinnedError) Is(target error) bool { + ierr, ok := target.(ipfsError) + if !ok { + return false + } + return strings.HasSuffix(ierr.Message, "not pinned") } -type ipfsPinLsResp struct { - Keys map[string]ipfsPinType +func (unpinned ipfsUnpinnedError) Error() string { + return ipfsError(unpinned).Error() } type ipfsIDResp struct { @@ -493,33 +499,62 @@ func (ipfs *Connector) Unpin(ctx context.Context, hash cid.Cid) error { } // PinLs performs a "pin ls --type typeFilter" request against the configured -// IPFS daemon and returns a map of cid strings and their status. -func (ipfs *Connector) PinLs(ctx context.Context, typeFilter string) (map[string]api.IPFSPinStatus, error) { +// IPFS daemon and sends the results on the given channel. Returns when done. +func (ipfs *Connector) PinLs(ctx context.Context, typeFilters []string, out chan<- api.IPFSPinInfo) error { + defer close(out) + bodies := make([]io.ReadCloser, len(typeFilters)) + ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/PinLs") defer span.End() ctx, cancel := context.WithTimeout(ctx, ipfs.config.IPFSRequestTimeout) defer cancel() - body, err := ipfs.postCtx(ctx, "pin/ls?type="+typeFilter, "", nil) - // Some error talking to the daemon - if err != nil { - return nil, err + var err error + +nextFilter: + for i, typeFilter := range typeFilters { + // Post and read streaming response + path := "pin/ls?stream=true&type=" + typeFilter + bodies[i], err = ipfs.postCtxStreamResponse(ctx, path, "", nil) + if err != nil { + logger.Error("error querying pinset: %s", err) + return err + } + defer bodies[i].Close() + + dec := json.NewDecoder(bodies[i]) + + for { + select { + case <-ctx.Done(): + err = fmt.Errorf("aborting pin/ls operation: %w", ctx.Err()) + logger.Error(err) + return err + default: + } + + var ipfsPin api.IPFSPinInfo + err = dec.Decode(&ipfsPin) + if err == io.EOF { + break nextFilter + } + if err != nil { + err = fmt.Errorf("error decoding ipfs pin: %w", err) + return err + } + + select { + case <-ctx.Done(): + err = fmt.Errorf("aborting pin/ls operation: %w", ctx.Err()) + logger.Error(err) + return err + case out <- ipfsPin: + } + } } - var res ipfsPinLsResp - err = json.Unmarshal(body, &res) - if err != nil { - logger.Error("parsing pin/ls response") - logger.Error(string(body)) - return nil, err - } - - statusMap := make(map[string]api.IPFSPinStatus) - for k, v := range res.Keys { - statusMap[k] = api.IPFSPinStatusFromString(v.Type) - } - return statusMap, nil + return nil } // PinLsCid performs a "pin ls " request. It will use "type=recursive" or @@ -532,35 +567,31 @@ func (ipfs *Connector) PinLsCid(ctx context.Context, pin api.Pin) (api.IPFSPinSt ctx, cancel := context.WithTimeout(ctx, ipfs.config.IPFSRequestTimeout) defer cancel() + if !pin.Defined() { + return api.IPFSPinStatusBug, errors.New("calling PinLsCid without a defined CID") + } + pinType := pin.MaxDepth.ToPinMode().String() - lsPath := fmt.Sprintf("pin/ls?arg=%s&type=%s", pin.Cid, pinType) - body, err := ipfs.postCtx(ctx, lsPath, "", nil) - if body == nil && err != nil { // Network error, daemon down - return api.IPFSPinStatusError, err - } - - if err != nil { // we could not find the pin - return api.IPFSPinStatusUnpinned, nil - } - - var res ipfsPinLsResp - err = json.Unmarshal(body, &res) + lsPath := fmt.Sprintf("pin/ls?stream=true&arg=%s&type=%s", pin.Cid, pinType) + body, err := ipfs.postCtxStreamResponse(ctx, lsPath, "", nil) if err != nil { - logger.Error("error parsing pin/ls?arg=cid response:") - logger.Error(string(body)) + if errors.Is(ipfsUnpinnedError{}, err) { + return api.IPFSPinStatusUnpinned, nil + } + return api.IPFSPinStatusError, err + } + defer body.Close() + + var res api.IPFSPinInfo + dec := json.NewDecoder(body) + + err = dec.Decode(&res) + if err != nil { + logger.Error("error parsing pin/ls?arg=cid response") return api.IPFSPinStatusError, err } - // We do not know what string format the returned key has so - // we parse as CID. There should only be one returned key. - for k, pinObj := range res.Keys { - c, err := cid.Decode(k) - if err != nil || !c.Equals(pin.Cid) { - continue - } - return api.IPFSPinStatusFromString(pinObj.Type), nil - } - return api.IPFSPinStatusError, errors.New("expected to find the pin in the response") + return res.Type, nil } func (ipfs *Connector) doPostCtx(ctx context.Context, client *http.Client, apiURL, path string, contentType string, postBody io.Reader) (*http.Response, error) { @@ -601,7 +632,7 @@ func checkResponse(path string, res *http.Response) ([]byte, error) { // No error response with useful message from ipfs return nil, fmt.Errorf( - "IPFS request unsuccessful (%s). Code %d. Body: %s", + "IPFS request failed (is it running?) (%s). Code %d: %s", path, res.StatusCode, string(body)) @@ -611,18 +642,13 @@ func checkResponse(path string, res *http.Response) ([]byte, error) { // the ipfs daemon, reads the full body of the response and // returns it after checking for errors. func (ipfs *Connector) postCtx(ctx context.Context, path string, contentType string, postBody io.Reader) ([]byte, error) { - res, err := ipfs.doPostCtx(ctx, ipfs.client, ipfs.apiURL(), path, contentType, postBody) + rdr, err := ipfs.postCtxStreamResponse(ctx, path, contentType, postBody) if err != nil { return nil, err } - defer res.Body.Close() + defer rdr.Close() - errBody, err := checkResponse(path, res) - if err != nil { - return errBody, err - } - - body, err := ioutil.ReadAll(res.Body) + body, err := ioutil.ReadAll(rdr) if err != nil { logger.Errorf("error reading response body: %s", err) return nil, err @@ -630,6 +656,21 @@ func (ipfs *Connector) postCtx(ctx context.Context, path string, contentType str return body, nil } +// postCtxStreamResponse makes a POST request against the ipfs daemon, and +// returns the body reader after checking the request for errros. +func (ipfs *Connector) postCtxStreamResponse(ctx context.Context, path string, contentType string, postBody io.Reader) (io.ReadCloser, error) { + res, err := ipfs.doPostCtx(ctx, ipfs.client, ipfs.apiURL(), path, contentType, postBody) + if err != nil { + return nil, err + } + + _, err = checkResponse(path, res) + if err != nil { + return nil, err + } + return res.Body, nil +} + // apiURL is a short-hand for building the url of the IPFS // daemon API. func (ipfs *Connector) apiURL() string { diff --git a/ipfsconn/ipfshttp/ipfshttp_test.go b/ipfsconn/ipfshttp/ipfshttp_test.go index 1bc2a9da..e6d844e6 100644 --- a/ipfsconn/ipfshttp/ipfshttp_test.go +++ b/ipfsconn/ipfshttp/ipfshttp_test.go @@ -219,6 +219,27 @@ func TestIPFSPinLsCid_DifferentEncoding(t *testing.T) { } } +func collectPins(t *testing.T, pch <-chan api.IPFSPinInfo) []api.IPFSPinInfo { + t.Helper() + + var pins []api.IPFSPinInfo + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + for { + select { + case <-ctx.Done(): + t.Fatal(ctx.Err()) + return nil + case p, ok := <-pch: + if !ok { + return pins + } + pins = append(pins, p) + } + } +} + func TestIPFSPinLs(t *testing.T) { ctx := context.Background() ipfs, mock := testIPFSConnector(t) @@ -229,16 +250,21 @@ func TestIPFSPinLs(t *testing.T) { ipfs.Pin(ctx, api.PinCid(c)) ipfs.Pin(ctx, api.PinCid(c2)) - ipsMap, err := ipfs.PinLs(ctx, "") - if err != nil { - t.Error("should not error") + pinCh := make(chan api.IPFSPinInfo, 10) + go func() { + err := ipfs.PinLs(ctx, []string{""}, pinCh) + if err != nil { + t.Error("should not error") + } + }() + + pins := collectPins(t, pinCh) + + if len(pins) != 2 { + t.Fatal("the pin list does not contain the expected number of keys") } - if len(ipsMap) != 2 { - t.Fatal("the map does not contain expected keys") - } - - if !ipsMap[test.Cid1.String()].IsPinned(-1) || !ipsMap[test.Cid2.String()].IsPinned(-1) { + if !pins[0].Type.IsPinned(-1) || !pins[1].Type.IsPinned(-1) { t.Error("c1 and c2 should appear pinned") } } diff --git a/peer_manager_test.go b/peer_manager_test.go index 521e754c..62329bf3 100644 --- a/peer_manager_test.go +++ b/peer_manager_test.go @@ -114,7 +114,7 @@ func TestClustersPeerAdd(t *testing.T) { } // Check that they are part of the consensus - pins, err := c.Pins(ctx) + pins, err := c.pinsSlice(ctx) if err != nil { t.Fatal(err) } @@ -463,7 +463,7 @@ func TestClustersPeerRemoveReallocsPins(t *testing.T) { // Find out which pins are associated to the chosen peer. interestingCids := []cid.Cid{} - pins, err := chosen.Pins(ctx) + pins, err := chosen.pinsSlice(ctx) if err != nil { t.Fatal(err) } @@ -537,7 +537,7 @@ func TestClustersPeerJoin(t *testing.T) { if len(peers) != nClusters { t.Error("all peers should be connected") } - pins, err := c.Pins(ctx) + pins, err := c.pinsSlice(ctx) if err != nil { t.Fatal(err) } @@ -575,7 +575,7 @@ func TestClustersPeerJoinAllAtOnce(t *testing.T) { if len(peers) != nClusters { t.Error("all peers should be connected") } - pins, err := c.Pins(ctx) + pins, err := c.pinsSlice(ctx) if err != nil { t.Fatal(err) } diff --git a/pintracker/optracker/operationtracker.go b/pintracker/optracker/operationtracker.go index 0d1a4aec..c0a853cf 100644 --- a/pintracker/optracker/operationtracker.go +++ b/pintracker/optracker/operationtracker.go @@ -140,7 +140,7 @@ func (opt *OperationTracker) SetError(ctx context.Context, c cid.Cid, err error) } } -func (opt *OperationTracker) unsafePinInfo(ctx context.Context, op *Operation) api.PinInfo { +func (opt *OperationTracker) unsafePinInfo(ctx context.Context, op *Operation, ipfs api.IPFSID) api.PinInfo { if op == nil { return api.PinInfo{ Cid: cid.Undef, @@ -162,26 +162,27 @@ func (opt *OperationTracker) unsafePinInfo(ctx context.Context, op *Operation) a Peer: opt.pid, Name: op.Pin().Name, PinInfoShort: api.PinInfoShort{ - PeerName: opt.peerName, - IPFS: "", - Status: op.ToTrackerStatus(), - TS: op.Timestamp(), - AttemptCount: op.AttemptCount(), - PriorityPin: op.PriorityPin(), - Error: op.Error(), + PeerName: opt.peerName, + IPFS: ipfs.ID, + IPFSAddresses: ipfs.Addresses, + Status: op.ToTrackerStatus(), + TS: op.Timestamp(), + AttemptCount: op.AttemptCount(), + PriorityPin: op.PriorityPin(), + Error: op.Error(), }, } } // Get returns a PinInfo object for Cid. -func (opt *OperationTracker) Get(ctx context.Context, c cid.Cid) api.PinInfo { +func (opt *OperationTracker) Get(ctx context.Context, c cid.Cid, ipfs api.IPFSID) api.PinInfo { ctx, span := trace.StartSpan(ctx, "optracker/GetAll") defer span.End() opt.mu.RLock() defer opt.mu.RUnlock() op := opt.operations[c] - pInfo := opt.unsafePinInfo(ctx, op) + pInfo := opt.unsafePinInfo(ctx, op, ipfs) if pInfo.Cid == cid.Undef { pInfo.Cid = c } @@ -190,7 +191,7 @@ func (opt *OperationTracker) Get(ctx context.Context, c cid.Cid) api.PinInfo { // GetExists returns a PinInfo object for a Cid only if there exists // an associated Operation. -func (opt *OperationTracker) GetExists(ctx context.Context, c cid.Cid) (api.PinInfo, bool) { +func (opt *OperationTracker) GetExists(ctx context.Context, c cid.Cid, ipfs api.IPFSID) (api.PinInfo, bool) { ctx, span := trace.StartSpan(ctx, "optracker/GetExists") defer span.End() @@ -200,25 +201,51 @@ func (opt *OperationTracker) GetExists(ctx context.Context, c cid.Cid) (api.PinI if !ok { return api.PinInfo{}, false } - pInfo := opt.unsafePinInfo(ctx, op) + pInfo := opt.unsafePinInfo(ctx, op, ipfs) return pInfo, true } // GetAll returns PinInfo objects for all known operations. -func (opt *OperationTracker) GetAll(ctx context.Context) []api.PinInfo { +func (opt *OperationTracker) GetAll(ctx context.Context, ipfs api.IPFSID) []api.PinInfo { ctx, span := trace.StartSpan(ctx, "optracker/GetAll") defer span.End() + ch := make(chan api.PinInfo, 1024) var pinfos []api.PinInfo - opt.mu.RLock() - defer opt.mu.RUnlock() - for _, op := range opt.operations { - pinfo := opt.unsafePinInfo(ctx, op) + go opt.GetAllChannel(ctx, api.TrackerStatusUndefined, ipfs, ch) + for pinfo := range ch { pinfos = append(pinfos, pinfo) } return pinfos } +// GetAllChannel returns all known operations that match the filter on the +// provided channel. Blocks until done. +func (opt *OperationTracker) GetAllChannel(ctx context.Context, filter api.TrackerStatus, ipfs api.IPFSID, out chan<- api.PinInfo) error { + defer close(out) + + opt.mu.RLock() + defer opt.mu.RUnlock() + + for _, op := range opt.operations { + pinfo := opt.unsafePinInfo(ctx, op, ipfs) + if pinfo.Status.Match(filter) { + select { + case <-ctx.Done(): + return fmt.Errorf("listing operations aborted: %w", ctx.Err()) + default: + } + + select { + case <-ctx.Done(): + return fmt.Errorf("listing operations aborted: %w", ctx.Err()) + case out <- pinfo: + } + } + } + return nil +} + // CleanAllDone deletes any operation from the tracker that is in PhaseDone. func (opt *OperationTracker) CleanAllDone(ctx context.Context) { opt.mu.Lock() @@ -245,13 +272,13 @@ func (opt *OperationTracker) OpContext(ctx context.Context, c cid.Cid) context.C // Operations that matched the provided filter. Note, only supports // filters of type OperationType or Phase, any other type // will result in a nil slice being returned. -func (opt *OperationTracker) Filter(ctx context.Context, filters ...interface{}) []api.PinInfo { +func (opt *OperationTracker) Filter(ctx context.Context, ipfs api.IPFSID, filters ...interface{}) []api.PinInfo { var pinfos []api.PinInfo opt.mu.RLock() defer opt.mu.RUnlock() ops := filterOpsMap(ctx, opt.operations, filters) for _, op := range ops { - pinfo := opt.unsafePinInfo(ctx, op) + pinfo := opt.unsafePinInfo(ctx, op, ipfs) pinfos = append(pinfos, pinfo) } return pinfos diff --git a/pintracker/optracker/operationtracker_test.go b/pintracker/optracker/operationtracker_test.go index 3f4bf450..493c396e 100644 --- a/pintracker/optracker/operationtracker_test.go +++ b/pintracker/optracker/operationtracker_test.go @@ -126,7 +126,7 @@ func TestOperationTracker_SetError(t *testing.T) { opt := testOperationTracker(t) opt.TrackNewOperation(ctx, api.PinCid(test.Cid1), OperationPin, PhaseDone) opt.SetError(ctx, test.Cid1, errors.New("fake error")) - pinfo := opt.Get(ctx, test.Cid1) + pinfo := opt.Get(ctx, test.Cid1, api.IPFSID{}) if pinfo.Status != api.TrackerStatusPinError { t.Error("should have updated the status") } @@ -148,7 +148,7 @@ func TestOperationTracker_Get(t *testing.T) { opt.TrackNewOperation(ctx, api.PinCid(test.Cid1), OperationPin, PhaseDone) t.Run("Get with existing item", func(t *testing.T) { - pinfo := opt.Get(ctx, test.Cid1) + pinfo := opt.Get(ctx, test.Cid1, api.IPFSID{}) if pinfo.Status != api.TrackerStatusPinned { t.Error("bad status") } @@ -163,7 +163,7 @@ func TestOperationTracker_Get(t *testing.T) { }) t.Run("Get with unexisting item", func(t *testing.T) { - pinfo := opt.Get(ctx, test.Cid2) + pinfo := opt.Get(ctx, test.Cid2, api.IPFSID{}) if pinfo.Status != api.TrackerStatusUnpinned { t.Error("bad status") } @@ -181,7 +181,7 @@ func TestOperationTracker_GetAll(t *testing.T) { ctx := context.Background() opt := testOperationTracker(t) opt.TrackNewOperation(ctx, api.PinCid(test.Cid1), OperationPin, PhaseInProgress) - pinfos := opt.GetAll(ctx) + pinfos := opt.GetAll(ctx, api.IPFSID{}) if len(pinfos) != 1 { t.Fatal("expected 1 item") } diff --git a/pintracker/pintracker_test.go b/pintracker/pintracker_test.go index 11883339..9a0305b6 100644 --- a/pintracker/pintracker_test.go +++ b/pintracker/pintracker_test.go @@ -165,6 +165,28 @@ func TestPinTracker_Untrack(t *testing.T) { } } +func collectPinInfos(t *testing.T, out chan api.PinInfo) []api.PinInfo { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var pis []api.PinInfo + for { + select { + case <-ctx.Done(): + t.Error("took too long") + return nil + case pi, ok := <-out: + if !ok { + return pis + } + pis = append(pis, pi) + } + } + +} + func TestPinTracker_StatusAll(t *testing.T) { type args struct { c api.Pin @@ -216,7 +238,16 @@ func TestPinTracker_StatusAll(t *testing.T) { t.Errorf("PinTracker.Track() error = %v", err) } time.Sleep(200 * time.Millisecond) - got := tt.args.tracker.StatusAll(context.Background(), api.TrackerStatusUndefined) + infos := make(chan api.PinInfo) + go func() { + err := tt.args.tracker.StatusAll(context.Background(), api.TrackerStatusUndefined, infos) + if err != nil { + t.Error() + } + }() + + got := collectPinInfos(t, infos) + if len(got) != len(tt.want) { for _, pi := range got { t.Logf("pinfo: %v", pi) @@ -240,31 +271,6 @@ func TestPinTracker_StatusAll(t *testing.T) { } } -func BenchmarkPinTracker_StatusAll(b *testing.B) { - type args struct { - tracker ipfscluster.PinTracker - } - tests := []struct { - name string - args args - }{ - { - "basic stateless track", - args{ - testStatelessPinTracker(b), - }, - }, - } - for _, tt := range tests { - b.Run(tt.name, func(b *testing.B) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - tt.args.tracker.StatusAll(context.Background(), api.TrackerStatusUndefined) - } - }) - } -} - func TestPinTracker_Status(t *testing.T) { type args struct { c cid.Cid @@ -350,11 +356,16 @@ func TestPinTracker_RecoverAll(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := tt.args.tracker.RecoverAll(context.Background()) - if (err != nil) != tt.wantErr { - t.Errorf("PinTracker.RecoverAll() error = %v, wantErr %v", err, tt.wantErr) - return - } + infos := make(chan api.PinInfo) + go func() { + err := tt.args.tracker.RecoverAll(context.Background(), infos) + if (err != nil) != tt.wantErr { + t.Errorf("PinTracker.RecoverAll() error = %v, wantErr %v", err, tt.wantErr) + return + } + }() + + got := collectPinInfos(t, infos) if len(got) != len(tt.want) { for _, pi := range got { diff --git a/pintracker/stateless/stateless.go b/pintracker/stateless/stateless.go index 57c4961f..fd43823a 100644 --- a/pintracker/stateless/stateless.go +++ b/pintracker/stateless/stateless.go @@ -6,6 +6,7 @@ package stateless import ( "context" "errors" + "fmt" "sync" "time" @@ -23,6 +24,8 @@ import ( var logger = logging.Logger("pintracker") +const pinsChannelSize = 1024 + var ( // ErrFullQueue is the error used when pin or unpin operation channel is full. ErrFullQueue = errors.New("pin/unpin operation queue is full. Try increasing max_pin_queue_size") @@ -321,36 +324,134 @@ func (spt *Tracker) Untrack(ctx context.Context, c cid.Cid) error { } // StatusAll returns information for all Cids pinned to the local IPFS node. -func (spt *Tracker) StatusAll(ctx context.Context, filter api.TrackerStatus) []api.PinInfo { +func (spt *Tracker) StatusAll(ctx context.Context, filter api.TrackerStatus, out chan<- api.PinInfo) error { ctx, span := trace.StartSpan(ctx, "tracker/stateless/StatusAll") defer span.End() - pininfos, err := spt.localStatus(ctx, true, filter) - if err != nil { - return nil - } - - // get all inflight operations from optracker and put them into the - // map, deduplicating any existing items with their inflight operation. - // - // we cannot filter in GetAll, because we are meant to replace items in - // pininfos and set the correct status, as otherwise they will remain in - // PinError. ipfsid := spt.getIPFSID(ctx) - for _, infop := range spt.optracker.GetAll(ctx) { - infop.IPFS = ipfsid.ID - infop.IPFSAddresses = ipfsid.Addresses - pininfos[infop.Cid] = infop + + // Any other states are just operation-tracker states, so we just give + // those and return. + if !filter.Match( + api.TrackerStatusPinned | api.TrackerStatusUnexpectedlyUnpinned | + api.TrackerStatusSharded | api.TrackerStatusRemote) { + return spt.optracker.GetAllChannel(ctx, filter, ipfsid, out) } - var pis []api.PinInfo - for _, pi := range pininfos { - // Last filter. - if pi.Status.Match(filter) { - pis = append(pis, pi) + defer close(out) + + // get global state - cluster pinset + st, err := spt.getState(ctx) + if err != nil { + logger.Error(err) + return err + } + + var ipfsRecursivePins map[api.Cid]api.IPFSPinStatus + // Only query IPFS if we want to status for pinned items + if filter.Match(api.TrackerStatusPinned | api.TrackerStatusUnexpectedlyUnpinned) { + ipfsRecursivePins = make(map[api.Cid]api.IPFSPinStatus) + // At some point we need a full map of what we have and what + // we don't. The IPFS pinset is the smallest thing we can keep + // on memory. + ipfsPinsCh, err := spt.ipfsPins(ctx) + if err != nil { + logger.Error(err) + return err + } + for ipfsPinInfo := range ipfsPinsCh { + ipfsRecursivePins[ipfsPinInfo.Cid] = ipfsPinInfo.Type } } - return pis + + // Prepare pinset streaming + statePins := make(chan api.Pin, pinsChannelSize) + err = st.List(ctx, statePins) + if err != nil { + logger.Error(err) + return err + } + + // a shorthand for this select. + trySend := func(info api.PinInfo) bool { + select { + case <-ctx.Done(): + return false + case out <- info: + return true + } + } + + // For every item in the state. + for p := range statePins { + select { + case <-ctx.Done(): + default: + } + + // if there is an operation, issue that and move on + info, ok := spt.optracker.GetExists(ctx, p.Cid, ipfsid) + if ok && filter.Match(info.Status) { + if !trySend(info) { + return fmt.Errorf("error issuing PinInfo: %w", ctx.Err()) + } + continue // next pin + } + + // Preliminary PinInfo for this Pin. + info = api.PinInfo{ + Cid: p.Cid, + Name: p.Name, + Peer: spt.peerID, + Allocations: p.Allocations, + Origins: p.Origins, + Created: p.Timestamp, + Metadata: p.Metadata, + + PinInfoShort: api.PinInfoShort{ + PeerName: spt.peerName, + IPFS: ipfsid.ID, + IPFSAddresses: ipfsid.Addresses, + Status: api.TrackerStatusUndefined, // TBD + TS: p.Timestamp, + Error: "", + AttemptCount: 0, + PriorityPin: false, + }, + } + + ipfsStatus, pinnedInIpfs := ipfsRecursivePins[api.Cid(p.Cid)] + + switch { + case p.Type == api.MetaType: + info.Status = api.TrackerStatusSharded + case p.IsRemotePin(spt.peerID): + info.Status = api.TrackerStatusRemote + case pinnedInIpfs: + // No need to filter. pinnedInIpfs is false + // unless the filter is Pinned | + // UnexpectedlyUnpinned. We filter at the end. + info.Status = ipfsStatus.ToTrackerStatus() + default: + // Not on an operation + // Not a meta pin + // Not a remote pin + // Not a pin on ipfs + + // We understand that this is something that + // should be pinned on IPFS and it is not. + info.Status = api.TrackerStatusUnexpectedlyUnpinned + info.Error = errUnexpectedlyUnpinned.Error() + } + if !filter.Match(info.Status) { + continue + } + + if !trySend(info) { + return fmt.Errorf("error issuing PinInfo: %w", ctx.Err()) + } + } + return nil } // Status returns information for a Cid pinned to the local IPFS node. @@ -361,10 +462,7 @@ func (spt *Tracker) Status(ctx context.Context, c cid.Cid) api.PinInfo { ipfsid := spt.getIPFSID(ctx) // check if c has an inflight operation or errorred operation in optracker - if oppi, ok := spt.optracker.GetExists(ctx, c); ok { - // if it does return the status of the operation - oppi.IPFS = ipfsid.ID - oppi.IPFSAddresses = ipfsid.Addresses + if oppi, ok := spt.optracker.GetExists(ctx, c, ipfsid); ok { return oppi } @@ -452,31 +550,46 @@ func (spt *Tracker) Status(ctx context.Context, c cid.Cid) api.PinInfo { } // RecoverAll attempts to recover all items tracked by this peer. It returns -// items that have been re-queued. -func (spt *Tracker) RecoverAll(ctx context.Context) ([]api.PinInfo, error) { +// any errors or when it is done re-tracking. +func (spt *Tracker) RecoverAll(ctx context.Context, out chan<- api.PinInfo) error { + defer close(out) + ctx, span := trace.StartSpan(ctx, "tracker/stateless/RecoverAll") defer span.End() - // FIXME: make sure this returns a channel. - statuses := spt.StatusAll(ctx, api.TrackerStatusUndefined) - resp := make([]api.PinInfo, 0) - for _, st := range statuses { + statusesCh := make(chan api.PinInfo, 1024) + err := spt.StatusAll(ctx, api.TrackerStatusUndefined, statusesCh) + if err != nil { + return err + } + + for st := range statusesCh { // Break out if we shutdown. We might be going through // a very long list of statuses. select { case <-spt.ctx.Done(): - return nil, spt.ctx.Err() + err = fmt.Errorf("RecoverAll aborted: %w", ctx.Err()) + logger.Error(err) + return err default: - r, err := spt.recoverWithPinInfo(ctx, st) + p, err := spt.recoverWithPinInfo(ctx, st) if err != nil { - return resp, err + err = fmt.Errorf("RecoverAll error: %w", err) + logger.Error(err) + return err } - if r.Defined() { - resp = append(resp, r) + if p.Defined() { + select { + case <-ctx.Done(): + err = fmt.Errorf("RecoverAll aborted: %w", ctx.Err()) + logger.Error(err) + return err + case out <- p: + } } } } - return resp, nil + return nil } // Recover will trigger pinning or unpinning for items in @@ -485,13 +598,7 @@ func (spt *Tracker) Recover(ctx context.Context, c cid.Cid) (api.PinInfo, error) ctx, span := trace.StartSpan(ctx, "tracker/stateless/Recover") defer span.End() - // Check if we have a status in the operation tracker and use that - // pininfo. Otherwise, get a status by checking against IPFS and use - // that. - pi, ok := spt.optracker.GetExists(ctx, c) - if !ok { - pi = spt.Status(ctx, c) - } + pi := spt.Status(ctx, c) recPi, err := spt.recoverWithPinInfo(ctx, pi) // if it was not enqueued, no updated pin-info is returned. @@ -524,158 +631,29 @@ func (spt *Tracker) recoverWithPinInfo(ctx context.Context, pi api.PinInfo) (api return spt.Status(ctx, pi.Cid), nil } -func (spt *Tracker) ipfsStatusAll(ctx context.Context) (map[cid.Cid]api.PinInfo, error) { +func (spt *Tracker) ipfsPins(ctx context.Context) (<-chan api.IPFSPinInfo, error) { ctx, span := trace.StartSpan(ctx, "tracker/stateless/ipfsStatusAll") defer span.End() - var ipsMap map[string]api.IPFSPinStatus - err := spt.rpcClient.CallContext( - ctx, - "", - "IPFSConnector", - "PinLs", - "recursive", - &ipsMap, - ) - if err != nil { - logger.Error(err) - return nil, err - } - ipfsid := spt.getIPFSID(ctx) - pins := make(map[cid.Cid]api.PinInfo, len(ipsMap)) - for cidstr, ips := range ipsMap { - c, err := cid.Decode(cidstr) + in := make(chan []string, 1) // type filter. + in <- []string{"recursive", "direct"} + close(in) + out := make(chan api.IPFSPinInfo, pinsChannelSize) + + go func() { + err := spt.rpcClient.Stream( + ctx, + "", + "IPFSConnector", + "PinLs", + in, + out, + ) if err != nil { logger.Error(err) - continue } - p := api.PinInfo{ - Cid: c, - Name: "", // to be filled later - Allocations: nil, // to be filled later - Origins: nil, // to be filled later - //Created: nil, // to be filled later - Metadata: nil, // to be filled later - Peer: spt.peerID, - PinInfoShort: api.PinInfoShort{ - PeerName: spt.peerName, - IPFS: ipfsid.ID, - IPFSAddresses: ipfsid.Addresses, - Status: ips.ToTrackerStatus(), - TS: time.Now(), // to be set later - AttemptCount: 0, - PriorityPin: false, - }, - } - pins[c] = p - } - return pins, nil -} - -// localStatus returns a joint set of consensusState and ipfsStatus marking -// pins which should be meta or remote and leaving any ipfs pins that aren't -// in the consensusState out. If incExtra is true, Remote and Sharded pins -// will be added to the status slice. If a filter is provided, only statuses -// matching the filter will be returned. -func (spt *Tracker) localStatus(ctx context.Context, incExtra bool, filter api.TrackerStatus) (map[cid.Cid]api.PinInfo, error) { - ctx, span := trace.StartSpan(ctx, "tracker/stateless/localStatus") - defer span.End() - - // get shared state - st, err := spt.getState(ctx) - if err != nil { - logger.Error(err) - return nil, err - } - - // Only list the full pinset if we are interested in pin types that - // require it. Otherwise said, this whole method is mostly a no-op - // when filtering for queued/error items which are all in the operation - // tracker. - var statePins <-chan api.Pin - if filter.Match( - api.TrackerStatusPinned | api.TrackerStatusUnexpectedlyUnpinned | - api.TrackerStatusSharded | api.TrackerStatusRemote) { - statePins, err = st.List(ctx) - if err != nil { - logger.Error(err) - return nil, err - } - } else { - // no state pins - ch := make(chan api.Pin) - close(ch) - statePins = ch - } - - var localpis map[cid.Cid]api.PinInfo - // Only query IPFS if we want to status for pinned items - if filter.Match(api.TrackerStatusPinned | api.TrackerStatusUnexpectedlyUnpinned) { - localpis, err = spt.ipfsStatusAll(ctx) - if err != nil { - logger.Error(err) - return nil, err - } - } - - pininfos := make(map[cid.Cid]api.PinInfo, len(statePins)) - ipfsid := spt.getIPFSID(ctx) - for p := range statePins { - ipfsInfo, pinnedInIpfs := localpis[p.Cid] - // base pinInfo object - status to be filled. - pinInfo := api.PinInfo{ - Cid: p.Cid, - Name: p.Name, - Peer: spt.peerID, - Allocations: p.Allocations, - Origins: p.Origins, - Created: p.Timestamp, - Metadata: p.Metadata, - PinInfoShort: api.PinInfoShort{ - PeerName: spt.peerName, - IPFS: ipfsid.ID, - IPFSAddresses: ipfsid.Addresses, - TS: p.Timestamp, - AttemptCount: 0, - PriorityPin: false, - }, - } - - switch { - case p.Type == api.MetaType: - if !incExtra || !filter.Match(api.TrackerStatusSharded) { - continue - } - pinInfo.Status = api.TrackerStatusSharded - pininfos[p.Cid] = pinInfo - case p.IsRemotePin(spt.peerID): - if !incExtra || !filter.Match(api.TrackerStatusRemote) { - continue - } - pinInfo.Status = api.TrackerStatusRemote - pininfos[p.Cid] = pinInfo - case pinnedInIpfs: // always false unless filter matches TrackerStatusPinnned - ipfsInfo.Name = p.Name - ipfsInfo.TS = p.Timestamp - ipfsInfo.Allocations = p.Allocations - ipfsInfo.Origins = p.Origins - ipfsInfo.Created = p.Timestamp - ipfsInfo.Metadata = p.Metadata - pininfos[p.Cid] = ipfsInfo - default: - // report as UNEXPECTEDLY_UNPINNED for this peer. - // this will be overwritten if the operation tracker - // has more info for this (an ongoing pinning - // operation). Otherwise, it means something should be - // pinned and it is not known by IPFS. Should be - // handled to "recover". - - pinInfo.Status = api.TrackerStatusUnexpectedlyUnpinned - pinInfo.Error = errUnexpectedlyUnpinned.Error() - pininfos[p.Cid] = pinInfo - } - } - return pininfos, nil + }() + return out, nil } // func (spt *Tracker) getErrorsAll(ctx context.Context) []api.PinInfo { diff --git a/pintracker/stateless/stateless_test.go b/pintracker/stateless/stateless_test.go index 9f4e1d4b..d6f18980 100644 --- a/pintracker/stateless/stateless_test.go +++ b/pintracker/stateless/stateless_test.go @@ -64,13 +64,17 @@ func (mock *mockIPFS) Unpin(ctx context.Context, in api.Pin, out *struct{}) erro return nil } -func (mock *mockIPFS) PinLs(ctx context.Context, in string, out *map[string]api.IPFSPinStatus) error { - // Must be consistent with PinLsCid - m := map[string]api.IPFSPinStatus{ - test.Cid1.String(): api.IPFSPinStatusRecursive, - test.Cid2.String(): api.IPFSPinStatusRecursive, +func (mock *mockIPFS) PinLs(ctx context.Context, in <-chan []string, out chan<- api.IPFSPinInfo) error { + out <- api.IPFSPinInfo{ + Cid: api.Cid(test.Cid1), + Type: api.IPFSPinStatusRecursive, } - *out = m + + out <- api.IPFSPinInfo{ + Cid: api.Cid(test.Cid2), + Type: api.IPFSPinStatusRecursive, + } + close(out) return nil } @@ -207,7 +211,7 @@ func TestTrackUntrackWithCancel(t *testing.T) { time.Sleep(100 * time.Millisecond) // let pinning start - pInfo := spt.optracker.Get(ctx, slowPin.Cid) + pInfo := spt.optracker.Get(ctx, slowPin.Cid, api.IPFSID{}) if pInfo.Status == api.TrackerStatusUnpinned { t.Fatal("slowPin should be tracked") } @@ -264,7 +268,7 @@ func TestTrackUntrackWithNoCancel(t *testing.T) { } // fastPin should be queued because slow pin is pinning - fastPInfo := spt.optracker.Get(ctx, fastPin.Cid) + fastPInfo := spt.optracker.Get(ctx, fastPin.Cid, api.IPFSID{}) if fastPInfo.Status == api.TrackerStatusUnpinned { t.Fatal("fastPin should be tracked") } @@ -281,7 +285,7 @@ func TestTrackUntrackWithNoCancel(t *testing.T) { t.Errorf("fastPin should be queued to pin but is %s", fastPInfo.Status) } - pi := spt.optracker.Get(ctx, fastPin.Cid) + pi := spt.optracker.Get(ctx, fastPin.Cid, api.IPFSID{}) if pi.Cid == cid.Undef { t.Error("fastPin should have been removed from tracker") } @@ -313,7 +317,7 @@ func TestUntrackTrackWithCancel(t *testing.T) { time.Sleep(100 * time.Millisecond) - pi := spt.optracker.Get(ctx, slowPin.Cid) + pi := spt.optracker.Get(ctx, slowPin.Cid, api.IPFSID{}) if pi.Cid == cid.Undef { t.Fatal("expected slowPin to be tracked") } @@ -374,7 +378,7 @@ func TestUntrackTrackWithNoCancel(t *testing.T) { t.Fatal(err) } - pi := spt.optracker.Get(ctx, fastPin.Cid) + pi := spt.optracker.Get(ctx, fastPin.Cid, api.IPFSID{}) if pi.Cid == cid.Undef { t.Fatal("c untrack operation should be tracked") } @@ -405,11 +409,10 @@ func TestStatusAll(t *testing.T) { // - Build a state with one pins (Cid1,Cid4) // - The IPFS Mock reports Cid1 and Cid2 // - Track a SlowCid additionally - - spt := testStatelessPinTracker(t, normalPin, normalPin2) + slowPin := api.PinWithOpts(test.SlowCid1, pinOpts) + spt := testStatelessPinTracker(t, normalPin, normalPin2, slowPin) defer spt.Shutdown(ctx) - slowPin := api.PinWithOpts(test.SlowCid1, pinOpts) err := spt.Track(ctx, slowPin) if err != nil { t.Fatal(err) @@ -421,20 +424,23 @@ func TestStatusAll(t *testing.T) { // * A slow CID pinning // * Cid1 is pinned // * Cid4 should be in PinError (it's in the state but not on IPFS) - stAll := spt.StatusAll(ctx, api.TrackerStatusUndefined) - if len(stAll) != 3 { - t.Errorf("wrong status length. Expected 3, got: %d", len(stAll)) + stAll := make(chan api.PinInfo, 10) + err = spt.StatusAll(ctx, api.TrackerStatusUndefined, stAll) + if err != nil { + t.Fatal(err) } - for _, pi := range stAll { + n := 0 + for pi := range stAll { + n++ switch pi.Cid { case test.Cid1: if pi.Status != api.TrackerStatusPinned { - t.Error("cid1 should be pinned") + t.Error(test.Cid1, " should be pinned") } case test.Cid4: if pi.Status != api.TrackerStatusUnexpectedlyUnpinned { - t.Error("cid2 should be in unexpectedly_unpinned status") + t.Error(test.Cid2, " should be in unexpectedly_unpinned status") } case test.SlowCid1: if pi.Status != api.TrackerStatusPinning { @@ -447,6 +453,9 @@ func TestStatusAll(t *testing.T) { t.Error("IPFS field should be set") } } + if n != 3 { + t.Errorf("wrong status length. Expected 3, got: %d", n) + } } // TestStatus checks that the Status calls correctly reports tracked @@ -565,12 +574,3 @@ func TestAttemptCountAndPriority(t *testing.T) { t.Errorf("errPin should have 2 attempt counts to unpin: %+v", st) } } - -func BenchmarkTracker_localStatus(b *testing.B) { - tracker := testStatelessPinTracker(b) - ctx := context.Background() - b.ResetTimer() - for i := 0; i < b.N; i++ { - tracker.localStatus(ctx, true, api.TrackerStatusUndefined) - } -} diff --git a/rpc_api.go b/rpc_api.go index df8f0470..c1b9c67a 100644 --- a/rpc_api.go +++ b/rpc_api.go @@ -2,6 +2,7 @@ package ipfscluster import ( "context" + "errors" "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/state" @@ -32,6 +33,8 @@ const ( // RPCEndpointType controls how access is granted to an RPC endpoint type RPCEndpointType int +const rpcStreamBufferSize = 1024 + // A trick to find where something is used (i.e. Cluster.Pin): // grep -R -B 3 '"Pin"' | grep -C 1 '"Cluster"'. // This does not cover globalPinInfo*(...) broadcasts nor redirects to leader @@ -63,6 +66,7 @@ func newRPCServer(c *Cluster) (*rpc.Server, error) { version.RPCProtocol, rpc.WithServerStatsHandler(&ocgorpc.ServerHandler{}), rpc.WithAuthorizeFunc(authF), + rpc.WithStreamBufferSize(rpcStreamBufferSize), ) } else { s = rpc.NewServer(c.host, version.RPCProtocol, rpc.WithAuthorizeFunc(authF)) @@ -201,17 +205,7 @@ func (rpcapi *ClusterRPCAPI) UnpinPath(ctx context.Context, in api.PinPath, out // Pins runs Cluster.Pins(). func (rpcapi *ClusterRPCAPI) Pins(ctx context.Context, in <-chan struct{}, out chan<- api.Pin) error { - pinCh, err := rpcapi.c.PinsChannel(ctx) - if err != nil { - return err - } - - for pin := range pinCh { - out <- pin - } - - close(out) - return ctx.Err() + return rpcapi.c.Pins(ctx, out) } // PinGet runs Cluster.PinGet(). @@ -275,20 +269,15 @@ func (rpcapi *ClusterRPCAPI) Join(ctx context.Context, in api.Multiaddr, out *st } // StatusAll runs Cluster.StatusAll(). -func (rpcapi *ClusterRPCAPI) StatusAll(ctx context.Context, in api.TrackerStatus, out *[]api.GlobalPinInfo) error { - pinfos, err := rpcapi.c.StatusAll(ctx, in) - if err != nil { - return err - } - *out = pinfos - return nil +func (rpcapi *ClusterRPCAPI) StatusAll(ctx context.Context, in <-chan api.TrackerStatus, out chan<- api.GlobalPinInfo) error { + filter := <-in + return rpcapi.c.StatusAll(ctx, filter, out) } // StatusAllLocal runs Cluster.StatusAllLocal(). -func (rpcapi *ClusterRPCAPI) StatusAllLocal(ctx context.Context, in api.TrackerStatus, out *[]api.PinInfo) error { - pinfos := rpcapi.c.StatusAllLocal(ctx, in) - *out = pinfos - return nil +func (rpcapi *ClusterRPCAPI) StatusAllLocal(ctx context.Context, in <-chan api.TrackerStatus, out chan<- api.PinInfo) error { + filter := <-in + return rpcapi.c.StatusAllLocal(ctx, filter, out) } // Status runs Cluster.Status(). @@ -309,23 +298,13 @@ func (rpcapi *ClusterRPCAPI) StatusLocal(ctx context.Context, in cid.Cid, out *a } // RecoverAll runs Cluster.RecoverAll(). -func (rpcapi *ClusterRPCAPI) RecoverAll(ctx context.Context, in struct{}, out *[]api.GlobalPinInfo) error { - pinfos, err := rpcapi.c.RecoverAll(ctx) - if err != nil { - return err - } - *out = pinfos - return nil +func (rpcapi *ClusterRPCAPI) RecoverAll(ctx context.Context, in <-chan struct{}, out chan<- api.GlobalPinInfo) error { + return rpcapi.c.RecoverAll(ctx, out) } // RecoverAllLocal runs Cluster.RecoverAllLocal(). -func (rpcapi *ClusterRPCAPI) RecoverAllLocal(ctx context.Context, in struct{}, out *[]api.PinInfo) error { - pinfos, err := rpcapi.c.RecoverAllLocal(ctx) - if err != nil { - return err - } - *out = pinfos - return nil +func (rpcapi *ClusterRPCAPI) RecoverAllLocal(ctx context.Context, in <-chan struct{}, out chan<- api.PinInfo) error { + return rpcapi.c.RecoverAllLocal(ctx, out) } // Recover runs Cluster.Recover(). @@ -469,11 +448,17 @@ func (rpcapi *PinTrackerRPCAPI) Untrack(ctx context.Context, in api.Pin, out *st } // StatusAll runs PinTracker.StatusAll(). -func (rpcapi *PinTrackerRPCAPI) StatusAll(ctx context.Context, in api.TrackerStatus, out *[]api.PinInfo) error { +func (rpcapi *PinTrackerRPCAPI) StatusAll(ctx context.Context, in <-chan api.TrackerStatus, out chan<- api.PinInfo) error { ctx, span := trace.StartSpan(ctx, "rpc/tracker/StatusAll") defer span.End() - *out = rpcapi.tracker.StatusAll(ctx, in) - return nil + + select { + case <-ctx.Done(): + close(out) + return ctx.Err() + case filter := <-in: + return rpcapi.tracker.StatusAll(ctx, filter, out) + } } // Status runs PinTracker.Status(). @@ -486,15 +471,10 @@ func (rpcapi *PinTrackerRPCAPI) Status(ctx context.Context, in cid.Cid, out *api } // RecoverAll runs PinTracker.RecoverAll().f -func (rpcapi *PinTrackerRPCAPI) RecoverAll(ctx context.Context, in struct{}, out *[]api.PinInfo) error { +func (rpcapi *PinTrackerRPCAPI) RecoverAll(ctx context.Context, in <-chan struct{}, out chan<- api.PinInfo) error { ctx, span := trace.StartSpan(ctx, "rpc/tracker/RecoverAll") defer span.End() - pinfos, err := rpcapi.tracker.RecoverAll(ctx) - if err != nil { - return err - } - *out = pinfos - return nil + return rpcapi.tracker.RecoverAll(ctx, out) } // Recover runs PinTracker.Recover(). @@ -533,13 +513,18 @@ func (rpcapi *IPFSConnectorRPCAPI) PinLsCid(ctx context.Context, in api.Pin, out } // PinLs runs IPFSConnector.PinLs(). -func (rpcapi *IPFSConnectorRPCAPI) PinLs(ctx context.Context, in string, out *map[string]api.IPFSPinStatus) error { - m, err := rpcapi.ipfs.PinLs(ctx, in) - if err != nil { - return err +func (rpcapi *IPFSConnectorRPCAPI) PinLs(ctx context.Context, in <-chan []string, out chan<- api.IPFSPinInfo) error { + select { + case <-ctx.Done(): + close(out) + return ctx.Err() + case pinTypes, ok := <-in: + if !ok { + close(out) + return errors.New("no pinType provided for pin/ls") + } + return rpcapi.ipfs.PinLs(ctx, pinTypes, out) } - *out = m - return nil } // ConfigKey runs IPFSConnector.ConfigKey(). diff --git a/sharness/t0054-service-state-clean.sh b/sharness/t0054-service-state-clean.sh index 556a5e50..90ac021f 100755 --- a/sharness/t0054-service-state-clean.sh +++ b/sharness/t0054-service-state-clean.sh @@ -12,17 +12,17 @@ test_expect_success IPFS,CLUSTER "state cleanup refreshes state on restart (crdt ipfs-cluster-ctl pin add "$cid" && sleep 5 && ipfs-cluster-ctl pin ls "$cid" | grep -q "$cid" && ipfs-cluster-ctl status "$cid" | grep -q -i "PINNED" && - [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq ". | length")" ] && + [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq -n "[inputs] | length")" ] && cluster_kill && sleep 5 && ipfs-cluster-service --config "test-config" state cleanup -f && cluster_start && sleep 5 && - [ 0 -eq "$(ipfs-cluster-ctl --enc=json status | jq ". | length")" ] + [ 0 -eq "$(ipfs-cluster-ctl --enc=json status | jq -n "[inputs] | length")" ] ' test_expect_success IPFS,CLUSTER "export + cleanup + import == noop (crdt)" ' cid=`docker exec ipfs sh -c "echo test_54 | ipfs add -q"` && ipfs-cluster-ctl pin add "$cid" && sleep 5 && - [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq ". | length")" ] && + [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq -n "[inputs] | length")" ] && cluster_kill && sleep 5 && ipfs-cluster-service --config "test-config" state export -f import.json && ipfs-cluster-service --config "test-config" state cleanup -f && @@ -30,7 +30,7 @@ test_expect_success IPFS,CLUSTER "export + cleanup + import == noop (crdt)" ' cluster_start && sleep 5 && ipfs-cluster-ctl pin ls "$cid" | grep -q "$cid" && ipfs-cluster-ctl status "$cid" | grep -q -i "PINNED" && - [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq ". | length")" ] + [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq -n "[inputs] | length")" ] ' cluster_kill @@ -42,17 +42,17 @@ test_expect_success IPFS,CLUSTER "state cleanup refreshes state on restart (raft ipfs-cluster-ctl pin add "$cid" && sleep 5 && ipfs-cluster-ctl pin ls "$cid" | grep -q "$cid" && ipfs-cluster-ctl status "$cid" | grep -q -i "PINNED" && - [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq ". | length")" ] && + [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq -n "[inputs] | length")" ] && cluster_kill && sleep 5 && ipfs-cluster-service --config "test-config" state cleanup -f && cluster_start && sleep 5 && - [ 0 -eq "$(ipfs-cluster-ctl --enc=json status | jq ". | length")" ] + [ 0 -eq "$(ipfs-cluster-ctl --enc=json status | jq -n "[inputs] | length")" ] ' test_expect_success IPFS,CLUSTER "export + cleanup + import == noop (raft)" ' cid=`docker exec ipfs sh -c "echo test_54 | ipfs add -q"` && ipfs-cluster-ctl pin add "$cid" && sleep 5 && - [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq ". | length")" ] && + [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq -n "[inputs] | length")" ] && cluster_kill && sleep 5 && ipfs-cluster-service --config "test-config" state export -f import.json && ipfs-cluster-service --config "test-config" state cleanup -f && @@ -60,7 +60,7 @@ test_expect_success IPFS,CLUSTER "export + cleanup + import == noop (raft)" ' cluster_start && sleep 5 && ipfs-cluster-ctl pin ls "$cid" | grep -q "$cid" && ipfs-cluster-ctl status "$cid" | grep -q -i "PINNED" && - [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq ". | length")" ] + [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq -n "[inputs] | length")" ] ' diff --git a/state/dsstate/datastore.go b/state/dsstate/datastore.go index 2d59ef79..82e340f8 100644 --- a/state/dsstate/datastore.go +++ b/state/dsstate/datastore.go @@ -4,6 +4,7 @@ package dsstate import ( "context" + "fmt" "io" "github.com/ipfs/ipfs-cluster/api" @@ -122,9 +123,11 @@ func (st *State) Has(ctx context.Context, c cid.Cid) (bool, error) { return ok, nil } -// List returns the unsorted list of all Pins that have been added to the -// datastore. -func (st *State) List(ctx context.Context) (<-chan api.Pin, error) { +// List sends all the pins on the pinset on the given channel. +// Returns and closes channel when done. +func (st *State) List(ctx context.Context, out chan<- api.Pin) error { + defer close(out) + _, span := trace.StartSpan(ctx, "state/dsstate/List") defer span.End() @@ -134,52 +137,49 @@ func (st *State) List(ctx context.Context) (<-chan api.Pin, error) { results, err := st.dsRead.Query(ctx, q) if err != nil { - return nil, err + return err } - pinsCh := make(chan api.Pin, 1024) - go func() { - defer close(pinsCh) + defer results.Close() - defer results.Close() - - total := 0 - for r := range results.Next() { - // Abort if we shutdown. - select { - case <-ctx.Done(): - logger.Warningf("Full pinset listing aborted: %s", ctx.Err()) - return - default: - } - if r.Error != nil { - logger.Errorf("error in query result: %s", r.Error) - return - } - k := ds.NewKey(r.Key) - ci, err := st.unkey(k) - if err != nil { - logger.Warn("bad key (ignoring). key: ", k, "error: ", err) - continue - } - - p, err := st.deserializePin(ci, r.Value) - if err != nil { - logger.Errorf("error deserializing pin (%s): %s", r.Key, err) - continue - } - pinsCh <- p - - if total > 0 && total%500000 == 0 { - logger.Infof("Full pinset listing in progress: %d pins so far", total) - } - total++ + total := 0 + for r := range results.Next() { + // Abort if we shutdown. + select { + case <-ctx.Done(): + err = fmt.Errorf("full pinset listing aborted: %w", ctx.Err()) + logger.Warning(err) + return err + default: } - if total >= 500000 { - logger.Infof("Full pinset listing finished: %d pins", total) + if r.Error != nil { + err := fmt.Errorf("error in query result: %w", r.Error) + logger.Error(err) + return err + } + k := ds.NewKey(r.Key) + ci, err := st.unkey(k) + if err != nil { + logger.Warn("bad key (ignoring). key: ", k, "error: ", err) + continue } - }() - return pinsCh, nil + p, err := st.deserializePin(ci, r.Value) + if err != nil { + logger.Errorf("error deserializing pin (%s): %s", r.Key, err) + continue + } + out <- p + + if total > 0 && total%500000 == 0 { + logger.Infof("Full pinset listing in progress: %d pins so far", total) + } + total++ + } + if total >= 500000 { + logger.Infof("Full pinset listing finished: %d pins", total) + } + + return nil } // Migrate migrates an older state version to the current one. diff --git a/state/dsstate/datastore_test.go b/state/dsstate/datastore_test.go index c79d2b33..6ece8c54 100644 --- a/state/dsstate/datastore_test.go +++ b/state/dsstate/datastore_test.go @@ -93,10 +93,13 @@ func TestList(t *testing.T) { }() st := newState(t) st.Add(ctx, c) - pinCh, err := st.List(ctx) - if err != nil { - t.Fatal(err) - } + out := make(chan api.Pin) + go func() { + err := st.List(ctx, out) + if err != nil { + t.Error(err) + } + }() ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -104,7 +107,7 @@ func TestList(t *testing.T) { var list0 api.Pin for { select { - case p, ok := <-pinCh: + case p, ok := <-out: if !ok && !list0.Cid.Defined() { t.Fatal("should have read list0 first") } diff --git a/state/empty.go b/state/empty.go index c8833120..64523a5b 100644 --- a/state/empty.go +++ b/state/empty.go @@ -10,10 +10,9 @@ import ( type empty struct{} -func (e *empty) List(ctx context.Context) (<-chan api.Pin, error) { - ch := make(chan api.Pin) - close(ch) - return ch, nil +func (e *empty) List(ctx context.Context, out chan<- api.Pin) error { + close(out) + return nil } func (e *empty) Has(ctx context.Context, c cid.Cid) (bool, error) { diff --git a/state/interface.go b/state/interface.go index 6b060bc2..4de202eb 100644 --- a/state/interface.go +++ b/state/interface.go @@ -34,7 +34,7 @@ type State interface { // ReadOnly represents the read side of a State. type ReadOnly interface { // List lists all the pins in the state. - List(context.Context) (<-chan api.Pin, error) + List(context.Context, chan<- api.Pin) error // Has returns true if the state is holding information for a Cid. Has(context.Context, cid.Cid) (bool, error) // Get returns the information attacthed to this pin, if any. If the diff --git a/test/ipfs_mock.go b/test/ipfs_mock.go index e075bdab..1b8edf3e 100644 --- a/test/ipfs_mock.go +++ b/test/ipfs_mock.go @@ -58,7 +58,7 @@ type mockPinType struct { Type string } -type mockPinLsResp struct { +type mockPinLsAllResp struct { Keys map[string]mockPinType } @@ -268,19 +268,35 @@ func (m *IpfsMock) handler(w http.ResponseWriter, r *http.Request) { j, _ := json.Marshal(resp) w.Write(j) case "pin/ls": + query := r.URL.Query() + stream := query.Get("stream") == "true" + arg, ok := extractCid(r.URL) if !ok { - rMap := make(map[string]mockPinType) - pins, err := m.pinMap.List(ctx) - if err != nil { - goto ERROR + pins := make(chan api.Pin, 10) + + go func() { + m.pinMap.List(ctx, pins) + }() + + if stream { + for p := range pins { + j, _ := json.Marshal(api.IPFSPinInfo{ + Cid: api.Cid(p.Cid), + Type: p.Mode.ToIPFSPinStatus(), + }) + w.Write(j) + } + break + } else { + rMap := make(map[string]mockPinType) + for p := range pins { + rMap[p.Cid.String()] = mockPinType{p.Mode.String()} + } + j, _ := json.Marshal(mockPinLsAllResp{rMap}) + w.Write(j) + break } - for p := range pins { - rMap[p.Cid.String()] = mockPinType{p.Mode.String()} - } - j, _ := json.Marshal(mockPinLsResp{rMap}) - w.Write(j) - break } cidStr := arg @@ -301,16 +317,28 @@ func (m *IpfsMock) handler(w http.ResponseWriter, r *http.Request) { return } - if c.Equals(Cid4) { - // this a v1 cid. Do not return default-base32 but base58btc encoding of it - w.Write([]byte(`{ "Keys": { "zCT5htkdztJi3x4zBNHo8TRvGHPLTdHUdCLKgTGMgQcRKSLoWxK1": { "Type": "recursive" }}}`)) - return + if stream { + if c.Equals(Cid4) { + // this a v1 cid. Do not return default-base32 but base58btc encoding of it + w.Write([]byte(`{ "Cid": "zCT5htkdztJi3x4zBNHo8TRvGHPLTdHUdCLKgTGMgQcRKSLoWxK1", "Type": "recursive" }`)) + break + } + j, _ := json.Marshal(api.IPFSPinInfo{ + Cid: api.Cid(pinObj.Cid), + Type: pinObj.Mode.ToIPFSPinStatus(), + }) + w.Write(j) + } else { + if c.Equals(Cid4) { + // this a v1 cid. Do not return default-base32 but base58btc encoding of it + w.Write([]byte(`{ "Keys": { "zCT5htkdztJi3x4zBNHo8TRvGHPLTdHUdCLKgTGMgQcRKSLoWxK1": { "Type": "recursive" }}}`)) + break + } + rMap := make(map[string]mockPinType) + rMap[cidStr] = mockPinType{pinObj.Mode.String()} + j, _ := json.Marshal(mockPinLsAllResp{rMap}) + w.Write(j) } - rMap := make(map[string]mockPinType) - rMap[cidStr] = mockPinType{pinObj.Mode.String()} - j, _ := json.Marshal(mockPinLsResp{rMap}) - w.Write(j) - case "swarm/connect": arg, ok := extractCid(r.URL) if !ok { @@ -424,10 +452,10 @@ func (m *IpfsMock) handler(w http.ResponseWriter, r *http.Request) { case "repo/stat": sizeOnly := r.URL.Query().Get("size-only") - pinsCh, err := m.pinMap.List(ctx) - if err != nil { - goto ERROR - } + pinsCh := make(chan api.Pin, 10) + go func() { + m.pinMap.List(ctx, pinsCh) + }() var pins []api.Pin for p := range pinsCh { diff --git a/test/rpc_api_mock.go b/test/rpc_api_mock.go index 9722809f..8d358c56 100644 --- a/test/rpc_api_mock.go +++ b/test/rpc_api_mock.go @@ -34,8 +34,8 @@ func NewMockRPCClient(t testing.TB) *rpc.Client { // NewMockRPCClientWithHost returns a mock ipfs-cluster RPC server // initialized with a given host. func NewMockRPCClientWithHost(t testing.TB, h host.Host) *rpc.Client { - s := rpc.NewServer(h, "mock") - c := rpc.NewClientWithServer(h, "mock", s) + s := rpc.NewServer(h, "mock", rpc.WithStreamBufferSize(1024)) + c := rpc.NewClientWithServer(h, "mock", s, rpc.WithMultiStreamBufferSize(1024)) err := s.RegisterName("Cluster", &mockCluster{}) if err != nil { t.Fatal(err) @@ -230,7 +230,10 @@ func (mock *mockCluster) ConnectGraph(ctx context.Context, in struct{}, out *api return nil } -func (mock *mockCluster) StatusAll(ctx context.Context, in api.TrackerStatus, out *[]api.GlobalPinInfo) error { +func (mock *mockCluster) StatusAll(ctx context.Context, in <-chan api.TrackerStatus, out chan<- api.GlobalPinInfo) error { + defer close(out) + filter := <-in + pid := peer.Encode(PeerID1) gPinInfos := []api.GlobalPinInfo{ { @@ -272,23 +275,21 @@ func (mock *mockCluster) StatusAll(ctx context.Context, in api.TrackerStatus, ou // a single peer, we will not have an entry for the cid at all. for _, gpi := range gPinInfos { for id, pi := range gpi.PeerMap { - if !in.Match(pi.Status) { + if !filter.Match(pi.Status) { delete(gpi.PeerMap, id) } } } - filtered := make([]api.GlobalPinInfo, 0, len(gPinInfos)) for _, gpi := range gPinInfos { if len(gpi.PeerMap) > 0 { - filtered = append(filtered, gpi) + out <- gpi } } - *out = filtered return nil } -func (mock *mockCluster) StatusAllLocal(ctx context.Context, in api.TrackerStatus, out *[]api.PinInfo) error { +func (mock *mockCluster) StatusAllLocal(ctx context.Context, in <-chan api.TrackerStatus, out chan<- api.PinInfo) error { return (&mockPinTracker{}).StatusAll(ctx, in, out) } @@ -324,11 +325,14 @@ func (mock *mockCluster) StatusLocal(ctx context.Context, in cid.Cid, out *api.P return (&mockPinTracker{}).Status(ctx, in, out) } -func (mock *mockCluster) RecoverAll(ctx context.Context, in struct{}, out *[]api.GlobalPinInfo) error { - return mock.StatusAll(ctx, api.TrackerStatusUndefined, out) +func (mock *mockCluster) RecoverAll(ctx context.Context, in <-chan struct{}, out chan<- api.GlobalPinInfo) error { + f := make(chan api.TrackerStatus, 1) + f <- api.TrackerStatusUndefined + close(f) + return mock.StatusAll(ctx, f, out) } -func (mock *mockCluster) RecoverAllLocal(ctx context.Context, in struct{}, out *[]api.PinInfo) error { +func (mock *mockCluster) RecoverAllLocal(ctx context.Context, in <-chan struct{}, out chan<- api.PinInfo) error { return (&mockPinTracker{}).RecoverAll(ctx, in, out) } @@ -421,7 +425,10 @@ func (mock *mockPinTracker) Untrack(ctx context.Context, in api.Pin, out *struct return nil } -func (mock *mockPinTracker) StatusAll(ctx context.Context, in api.TrackerStatus, out *[]api.PinInfo) error { +func (mock *mockPinTracker) StatusAll(ctx context.Context, in <-chan api.TrackerStatus, out chan<- api.PinInfo) error { + defer close(out) + filter := <-in + pinInfos := []api.PinInfo{ { Cid: Cid1, @@ -440,14 +447,11 @@ func (mock *mockPinTracker) StatusAll(ctx context.Context, in api.TrackerStatus, }, }, } - filtered := make([]api.PinInfo, 0, len(pinInfos)) for _, pi := range pinInfos { - if in.Match(pi.Status) { - filtered = append(filtered, pi) + if filter.Match(pi.Status) { + out <- pi } } - - *out = filtered return nil } @@ -467,8 +471,8 @@ func (mock *mockPinTracker) Status(ctx context.Context, in cid.Cid, out *api.Pin return nil } -func (mock *mockPinTracker) RecoverAll(ctx context.Context, in struct{}, out *[]api.PinInfo) error { - *out = make([]api.PinInfo, 0) +func (mock *mockPinTracker) RecoverAll(ctx context.Context, in <-chan struct{}, out chan<- api.PinInfo) error { + close(out) return nil } @@ -534,12 +538,10 @@ func (mock *mockIPFSConnector) PinLsCid(ctx context.Context, in api.Pin, out *ap return nil } -func (mock *mockIPFSConnector) PinLs(ctx context.Context, in string, out *map[string]api.IPFSPinStatus) error { - m := map[string]api.IPFSPinStatus{ - Cid1.String(): api.IPFSPinStatusRecursive, - Cid3.String(): api.IPFSPinStatusRecursive, - } - *out = m +func (mock *mockIPFSConnector) PinLs(ctx context.Context, in <-chan []string, out chan<- api.IPFSPinInfo) error { + out <- api.IPFSPinInfo{Cid: api.Cid(Cid1), Type: api.IPFSPinStatusRecursive} + out <- api.IPFSPinInfo{Cid: api.Cid(Cid3), Type: api.IPFSPinStatusRecursive} + close(out) return nil } diff --git a/test/sharding.go b/test/sharding.go index 5f85b0a4..fbf8ec3b 100644 --- a/test/sharding.go +++ b/test/sharding.go @@ -300,7 +300,7 @@ func (d *MockDAGService) Get(ctx context.Context, cid cid.Cid) (format.Node, err if n, ok := d.Nodes[cid]; ok { return n, nil } - return nil, format.ErrNotFound + return nil, format.ErrNotFound{Cid: cid} } // GetMany reads many nodes. @@ -312,7 +312,7 @@ func (d *MockDAGService) GetMany(ctx context.Context, cids []cid.Cid) <-chan *fo if n, ok := d.Nodes[c]; ok { out <- &format.NodeOption{Node: n} } else { - out <- &format.NodeOption{Err: format.ErrNotFound} + out <- &format.NodeOption{Err: format.ErrNotFound{Cid: c}} } } close(out)