cascade/ipfs-cluster
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pinsvc: fixes and testing for List endpoint

Browse Source
This commit is contained in:
Hector Sanjuan 2022-03-11 12:58:12 +01:00
parent fe4c0f61c9
commit f8812b3af7
4 changed files with 229 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
api/pinsvcapi/pinsvc/pinsvc.go
View File

@ -40,6 +40,43 @@ type Pin struct {
Meta map[string]string `json:"meta"`
}
// MatchesName returns in a pin status matches a name option with a given
// match strategy.
func (p Pin) MatchesName(nameOpt string, strategy MatchingStrategy) bool {
if nameOpt == "" {
return true
}
switch strategy {
case MatchingStrategyUndefined:
return true
case MatchingStrategyExact:
return nameOpt == p.Name
case MatchingStrategyIexact:
return strings.EqualFold(p.Name, nameOpt)
case MatchingStrategyPartial:
return strings.Contains(p.Name, nameOpt)
case MatchingStrategyIpartial:
return strings.Contains(strings.ToLower(p.Name), strings.ToLower(nameOpt))
default:
return true
}
return false
}
// MatchesMeta returns true if the pin status metadata matches the given. The
// metadata should have all the keys in the given metaOpts and the values
// should, be the same (metadata map includes metaOpts).
func (p Pin) MatchesMeta(metaOpts map[string]string) bool {
for k, v := range metaOpts {
if p.Meta[k] != v {
return false
}
}
return true
}
// Status represents a pin status.
type Status int
// Status values
@ -140,30 +177,30 @@ type PinList struct {
}
// Match defines a type of match for filtering pin lists.
type Match int
type MatchingStrategy int
// Values for matches.
const (
MatchUndefined Match = iota
MatchExact
MatchIexact
MatchPartial
MatchIpartial
MatchingStrategyUndefined MatchingStrategy = iota
MatchingStrategyExact
MatchingStrategyIexact
MatchingStrategyPartial
MatchingStrategyIpartial
)
// MatchFromString converts a string to its Match value.
func MatchFromString(str string) Match {
// MatchingStrategyFromString converts a string to its MatchingStrategy value.
func MatchingStrategyFromString(str string) MatchingStrategy {
switch str {
case "exact":
return MatchExact
return MatchingStrategyExact
case "iexact":
return MatchIexact
return MatchingStrategyIexact
case "partial":
return MatchPartial
return MatchingStrategyPartial
case "ipartial":
return MatchIpartial
return MatchingStrategyIpartial
default:
return MatchUndefined
return MatchingStrategyUndefined
}
}
@ -171,7 +208,7 @@ func MatchFromString(str string) Match {
type ListOptions struct {
Cids []cid.Cid
Name string
Match Match
MatchingStrategy MatchingStrategy
Status Status
Before time.Time
After time.Time
@ -191,9 +228,21 @@ func (lo *ListOptions) FromQuery(q url.Values) error {
}
}
lo.Name = q.Get("name")
lo.Match = MatchFromString(q.Get("match"))
lo.Status = StatusFromString(q.Get("status"))
n := q.Get("name")
if len(n) > 255 {
return fmt.Errorf("error in 'name' query param: longer than 255 chars")
}
lo.Name = n
lo.MatchingStrategy = MatchingStrategyFromString(q.Get("match"))
if lo.MatchingStrategy == MatchingStrategyUndefined {
lo.MatchingStrategy = MatchingStrategyExact // default
}
statusStr := q.Get("status")
lo.Status = StatusFromString(statusStr)
if statusStr != "" && lo.Status == StatusUndefined {
return fmt.Errorf("error decoding 'status' query param: no valid filter")
}
if bef := q.Get("before"); bef != "" {
err := lo.Before.UnmarshalText([]byte(bef))

51
api/pinsvcapi/pinsvcapi.go
View File

@ -11,6 +11,7 @@ import (
"encoding/json"
"errors"
"net/http"
"sync"
"time"
"github.com/gorilla/mux"
@ -19,6 +20,7 @@ import (
"github.com/ipfs/ipfs-cluster/api/common"
"github.com/ipfs/ipfs-cluster/api/pinsvcapi/pinsvc"
"github.com/ipfs/ipfs-cluster/state"
"go.uber.org/multierr"
logging "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p-core/host"
@ -37,9 +39,9 @@ func trackerStatusToSvcStatus(st types.TrackerStatus) pinsvc.Status {
case st.Match(types.TrackerStatusPinQueued):
return pinsvc.StatusQueued
case st.Match(types.TrackerStatusPinning):
return pinsvc.StatusPinned
case st.Match(types.TrackerStatusPinned):
return pinsvc.StatusPinning
case st.Match(types.TrackerStatusPinned):
return pinsvc.StatusPinned
default:
return pinsvc.StatusUndefined
}
@ -329,19 +331,42 @@ func (api *API) listPins(w http.ResponseWriter, r *http.Request) {
var pinList pinsvc.PinList
if len(opts.Cids) > 0 {
for i, c := range opts.Cids {
st, gpi, err := api.getPinObject(r.Context(), c)
if err != nil {
api.SendResponse(w, common.SetStatusAutomatically, err, nil)
return
// copy approach from restapi
type statusResult struct {
st pinsvc.PinStatus
err error
}
if !gpi.Match(tst) {
continue
stCh := make(chan statusResult, len(opts.Cids))
var wg sync.WaitGroup
wg.Add(len(opts.Cids))
go func() {
wg.Wait()
close(stCh)
}()
for _, ci := range opts.Cids {
go func(c cid.Cid) {
defer wg.Done()
st, _, err := api.getPinObject(r.Context(), c)
stCh <- statusResult{st: st, err: err}
}(ci)
}
pinList.Results = append(pinList.Results, st)
var err error
i := 0
for stResult := range stCh {
pinList.Results = append(pinList.Results, stResult.st)
err = multierr.Append(err, stResult.err)
if i+1 == opts.Limit {
break
}
i++
}
if err != nil {
api.SendResponse(w, common.SetStatusAutomatically, err, nil)
return
}
} else {
var globalPinInfos []*types.GlobalPinInfo
@ -359,6 +384,12 @@ func (api *API) listPins(w http.ResponseWriter, r *http.Request) {
}
for i, gpi := range globalPinInfos {
st := globalPinInfoToSvcPinStatus(gpi.Cid.String(), *gpi)
if !st.Pin.MatchesName(opts.Name, opts.MatchingStrategy) {
continue
}
if !st.Pin.MatchesMeta(opts.Meta) {
continue
}
pinList.Results = append(pinList.Results, st)
if i+1 == opts.Limit {
break

104
api/pinsvcapi/pinsvcapi_test.go
View File

@ -5,6 +5,8 @@ import (
"testing"
"time"
"github.com/ipfs/ipfs-cluster/api/common/test"
"github.com/ipfs/ipfs-cluster/api/pinsvcapi/pinsvc"
clustertest "github.com/ipfs/ipfs-cluster/test"
libp2p "github.com/libp2p/go-libp2p"
@ -55,3 +57,105 @@ func testAPI(t *testing.T) *API {
return testAPIwithConfig(t, cfg, "basic")
}
func TestAPIStatusAllEndpoint(t *testing.T) {
ctx := context.Background()
svcapi := testAPI(t)
defer svcapi.Shutdown(ctx)
tf := func(t *testing.T, url test.URLFunc) {
var resp pinsvc.PinList
test.MakeGet(t, svcapi, url(svcapi)+"/pins", &resp)
// mockPinTracker returns 3 items for Cluster.StatusAll
if resp.Count != 3 {
t.Fatal("Count should be 3")
}
if len(resp.Results) != 3 {
t.Fatal("There should be 3 results")
}
results := resp.Results
if results[0].Pin.Cid != clustertest.Cid1.String() ||
results[1].Status != pinsvc.StatusPinning {
t.Errorf("unexpected statusAll resp: %+v", results)
}
// Test status filters
var resp2 pinsvc.PinList
test.MakeGet(t, svcapi, url(svcapi)+"/pins?status=pinning", &resp2)
// mockPinTracker calls pintracker.StatusAll which returns 2
// items.
if resp2.Count != 1 {
t.Errorf("unexpected statusAll+status=pinning resp:\n %+v", resp2)
}
var resp3 pinsvc.PinList
test.MakeGet(t, svcapi, url(svcapi)+"/pins?status=queued", &resp3)
if resp3.Count != 0 {
t.Errorf("unexpected statusAll+status=queued resp:\n %+v", resp3)
}
var resp4 pinsvc.PinList
test.MakeGet(t, svcapi, url(svcapi)+"/pins?status=pinned", &resp4)
if resp4.Count != 1 {
t.Errorf("unexpected statusAll+status=queued resp:\n %+v", resp4)
}
var resp5 pinsvc.PinList
test.MakeGet(t, svcapi, url(svcapi)+"/pins?status=failed", &resp5)
if resp5.Count != 1 {
t.Errorf("unexpected statusAll+status=queued resp:\n %+v", resp5)
}
var resp6 pinsvc.PinList
test.MakeGet(t, svcapi, url(svcapi)+"/pins?status=failed,pinned", &resp6)
if resp6.Count != 2 {
t.Errorf("unexpected statusAll+status=failed,pinned resp:\n %+v", resp6)
}
// Test with cids
var resp7 pinsvc.PinList
test.MakeGet(t, svcapi, url(svcapi)+"/pins?cid=QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmq,QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmb", &resp7)
if resp7.Count != 2 {
t.Errorf("unexpected statusAll+cids resp:\n %+v", resp7)
}
// Test with cids+limit
var resp8 pinsvc.PinList
test.MakeGet(t, svcapi, url(svcapi)+"/pins?cid=QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmq,QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmb&limit=1", &resp8)
if resp8.Count != 1 {
t.Errorf("unexpected statusAll+cids+limit resp:\n %+v", resp8)
}
// Test with limit
var resp9 pinsvc.PinList
test.MakeGet(t, svcapi, url(svcapi)+"/pins?limit=1", &resp9)
if resp9.Count != 1 {
t.Errorf("unexpected statusAll+limit=1 resp:\n %+v", resp9)
}
// Test with name-match
var resp10 pinsvc.PinList
test.MakeGet(t, svcapi, url(svcapi)+"/pins?name=C&match=ipartial", &resp10)
if resp10.Count != 1 {
t.Errorf("unexpected statusAll+name resp:\n %+v", resp10)
}
// Test with meta-match
var resp11 pinsvc.PinList
test.MakeGet(t, svcapi, url(svcapi)+`/pins?meta={"ccc":"3c"}`, &resp11)
if resp11.Count != 1 {
t.Errorf("unexpected statusAll+meta resp:\n %+v", resp11)
}
var errorResp pinsvc.APIError
test.MakeGet(t, svcapi, url(svcapi)+"/pins?status=invalid", &errorResp)
if errorResp.Reason == "" {
t.Errorf("expected an error: %s", errorResp.Reason)
}
}
test.BothEndpoints(t, tf)
}

6
test/rpc_api_mock.go
View File

@ -228,6 +228,7 @@ func (mock *mockCluster) StatusAll(ctx context.Context, in api.TrackerStatus, ou
gPinInfos := []*api.GlobalPinInfo{
{
Cid: Cid1,
Name: "aaa",
PeerMap: map[string]api.PinInfoShort{
pid: {
Status: api.TrackerStatusPinned,
@ -237,6 +238,7 @@ func (mock *mockCluster) StatusAll(ctx context.Context, in api.TrackerStatus, ou
},
{
Cid: Cid2,
Name: "bbb",
PeerMap: map[string]api.PinInfoShort{
pid: {
Status: api.TrackerStatusPinning,
@ -246,6 +248,10 @@ func (mock *mockCluster) StatusAll(ctx context.Context, in api.TrackerStatus, ou
},
{
Cid: Cid3,
Name: "ccc",
Metadata: map[string]string{
"ccc": "3c",
},
PeerMap: map[string]api.PinInfoShort{
pid: {
Status: api.TrackerStatusPinError,