ipfs-cluster/api/pinsvcapi/pinsvc/pinsvc.go
Hector Sanjuan 9b9d76f92d Pinset streaming and method type revamp
This commit introduces the new go-libp2p-gorpc streaming capabilities for
Cluster. The main aim is to work towards heavily reducing memory usage when
working with very large pinsets.

As a side-effect, it takes the chance to revampt all types for all public
methods so that pointers to static what should be static objects are not used
anymore. This should heavily reduce heap allocations and GC activity.

The main change is that state.List now returns a channel from which to read
the pins, rather than pins being all loaded into a huge slice.

Things reading pins have been all updated to iterate on the channel rather
than on the slice. The full pinset is no longer fully loaded onto memory for
things that run regularly like StateSync().

Additionally, the /allocations endpoint of the rest API no longer returns an
array of pins, but rather streams json-encoded pin objects directly. This
change has extended to the restapi client (which puts pins into a channel as
they arrive) and to ipfs-cluster-ctl.

There are still pending improvements like StatusAll() calls which should also
stream responses, and specially BlockPut calls which should stream blocks
directly into IPFS on a single call.

These are coming up in future commits.
2022-03-19 03:02:55 +01:00

307 lines
7.6 KiB
Go

// Package pinsvc contains type definitions for the Pinning Services API
package pinsvc
import (
"encoding/json"
"errors"
"fmt"
"net/url"
"strconv"
"strings"
"time"
"github.com/ipfs/go-cid"
types "github.com/ipfs/ipfs-cluster/api"
)
func init() {
// intialize trackerStatusString
stringStatus = make(map[string]Status)
for k, v := range statusString {
stringStatus[v] = k
}
}
// APIError is returned by the API as a body when an error
// occurs. It implements the error interface.
type APIError struct {
Reason string `json:"reason"`
Details string `json:"details"`
}
func (apiErr APIError) Error() string {
return apiErr.Reason
}
// PinName is a string limited to 255 chars when serializing JSON.
type PinName string
// MarshalJSON converts the string to JSON.
func (pname PinName) MarshalJSON() ([]byte, error) {
return json.Marshal(string(pname))
}
// UnmarshalJSON reads the JSON string and errors if over 256 chars.
func (pname *PinName) UnmarshalJSON(data []byte) error {
if len(data) > 257 { // "a_string" 255 + 2 for quotes
return errors.New("pin name is over 255 chars")
}
var v string
err := json.Unmarshal(data, &v)
*pname = PinName(v)
return err
}
// Pin contains basic information about a Pin and pinning options.
type Pin struct {
Cid string `json:"cid"` // a cid.Cid does not json properly
Name PinName `json:"name"`
Origins []types.Multiaddr `json:"origins"`
Meta map[string]string `json:"meta"`
}
// Defined returns if the pinis empty (Cid not set).
func (p Pin) Defined() bool {
return p.Cid != ""
}
// MatchesName returns in a pin status matches a name option with a given
// match strategy.
func (p Pin) MatchesName(nameOpt string, strategy MatchingStrategy) bool {
if nameOpt == "" {
return true
}
name := string(p.Name)
switch strategy {
case MatchingStrategyUndefined:
return true
case MatchingStrategyExact:
return nameOpt == name
case MatchingStrategyIexact:
return strings.EqualFold(name, nameOpt)
case MatchingStrategyPartial:
return strings.Contains(name, nameOpt)
case MatchingStrategyIpartial:
return strings.Contains(strings.ToLower(name), strings.ToLower(nameOpt))
default:
return true
}
}
// MatchesMeta returns true if the pin status metadata matches the given. The
// metadata should have all the keys in the given metaOpts and the values
// should, be the same (metadata map includes metaOpts).
func (p Pin) MatchesMeta(metaOpts map[string]string) bool {
for k, v := range metaOpts {
if p.Meta[k] != v {
return false
}
}
return true
}
// Status represents a pin status, which defines the current state of the pin
// in the system.
type Status int
// Values for the Status type.
const (
StatusUndefined Status = 0
StatusQueued = 1 << iota
StatusPinned
StatusPinning
StatusFailed
)
var statusString = map[Status]string{
StatusUndefined: "undefined",
StatusQueued: "queued",
StatusPinned: "pinned",
StatusPinning: "pinning",
StatusFailed: "failed",
}
// values autofilled in init()
var stringStatus map[string]Status
// String converts a Status into a readable string.
// If the given Status is a filter (with several
// bits set), it will return a comma-separated list.
func (st Status) String() string {
var values []string
// simple and known composite values
if v, ok := statusString[st]; ok {
return v
}
// other filters
for k, v := range statusString {
if st&k > 0 {
values = append(values, v)
}
}
return strings.Join(values, ",")
}
// Match returns true if the tracker status matches the given filter.
func (st Status) Match(filter Status) bool {
return filter == StatusUndefined ||
st == StatusUndefined ||
st&filter > 0
}
// MarshalJSON uses the string representation of Status for JSON
// encoding.
func (st Status) MarshalJSON() ([]byte, error) {
return json.Marshal(st.String())
}
// UnmarshalJSON sets a tracker status from its JSON representation.
func (st *Status) UnmarshalJSON(data []byte) error {
var v string
err := json.Unmarshal(data, &v)
if err != nil {
return err
}
*st = StatusFromString(v)
return nil
}
// StatusFromString parses a string and returns the matching
// Status value. The string can be a comma-separated list
// representing a Status filter. Unknown status names are
// ignored.
func StatusFromString(str string) Status {
values := strings.Split(strings.Replace(str, " ", "", -1), ",")
status := StatusUndefined
for _, v := range values {
st, ok := stringStatus[v]
if ok {
status |= st
}
}
return status
}
// MatchingStrategy defines a type of match for filtering pin lists.
type MatchingStrategy int
// Values for MatchingStrategy.
const (
MatchingStrategyUndefined MatchingStrategy = iota
MatchingStrategyExact
MatchingStrategyIexact
MatchingStrategyPartial
MatchingStrategyIpartial
)
// MatchingStrategyFromString converts a string to its MatchingStrategy value.
func MatchingStrategyFromString(str string) MatchingStrategy {
switch str {
case "exact":
return MatchingStrategyExact
case "iexact":
return MatchingStrategyIexact
case "partial":
return MatchingStrategyPartial
case "ipartial":
return MatchingStrategyIpartial
default:
return MatchingStrategyUndefined
}
}
// PinStatus provides information about a Pin stored by the Pinning API.
type PinStatus struct {
RequestID string `json:"requestid"`
Status Status `json:"status"`
Created time.Time `json:"created"`
Pin Pin `json:"pin"`
Delegates []types.Multiaddr `json:"delegates"`
Info map[string]string `json:"info"`
}
// PinList is the result of a call to List pins
type PinList struct {
Count int `json:"count"`
Results []PinStatus `json:"results"`
}
// ListOptions represents possible options given to the List endpoint.
type ListOptions struct {
Cids []cid.Cid
Name string
MatchingStrategy MatchingStrategy
Status Status
Before time.Time
After time.Time
Limit int
Meta map[string]string
}
func (lo *ListOptions) FromQuery(q url.Values) error {
cidq := q.Get("cid")
if len(cidq) > 0 {
for _, cstr := range strings.Split(cidq, ",") {
c, err := cid.Decode(cstr)
if err != nil {
return fmt.Errorf("error decoding cid %s: %w", cstr, err)
}
lo.Cids = append(lo.Cids, c)
}
}
n := q.Get("name")
if len(n) > 255 {
return fmt.Errorf("error in 'name' query param: longer than 255 chars")
}
lo.Name = n
lo.MatchingStrategy = MatchingStrategyFromString(q.Get("match"))
if lo.MatchingStrategy == MatchingStrategyUndefined {
lo.MatchingStrategy = MatchingStrategyExact // default
}
statusStr := q.Get("status")
lo.Status = StatusFromString(statusStr)
// FIXME: This is a bit lazy, as "invalidxx,pinned" would result in a
// valid "pinned" filter.
if statusStr != "" && lo.Status == StatusUndefined {
return fmt.Errorf("error decoding 'status' query param: no valid filter")
}
if bef := q.Get("before"); bef != "" {
err := lo.Before.UnmarshalText([]byte(bef))
if err != nil {
return fmt.Errorf("error decoding 'before' query param: %s: %w", bef, err)
}
}
if after := q.Get("after"); after != "" {
err := lo.After.UnmarshalText([]byte(after))
if err != nil {
return fmt.Errorf("error decoding 'after' query param: %s: %w", after, err)
}
}
if v := q.Get("limit"); v != "" {
lim, err := strconv.ParseUint(v, 10, 64)
if err != nil {
return fmt.Errorf("error parsing 'limit' query param: %s: %w", v, err)
}
lo.Limit = int(lim)
}
if meta := q.Get("meta"); meta != "" {
err := json.Unmarshal([]byte(meta), &lo.Meta)
if err != nil {
return fmt.Errorf("error unmarshalling 'meta' query param: %s: %w", meta, err)
}
}
return nil
}