2016-12-06 21:29:59 +00:00
|
|
|
package ipfscluster
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2016-12-19 17:35:24 +00:00
|
|
|
"errors"
|
|
|
|
"fmt"
|
2016-12-06 21:29:59 +00:00
|
|
|
"sync"
|
2016-12-15 18:08:46 +00:00
|
|
|
"time"
|
2016-12-06 21:29:59 +00:00
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
peer "github.com/libp2p/go-libp2p-peer"
|
2016-12-06 21:29:59 +00:00
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
cid "github.com/ipfs/go-cid"
|
2016-12-06 21:29:59 +00:00
|
|
|
)
|
|
|
|
|
2016-12-15 18:08:46 +00:00
|
|
|
// A Pin or Unpin operation will be considered failed
|
|
|
|
// if the Cid has stayed in Pinning or Unpinning state
|
|
|
|
// for longer than these values.
|
|
|
|
var (
|
|
|
|
PinningTimeout = 15 * time.Minute
|
|
|
|
UnpinningTimeout = 10 * time.Second
|
|
|
|
)
|
|
|
|
|
2016-12-06 21:29:59 +00:00
|
|
|
const (
|
|
|
|
PinError = iota
|
2016-12-07 16:21:29 +00:00
|
|
|
UnpinError
|
2016-12-06 21:29:59 +00:00
|
|
|
Pinned
|
|
|
|
Pinning
|
|
|
|
Unpinning
|
|
|
|
Unpinned
|
2016-12-19 17:35:24 +00:00
|
|
|
RemotePin
|
2016-12-06 21:29:59 +00:00
|
|
|
)
|
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
type GlobalPinInfo struct {
|
|
|
|
Cid *cid.Cid
|
|
|
|
Status map[peer.ID]PinInfo
|
2016-12-06 21:29:59 +00:00
|
|
|
}
|
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
// PinInfo holds information about local pins. PinInfo is
|
|
|
|
// serialized when requesting the Global status, therefore
|
|
|
|
// we cannot use *cid.Cid.
|
|
|
|
type PinInfo struct {
|
|
|
|
CidStr string
|
|
|
|
Peer peer.ID
|
|
|
|
IPFS IPFSStatus
|
|
|
|
TS time.Time
|
|
|
|
}
|
2016-12-06 21:29:59 +00:00
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
type IPFSStatus int
|
|
|
|
|
|
|
|
func (st IPFSStatus) String() string {
|
2016-12-15 18:08:46 +00:00
|
|
|
switch st {
|
|
|
|
case PinError:
|
|
|
|
return "pin_error"
|
|
|
|
case UnpinError:
|
|
|
|
return "unpin_error"
|
|
|
|
case Pinned:
|
|
|
|
return "pinned"
|
|
|
|
case Pinning:
|
|
|
|
return "pinning"
|
|
|
|
case Unpinning:
|
|
|
|
return "unpinning"
|
|
|
|
case Unpinned:
|
|
|
|
return "unpinned"
|
|
|
|
}
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
|
2016-12-06 21:29:59 +00:00
|
|
|
type MapPinTracker struct {
|
2016-12-12 12:54:58 +00:00
|
|
|
mux sync.Mutex
|
2016-12-19 17:35:24 +00:00
|
|
|
status map[string]PinInfo
|
2016-12-06 21:29:59 +00:00
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
ctx context.Context
|
|
|
|
rpcCh chan RPC
|
|
|
|
peerID peer.ID
|
2016-12-09 19:54:46 +00:00
|
|
|
|
2016-12-15 13:07:19 +00:00
|
|
|
shutdownLock sync.Mutex
|
|
|
|
shutdown bool
|
|
|
|
shutdownCh chan struct{}
|
|
|
|
wg sync.WaitGroup
|
2016-12-06 21:29:59 +00:00
|
|
|
}
|
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
func NewMapPinTracker(cfg *Config) *MapPinTracker {
|
2016-12-09 19:54:46 +00:00
|
|
|
ctx := context.Background()
|
2016-12-19 17:35:24 +00:00
|
|
|
|
|
|
|
pID, err := peer.IDB58Decode(cfg.ID)
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
|
2016-12-06 21:29:59 +00:00
|
|
|
mpt := &MapPinTracker{
|
2016-12-09 19:54:46 +00:00
|
|
|
ctx: ctx,
|
2016-12-19 17:35:24 +00:00
|
|
|
status: make(map[string]PinInfo),
|
|
|
|
rpcCh: make(chan RPC, RPCMaxQueue),
|
|
|
|
peerID: pID,
|
2016-12-15 13:07:19 +00:00
|
|
|
shutdownCh: make(chan struct{}),
|
2016-12-06 21:29:59 +00:00
|
|
|
}
|
2016-12-15 13:19:41 +00:00
|
|
|
logger.Info("starting MapPinTracker")
|
2016-12-15 13:07:19 +00:00
|
|
|
mpt.run()
|
2016-12-06 21:29:59 +00:00
|
|
|
return mpt
|
|
|
|
}
|
|
|
|
|
|
|
|
func (mpt *MapPinTracker) run() {
|
2016-12-15 13:07:19 +00:00
|
|
|
mpt.wg.Add(1)
|
|
|
|
go func() {
|
|
|
|
defer mpt.wg.Done()
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
defer cancel()
|
|
|
|
mpt.ctx = ctx
|
|
|
|
<-mpt.shutdownCh
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (mpt *MapPinTracker) Shutdown() error {
|
|
|
|
mpt.shutdownLock.Lock()
|
|
|
|
defer mpt.shutdownLock.Unlock()
|
|
|
|
|
|
|
|
if mpt.shutdown {
|
|
|
|
logger.Debug("already shutdown")
|
|
|
|
return nil
|
2016-12-06 21:29:59 +00:00
|
|
|
}
|
2016-12-15 13:07:19 +00:00
|
|
|
|
2016-12-15 13:19:41 +00:00
|
|
|
logger.Info("stopping MapPinTracker")
|
2016-12-15 13:07:19 +00:00
|
|
|
mpt.shutdownCh <- struct{}{}
|
|
|
|
mpt.wg.Wait()
|
|
|
|
mpt.shutdown = true
|
|
|
|
return nil
|
2016-12-06 21:29:59 +00:00
|
|
|
}
|
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
func (mpt *MapPinTracker) unsafeSet(c *cid.Cid, s IPFSStatus) {
|
2016-12-07 16:21:29 +00:00
|
|
|
if s == Unpinned {
|
|
|
|
delete(mpt.status, c.String())
|
|
|
|
}
|
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
mpt.status[c.String()] = PinInfo{
|
|
|
|
// cid: c,
|
|
|
|
CidStr: c.String(),
|
|
|
|
Peer: mpt.peerID,
|
|
|
|
IPFS: s,
|
|
|
|
TS: time.Now(),
|
2016-12-06 21:29:59 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
func (mpt *MapPinTracker) set(c *cid.Cid, s IPFSStatus) {
|
|
|
|
mpt.mux.Lock()
|
|
|
|
defer mpt.mux.Unlock()
|
|
|
|
mpt.unsafeSet(c, s)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (mpt *MapPinTracker) get(c *cid.Cid) PinInfo {
|
2016-12-07 16:21:29 +00:00
|
|
|
mpt.mux.Lock()
|
|
|
|
defer mpt.mux.Unlock()
|
|
|
|
p, ok := mpt.status[c.String()]
|
|
|
|
if !ok {
|
2016-12-19 17:35:24 +00:00
|
|
|
return PinInfo{
|
|
|
|
// cid: c,
|
|
|
|
CidStr: c.String(),
|
|
|
|
Peer: mpt.peerID,
|
|
|
|
IPFS: Unpinned,
|
|
|
|
TS: time.Now(),
|
2016-12-07 16:21:29 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return p
|
|
|
|
}
|
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
func (mpt *MapPinTracker) pin(c *cid.Cid) error {
|
|
|
|
ctx, cancel := context.WithCancel(mpt.ctx)
|
|
|
|
defer cancel()
|
2016-12-06 21:29:59 +00:00
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
mpt.set(c, Pinning)
|
|
|
|
resp := MakeRPC(ctx, mpt.rpcCh, NewRPC(IPFSPinRPC, c), true)
|
|
|
|
if resp.Error != nil {
|
|
|
|
mpt.set(c, PinError)
|
|
|
|
return resp.Error
|
|
|
|
}
|
|
|
|
mpt.set(c, Pinned)
|
|
|
|
return nil
|
2016-12-06 21:29:59 +00:00
|
|
|
}
|
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
func (mpt *MapPinTracker) unpin(c *cid.Cid) error {
|
|
|
|
ctx, cancel := context.WithCancel(mpt.ctx)
|
|
|
|
defer cancel()
|
2016-12-06 21:29:59 +00:00
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
mpt.set(c, Unpinning)
|
|
|
|
resp := MakeRPC(ctx, mpt.rpcCh, NewRPC(IPFSUnpinRPC, c), true)
|
|
|
|
if resp.Error != nil {
|
|
|
|
mpt.set(c, PinError)
|
|
|
|
return resp.Error
|
|
|
|
}
|
|
|
|
mpt.set(c, Unpinned)
|
|
|
|
return nil
|
2016-12-06 21:29:59 +00:00
|
|
|
}
|
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
func (mpt *MapPinTracker) Track(c *cid.Cid) error {
|
|
|
|
return mpt.pin(c)
|
2016-12-06 21:29:59 +00:00
|
|
|
}
|
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
func (mpt *MapPinTracker) Untrack(c *cid.Cid) error {
|
|
|
|
return mpt.unpin(c)
|
2016-12-06 21:29:59 +00:00
|
|
|
}
|
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
func (mpt *MapPinTracker) LocalStatusCid(c *cid.Cid) PinInfo {
|
2016-12-07 16:21:29 +00:00
|
|
|
return mpt.get(c)
|
2016-12-06 21:29:59 +00:00
|
|
|
}
|
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
func (mpt *MapPinTracker) LocalStatus() []PinInfo {
|
2016-12-06 21:29:59 +00:00
|
|
|
mpt.mux.Lock()
|
|
|
|
defer mpt.mux.Unlock()
|
2016-12-19 17:35:24 +00:00
|
|
|
pins := make([]PinInfo, 0, len(mpt.status))
|
2016-12-06 21:29:59 +00:00
|
|
|
for _, v := range mpt.status {
|
|
|
|
pins = append(pins, v)
|
|
|
|
}
|
|
|
|
return pins
|
|
|
|
}
|
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
func (mpt *MapPinTracker) GlobalStatus() ([]GlobalPinInfo, error) {
|
|
|
|
ctx, cancel := context.WithCancel(mpt.ctx)
|
|
|
|
defer cancel()
|
|
|
|
|
|
|
|
rpc := NewRPC(TrackerLocalStatusRPC, nil)
|
|
|
|
wrpc := NewRPC(BroadcastRPC, rpc)
|
|
|
|
resp := MakeRPC(ctx, mpt.rpcCh, wrpc, true)
|
|
|
|
if resp.Error != nil {
|
|
|
|
return nil, resp.Error
|
|
|
|
}
|
|
|
|
|
|
|
|
responses, ok := resp.Data.([]RPCResponse)
|
|
|
|
if !ok {
|
|
|
|
return nil, errors.New("unexpected responses format")
|
|
|
|
}
|
|
|
|
fullMap := make(map[string]GlobalPinInfo)
|
|
|
|
|
|
|
|
mergePins := func(pins []PinInfo) {
|
|
|
|
for _, p := range pins {
|
|
|
|
item, ok := fullMap[p.CidStr]
|
|
|
|
c, _ := cid.Decode(p.CidStr)
|
|
|
|
if !ok {
|
|
|
|
fullMap[p.CidStr] = GlobalPinInfo{
|
|
|
|
Cid: c,
|
|
|
|
Status: map[peer.ID]PinInfo{
|
|
|
|
p.Peer: p,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
item.Status[p.Peer] = p
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for _, r := range responses {
|
|
|
|
if r.Error != nil {
|
|
|
|
logger.Error("error in one of the broadcast responses: ", r.Error)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
pins, ok := r.Data.([]PinInfo)
|
|
|
|
if !ok {
|
|
|
|
return nil, fmt.Errorf("unexpected response format: %+v", r.Data)
|
|
|
|
}
|
|
|
|
mergePins(pins)
|
|
|
|
}
|
|
|
|
|
|
|
|
status := make([]GlobalPinInfo, 0, len(fullMap))
|
|
|
|
for _, v := range fullMap {
|
|
|
|
status = append(status, v)
|
|
|
|
}
|
|
|
|
return status, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (mpt *MapPinTracker) GlobalStatusCid(c *cid.Cid) (GlobalPinInfo, error) {
|
|
|
|
ctx, cancel := context.WithCancel(mpt.ctx)
|
|
|
|
defer cancel()
|
|
|
|
|
|
|
|
pin := GlobalPinInfo{
|
|
|
|
Cid: c,
|
|
|
|
Status: make(map[peer.ID]PinInfo),
|
|
|
|
}
|
|
|
|
|
|
|
|
rpc := NewRPC(TrackerLocalStatusCidRPC, c)
|
|
|
|
wrpc := NewRPC(BroadcastRPC, rpc)
|
|
|
|
resp := MakeRPC(ctx, mpt.rpcCh, wrpc, true)
|
|
|
|
if resp.Error != nil {
|
|
|
|
return pin, resp.Error
|
|
|
|
}
|
|
|
|
|
|
|
|
responses, ok := resp.Data.([]RPCResponse)
|
|
|
|
if !ok {
|
|
|
|
return pin, errors.New("unexpected responses format")
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, r := range responses {
|
|
|
|
if r.Error != nil {
|
|
|
|
return pin, r.Error
|
|
|
|
}
|
|
|
|
info, ok := r.Data.(PinInfo)
|
|
|
|
if !ok {
|
|
|
|
return pin, errors.New("unexpected response format")
|
|
|
|
}
|
|
|
|
pin.Status[info.Peer] = info
|
|
|
|
}
|
|
|
|
|
|
|
|
return pin, nil
|
|
|
|
}
|
|
|
|
|
2016-12-07 16:21:29 +00:00
|
|
|
func (mpt *MapPinTracker) Sync(c *cid.Cid) bool {
|
|
|
|
ctx, cancel := context.WithCancel(mpt.ctx)
|
|
|
|
defer cancel()
|
|
|
|
|
|
|
|
p := mpt.get(c)
|
|
|
|
|
2016-12-14 16:25:21 +00:00
|
|
|
// We assume errors will need a Recover() so we return true
|
2016-12-19 17:35:24 +00:00
|
|
|
if p.IPFS == PinError || p.IPFS == UnpinError {
|
2016-12-14 16:25:21 +00:00
|
|
|
return true
|
2016-12-07 16:21:29 +00:00
|
|
|
}
|
|
|
|
|
2016-12-15 18:08:46 +00:00
|
|
|
resp := MakeRPC(ctx, mpt.rpcCh, NewRPC(IPFSIsPinnedRPC, c), true)
|
2016-12-07 16:21:29 +00:00
|
|
|
if resp.Error != nil {
|
2016-12-19 17:35:24 +00:00
|
|
|
if p.IPFS == Pinned || p.IPFS == Pinning {
|
2016-12-07 16:21:29 +00:00
|
|
|
mpt.set(c, PinError)
|
|
|
|
return true
|
|
|
|
}
|
2016-12-19 17:35:24 +00:00
|
|
|
if p.IPFS == Unpinned || p.IPFS == Unpinning {
|
2016-12-07 16:21:29 +00:00
|
|
|
mpt.set(c, UnpinError)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
ipfsPinned, ok := resp.Data.(bool)
|
|
|
|
if !ok {
|
|
|
|
logger.Error("wrong type of IPFSIsPinnedRPC response")
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
if ipfsPinned {
|
2016-12-19 17:35:24 +00:00
|
|
|
switch p.IPFS {
|
2016-12-07 16:21:29 +00:00
|
|
|
case Pinned:
|
|
|
|
return false
|
|
|
|
case Pinning:
|
|
|
|
mpt.set(c, Pinned)
|
|
|
|
return true
|
|
|
|
case Unpinning:
|
2016-12-15 18:08:46 +00:00
|
|
|
if time.Since(p.TS) > UnpinningTimeout {
|
|
|
|
mpt.set(c, UnpinError)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
2016-12-07 16:21:29 +00:00
|
|
|
case Unpinned:
|
|
|
|
mpt.set(c, UnpinError)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
} else {
|
2016-12-19 17:35:24 +00:00
|
|
|
switch p.IPFS {
|
2016-12-07 16:21:29 +00:00
|
|
|
case Pinned:
|
|
|
|
mpt.set(c, PinError)
|
|
|
|
return true
|
|
|
|
case Pinning:
|
2016-12-15 18:08:46 +00:00
|
|
|
if time.Since(p.TS) > PinningTimeout {
|
|
|
|
mpt.set(c, PinError)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
2016-12-07 16:21:29 +00:00
|
|
|
case Unpinning:
|
|
|
|
mpt.set(c, Unpinned)
|
|
|
|
return true
|
|
|
|
case Unpinned:
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
func (mpt *MapPinTracker) Recover(c *cid.Cid) error {
|
|
|
|
p := mpt.get(c)
|
2016-12-19 17:35:24 +00:00
|
|
|
if p.IPFS != PinError && p.IPFS != UnpinError {
|
2016-12-07 16:21:29 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
if p.IPFS == PinError {
|
|
|
|
mpt.pin(c)
|
2016-12-07 16:21:29 +00:00
|
|
|
}
|
2016-12-19 17:35:24 +00:00
|
|
|
if p.IPFS == UnpinError {
|
|
|
|
mpt.unpin(c)
|
2016-12-07 16:21:29 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
func (mpt *MapPinTracker) SyncAll() []PinInfo {
|
|
|
|
var changedPins []PinInfo
|
|
|
|
pins := mpt.LocalStatus()
|
2016-12-07 16:21:29 +00:00
|
|
|
for _, p := range pins {
|
2016-12-19 17:35:24 +00:00
|
|
|
c, _ := cid.Decode(p.CidStr)
|
|
|
|
changed := mpt.Sync(c)
|
2016-12-07 16:21:29 +00:00
|
|
|
if changed {
|
|
|
|
changedPins = append(changedPins, p)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return changedPins
|
|
|
|
}
|
|
|
|
|
2016-12-19 17:35:24 +00:00
|
|
|
func (mpt *MapPinTracker) SyncState(cState State) []*cid.Cid {
|
2016-12-09 19:54:46 +00:00
|
|
|
clusterPins := cState.ListPins()
|
|
|
|
clusterMap := make(map[string]struct{})
|
|
|
|
// Make a map for faster lookup
|
|
|
|
for _, c := range clusterPins {
|
|
|
|
var a struct{}
|
|
|
|
clusterMap[c.String()] = a
|
|
|
|
}
|
|
|
|
var toRemove []*cid.Cid
|
|
|
|
var toAdd []*cid.Cid
|
2016-12-19 17:35:24 +00:00
|
|
|
var changed []*cid.Cid
|
2016-12-09 19:54:46 +00:00
|
|
|
mpt.mux.Lock()
|
|
|
|
|
2016-12-15 18:08:46 +00:00
|
|
|
// Collect items in the State not in the tracker
|
2016-12-09 19:54:46 +00:00
|
|
|
for _, c := range clusterPins {
|
|
|
|
_, ok := mpt.status[c.String()]
|
|
|
|
if !ok {
|
|
|
|
toAdd = append(toAdd, c)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-12-15 18:08:46 +00:00
|
|
|
// Collect items in the tracker not in the State
|
2016-12-09 19:54:46 +00:00
|
|
|
for _, p := range mpt.status {
|
2016-12-19 17:35:24 +00:00
|
|
|
c, _ := cid.Decode(p.CidStr)
|
|
|
|
_, ok := clusterMap[p.CidStr]
|
2016-12-09 19:54:46 +00:00
|
|
|
if !ok {
|
2016-12-19 17:35:24 +00:00
|
|
|
toRemove = append(toRemove, c)
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Update new items and mark them as pinning error
|
|
|
|
for _, c := range toAdd {
|
2016-12-19 17:35:24 +00:00
|
|
|
mpt.unsafeSet(c, PinError)
|
|
|
|
changed = append(changed, c)
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Mark items that need to be removed as unpin error
|
|
|
|
for _, c := range toRemove {
|
2016-12-19 17:35:24 +00:00
|
|
|
mpt.unsafeSet(c, UnpinError)
|
|
|
|
changed = append(changed, c)
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
|
|
|
mpt.mux.Unlock()
|
|
|
|
return changed
|
|
|
|
}
|
|
|
|
|
2016-12-15 18:08:46 +00:00
|
|
|
func (mpt *MapPinTracker) RpcChan() <-chan RPC {
|
2016-12-06 21:29:59 +00:00
|
|
|
return mpt.rpcCh
|
|
|
|
}
|