2016-12-02 18:33:39 +00:00
|
|
|
package ipfscluster
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2017-02-13 15:46:53 +00:00
|
|
|
"errors"
|
2017-01-30 12:12:25 +00:00
|
|
|
"fmt"
|
2016-12-15 13:07:19 +00:00
|
|
|
"sync"
|
2017-01-23 19:29:05 +00:00
|
|
|
"time"
|
2016-12-02 18:33:39 +00:00
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/api"
|
|
|
|
|
2017-01-25 11:14:39 +00:00
|
|
|
rpc "github.com/hsanjuan/go-libp2p-gorpc"
|
2016-12-16 21:00:08 +00:00
|
|
|
cid "github.com/ipfs/go-cid"
|
2016-12-16 11:40:28 +00:00
|
|
|
host "github.com/libp2p/go-libp2p-host"
|
|
|
|
peer "github.com/libp2p/go-libp2p-peer"
|
|
|
|
peerstore "github.com/libp2p/go-libp2p-peerstore"
|
|
|
|
swarm "github.com/libp2p/go-libp2p-swarm"
|
|
|
|
basichost "github.com/libp2p/go-libp2p/p2p/host/basic"
|
2017-01-24 15:19:23 +00:00
|
|
|
ma "github.com/multiformats/go-multiaddr"
|
2016-12-02 18:33:39 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// Cluster is the main IPFS cluster component. It provides
|
2017-01-24 11:39:08 +00:00
|
|
|
// the go-API for it and orchestrates the components that make up the system.
|
2016-12-02 18:33:39 +00:00
|
|
|
type Cluster struct {
|
2017-02-09 15:29:17 +00:00
|
|
|
ctx context.Context
|
|
|
|
cancel func()
|
2016-12-02 18:33:39 +00:00
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
id peer.ID
|
2017-01-30 12:12:25 +00:00
|
|
|
config *Config
|
|
|
|
host host.Host
|
|
|
|
rpcServer *rpc.Server
|
|
|
|
rpcClient *rpc.Client
|
|
|
|
peerManager *peerManager
|
2016-12-02 18:33:39 +00:00
|
|
|
|
2016-12-15 18:08:46 +00:00
|
|
|
consensus *Consensus
|
|
|
|
api API
|
2016-12-02 18:33:39 +00:00
|
|
|
ipfs IPFSConnector
|
2016-12-15 18:08:46 +00:00
|
|
|
state State
|
2016-12-06 21:29:59 +00:00
|
|
|
tracker PinTracker
|
2017-02-13 15:46:53 +00:00
|
|
|
monitor PeerMonitor
|
|
|
|
allocator PinAllocator
|
|
|
|
informer Informer
|
2016-12-15 13:07:19 +00:00
|
|
|
|
|
|
|
shutdownLock sync.Mutex
|
|
|
|
shutdown bool
|
2017-01-30 12:12:25 +00:00
|
|
|
doneCh chan struct{}
|
|
|
|
readyCh chan struct{}
|
2016-12-15 13:07:19 +00:00
|
|
|
wg sync.WaitGroup
|
2017-02-02 22:52:06 +00:00
|
|
|
|
|
|
|
paMux sync.Mutex
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
// NewCluster builds a new IPFS Cluster peer. It initializes a LibP2P host,
|
|
|
|
// creates and RPC Server and client and sets up all components.
|
|
|
|
//
|
|
|
|
// The new cluster peer may still be performing initialization tasks when
|
|
|
|
// this call returns (consensus may still be bootstrapping). Use Cluster.Ready()
|
|
|
|
// if you need to wait until the peer is fully up.
|
2017-02-13 15:46:53 +00:00
|
|
|
func NewCluster(
|
|
|
|
cfg *Config,
|
|
|
|
api API,
|
|
|
|
ipfs IPFSConnector,
|
|
|
|
state State,
|
|
|
|
tracker PinTracker,
|
|
|
|
monitor PeerMonitor,
|
|
|
|
allocator PinAllocator,
|
|
|
|
informer Informer) (*Cluster, error) {
|
|
|
|
|
2017-02-09 15:29:17 +00:00
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
2016-12-02 18:33:39 +00:00
|
|
|
host, err := makeHost(ctx, cfg)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
logger.Infof("IPFS Cluster v%s listening on:", Version)
|
|
|
|
for _, addr := range host.Addrs() {
|
|
|
|
logger.Infof(" %s/ipfs/%s", addr, host.ID().Pretty())
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
c := &Cluster{
|
2017-02-13 15:46:53 +00:00
|
|
|
ctx: ctx,
|
|
|
|
cancel: cancel,
|
|
|
|
id: host.ID(),
|
|
|
|
config: cfg,
|
|
|
|
host: host,
|
|
|
|
api: api,
|
|
|
|
ipfs: ipfs,
|
|
|
|
state: state,
|
|
|
|
tracker: tracker,
|
|
|
|
monitor: monitor,
|
|
|
|
allocator: allocator,
|
|
|
|
informer: informer,
|
|
|
|
doneCh: make(chan struct{}),
|
|
|
|
readyCh: make(chan struct{}),
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
c.setupPeerManager()
|
|
|
|
err = c.setupRPC()
|
2016-12-23 18:35:37 +00:00
|
|
|
if err != nil {
|
2017-02-02 22:52:06 +00:00
|
|
|
c.Shutdown()
|
2016-12-23 18:35:37 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
err = c.setupConsensus()
|
2017-01-30 12:12:25 +00:00
|
|
|
if err != nil {
|
2017-02-02 22:52:06 +00:00
|
|
|
c.Shutdown()
|
2017-01-30 12:12:25 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
2017-02-02 22:52:06 +00:00
|
|
|
c.setupRPCClients()
|
2017-02-13 15:46:53 +00:00
|
|
|
c.bootstrap()
|
|
|
|
ok := c.bootstrap()
|
|
|
|
if !ok {
|
|
|
|
logger.Error("Bootstrap unsuccessful")
|
|
|
|
c.Shutdown()
|
|
|
|
return nil, errors.New("bootstrap unsuccessful")
|
|
|
|
}
|
|
|
|
go func() {
|
|
|
|
c.ready()
|
|
|
|
c.run()
|
|
|
|
}()
|
2017-02-02 22:52:06 +00:00
|
|
|
return c, nil
|
|
|
|
}
|
2017-01-30 12:12:25 +00:00
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
func (c *Cluster) setupPeerManager() {
|
|
|
|
pm := newPeerManager(c)
|
|
|
|
c.peerManager = pm
|
|
|
|
if len(c.config.ClusterPeers) > 0 {
|
|
|
|
c.peerManager.addFromMultiaddrs(c.config.ClusterPeers)
|
|
|
|
} else {
|
|
|
|
c.peerManager.addFromMultiaddrs(c.config.Bootstrap)
|
|
|
|
}
|
2017-01-30 12:12:25 +00:00
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (c *Cluster) setupRPC() error {
|
|
|
|
rpcServer := rpc.NewServer(c.host, RPCProtocol)
|
2017-02-08 17:04:08 +00:00
|
|
|
err := rpcServer.RegisterName("Cluster", &RPCAPI{c})
|
2017-02-02 22:52:06 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
c.rpcServer = rpcServer
|
|
|
|
rpcClient := rpc.NewClientWithServer(c.host, RPCProtocol, rpcServer)
|
|
|
|
c.rpcClient = rpcClient
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *Cluster) setupConsensus() error {
|
|
|
|
var startPeers []peer.ID
|
|
|
|
if len(c.config.ClusterPeers) > 0 {
|
|
|
|
startPeers = peersFromMultiaddrs(c.config.ClusterPeers)
|
|
|
|
} else {
|
|
|
|
startPeers = peersFromMultiaddrs(c.config.Bootstrap)
|
|
|
|
}
|
|
|
|
|
|
|
|
consensus, err := NewConsensus(
|
|
|
|
append(startPeers, c.host.ID()),
|
|
|
|
c.host,
|
|
|
|
c.config.ConsensusDataFolder,
|
|
|
|
c.state)
|
2017-01-30 12:12:25 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Errorf("error creating consensus: %s", err)
|
2017-02-02 22:52:06 +00:00
|
|
|
return err
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
2017-02-02 22:52:06 +00:00
|
|
|
c.consensus = consensus
|
|
|
|
return nil
|
|
|
|
}
|
2017-01-30 12:12:25 +00:00
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
func (c *Cluster) setupRPCClients() {
|
|
|
|
c.tracker.SetClient(c.rpcClient)
|
|
|
|
c.ipfs.SetClient(c.rpcClient)
|
|
|
|
c.api.SetClient(c.rpcClient)
|
|
|
|
c.consensus.SetClient(c.rpcClient)
|
2017-02-13 15:46:53 +00:00
|
|
|
c.monitor.SetClient(c.rpcClient)
|
|
|
|
c.allocator.SetClient(c.rpcClient)
|
|
|
|
c.informer.SetClient(c.rpcClient)
|
2017-02-02 22:52:06 +00:00
|
|
|
}
|
2016-12-09 19:54:46 +00:00
|
|
|
|
2017-02-13 15:46:53 +00:00
|
|
|
// stateSyncWatcher loops and triggers StateSync from time to time
|
2017-02-02 22:52:06 +00:00
|
|
|
func (c *Cluster) stateSyncWatcher() {
|
|
|
|
stateSyncTicker := time.NewTicker(
|
|
|
|
time.Duration(c.config.StateSyncSeconds) * time.Second)
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-stateSyncTicker.C:
|
|
|
|
c.StateSync()
|
|
|
|
case <-c.ctx.Done():
|
|
|
|
stateSyncTicker.Stop()
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-13 15:46:53 +00:00
|
|
|
// push metrics loops and pushes metrics to the leader's monitor
|
|
|
|
func (c *Cluster) pushInformerMetrics() {
|
|
|
|
timer := time.NewTimer(0) // fire immediately first
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-c.ctx.Done():
|
|
|
|
return
|
|
|
|
case <-timer.C:
|
|
|
|
// wait
|
|
|
|
}
|
|
|
|
|
|
|
|
leader, err := c.consensus.Leader()
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
// retry in 1 second
|
|
|
|
timer.Stop()
|
|
|
|
timer.Reset(1 * time.Second)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
metric := c.informer.GetMetric()
|
|
|
|
metric.Peer = c.id
|
|
|
|
|
|
|
|
err = c.rpcClient.Call(
|
|
|
|
leader,
|
|
|
|
"Cluster", "PeerMonitorLogMetric",
|
|
|
|
metric, &struct{}{})
|
|
|
|
if err != nil {
|
|
|
|
logger.Errorf("error pushing metric to %s", leader.Pretty())
|
|
|
|
}
|
|
|
|
|
|
|
|
logger.Debugf("pushed metric %s to %s", metric.Name, metric.Peer.Pretty())
|
|
|
|
|
|
|
|
timer.Stop() // no need to drain C if we are here
|
|
|
|
timer.Reset(metric.GetTTL() / 2)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
// run provides a cancellable context and launches some goroutines
|
|
|
|
// before signaling readyCh
|
|
|
|
func (c *Cluster) run() {
|
2017-02-09 15:29:17 +00:00
|
|
|
go c.stateSyncWatcher()
|
2017-02-13 15:46:53 +00:00
|
|
|
go c.pushInformerMetrics()
|
2017-02-02 22:52:06 +00:00
|
|
|
}
|
|
|
|
|
2017-02-13 15:46:53 +00:00
|
|
|
func (c *Cluster) ready() {
|
2017-02-02 22:52:06 +00:00
|
|
|
// We bootstrapped first because with dirty state consensus
|
|
|
|
// may have a peerset and not find a leader so we cannot wait
|
|
|
|
// for it.
|
|
|
|
timer := time.NewTimer(30 * time.Second)
|
|
|
|
select {
|
|
|
|
case <-timer.C:
|
|
|
|
logger.Error("consensus start timed out")
|
|
|
|
c.Shutdown()
|
|
|
|
return
|
|
|
|
case <-c.consensus.Ready():
|
|
|
|
case <-c.ctx.Done():
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Cluster is ready.
|
|
|
|
logger.Info("Cluster Peers (not including ourselves):")
|
|
|
|
peers := c.peerManager.peersAddrs()
|
|
|
|
if len(peers) == 0 {
|
|
|
|
logger.Info(" - No other peers")
|
|
|
|
}
|
|
|
|
for _, a := range c.peerManager.peersAddrs() {
|
|
|
|
logger.Infof(" - %s", a)
|
|
|
|
}
|
2017-02-13 15:46:53 +00:00
|
|
|
close(c.readyCh)
|
|
|
|
logger.Info("IPFS Cluster is ready")
|
2017-02-02 22:52:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (c *Cluster) bootstrap() bool {
|
|
|
|
// Cases in which we do not bootstrap
|
|
|
|
if len(c.config.Bootstrap) == 0 || len(c.config.ClusterPeers) > 0 {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, b := range c.config.Bootstrap {
|
|
|
|
logger.Infof("Bootstrapping to %s", b)
|
|
|
|
err := c.Join(b)
|
|
|
|
if err == nil {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
logger.Error(err)
|
|
|
|
}
|
|
|
|
return false
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
// Ready returns a channel which signals when this peer is
|
|
|
|
// fully initialized (including consensus).
|
|
|
|
func (c *Cluster) Ready() <-chan struct{} {
|
2017-02-02 22:52:06 +00:00
|
|
|
return c.readyCh
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
|
|
|
|
2016-12-02 18:33:39 +00:00
|
|
|
// Shutdown stops the IPFS cluster components
|
|
|
|
func (c *Cluster) Shutdown() error {
|
2016-12-15 13:07:19 +00:00
|
|
|
c.shutdownLock.Lock()
|
|
|
|
defer c.shutdownLock.Unlock()
|
2017-02-02 22:52:06 +00:00
|
|
|
|
2016-12-15 13:07:19 +00:00
|
|
|
if c.shutdown {
|
|
|
|
logger.Warning("Cluster is already shutdown")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-12-15 13:19:41 +00:00
|
|
|
logger.Info("shutting down IPFS Cluster")
|
2017-02-02 22:52:06 +00:00
|
|
|
|
|
|
|
if c.config.LeaveOnShutdown {
|
|
|
|
// best effort
|
|
|
|
logger.Warning("Attempting to leave Cluster. This may take some seconds")
|
|
|
|
err := c.consensus.LogRmPeer(c.host.ID())
|
|
|
|
if err != nil {
|
|
|
|
logger.Error("leaving cluster: " + err.Error())
|
|
|
|
} else {
|
|
|
|
time.Sleep(2 * time.Second)
|
|
|
|
}
|
|
|
|
c.peerManager.resetPeers()
|
|
|
|
}
|
|
|
|
|
2017-02-09 15:29:17 +00:00
|
|
|
// Cancel contexts
|
|
|
|
c.cancel()
|
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
if con := c.consensus; con != nil {
|
|
|
|
if err := con.Shutdown(); err != nil {
|
|
|
|
logger.Errorf("error stopping consensus: %s", err)
|
|
|
|
return err
|
|
|
|
}
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
2017-02-02 22:52:06 +00:00
|
|
|
|
|
|
|
c.peerManager.savePeers()
|
|
|
|
|
2016-12-02 18:33:39 +00:00
|
|
|
if err := c.api.Shutdown(); err != nil {
|
2016-12-15 13:19:41 +00:00
|
|
|
logger.Errorf("error stopping API: %s", err)
|
2016-12-02 18:33:39 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err := c.ipfs.Shutdown(); err != nil {
|
2016-12-15 13:19:41 +00:00
|
|
|
logger.Errorf("error stopping IPFS Connector: %s", err)
|
2016-12-02 18:33:39 +00:00
|
|
|
return err
|
|
|
|
}
|
2016-12-07 16:21:29 +00:00
|
|
|
|
|
|
|
if err := c.tracker.Shutdown(); err != nil {
|
2016-12-15 13:19:41 +00:00
|
|
|
logger.Errorf("error stopping PinTracker: %s", err)
|
2016-12-07 16:21:29 +00:00
|
|
|
return err
|
|
|
|
}
|
2016-12-15 13:07:19 +00:00
|
|
|
c.wg.Wait()
|
2016-12-23 18:35:37 +00:00
|
|
|
c.host.Close() // Shutdown all network services
|
2017-01-30 12:12:25 +00:00
|
|
|
c.shutdown = true
|
|
|
|
close(c.doneCh)
|
2016-12-09 19:54:46 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
// Done provides a way to learn if the Peer has been shutdown
|
|
|
|
// (for example, because it has been removed from the Cluster)
|
|
|
|
func (c *Cluster) Done() <-chan struct{} {
|
|
|
|
return c.doneCh
|
|
|
|
}
|
|
|
|
|
2017-01-25 17:07:19 +00:00
|
|
|
// ID returns information about the Cluster peer
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) ID() api.ID {
|
2017-01-26 18:59:31 +00:00
|
|
|
// ignore error since it is included in response object
|
|
|
|
ipfsID, _ := c.ipfs.ID()
|
|
|
|
var addrs []ma.Multiaddr
|
|
|
|
for _, addr := range c.host.Addrs() {
|
2017-02-02 22:52:06 +00:00
|
|
|
addrs = append(addrs, multiaddrJoin(addr, c.host.ID()))
|
2017-01-26 18:59:31 +00:00
|
|
|
}
|
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
return api.ID{
|
|
|
|
ID: c.host.ID(),
|
|
|
|
//PublicKey: c.host.Peerstore().PubKey(c.host.ID()),
|
2017-01-26 18:59:31 +00:00
|
|
|
Addresses: addrs,
|
2017-02-02 22:52:06 +00:00
|
|
|
ClusterPeers: c.peerManager.peersAddrs(),
|
2017-01-24 15:19:23 +00:00
|
|
|
Version: Version,
|
2017-01-24 15:55:37 +00:00
|
|
|
Commit: Commit,
|
2017-01-24 15:19:23 +00:00
|
|
|
RPCProtocolVersion: RPCProtocol,
|
2017-01-26 18:59:31 +00:00
|
|
|
IPFS: ipfsID,
|
2017-01-24 15:19:23 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
// PeerAdd adds a new peer to this Cluster.
|
|
|
|
//
|
2017-02-02 22:52:06 +00:00
|
|
|
// The new peer must be reachable. It will be added to the
|
|
|
|
// consensus and will receive the shared state (including the
|
|
|
|
// list of peers). The new peer should be a single-peer cluster,
|
|
|
|
// preferable without any relevant state.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) PeerAdd(addr ma.Multiaddr) (api.ID, error) {
|
2017-02-02 22:52:06 +00:00
|
|
|
// starting 10 nodes on the same box for testing
|
|
|
|
// causes deadlock and a global lock here
|
|
|
|
// seems to help.
|
|
|
|
c.paMux.Lock()
|
|
|
|
defer c.paMux.Unlock()
|
|
|
|
logger.Debugf("peerAdd called with %s", addr)
|
|
|
|
pid, decapAddr, err := multiaddrSplit(addr)
|
2017-01-30 12:12:25 +00:00
|
|
|
if err != nil {
|
2017-02-08 17:04:08 +00:00
|
|
|
id := api.ID{
|
2017-01-30 12:12:25 +00:00
|
|
|
Error: err.Error(),
|
|
|
|
}
|
|
|
|
return id, err
|
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
// Figure out its real address if we have one
|
|
|
|
remoteAddr := getRemoteMultiaddr(c.host, pid, decapAddr)
|
|
|
|
|
|
|
|
err = c.peerManager.addPeer(remoteAddr)
|
2017-01-30 12:12:25 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
2017-02-08 17:04:08 +00:00
|
|
|
id := api.ID{ID: pid, Error: err.Error()}
|
2017-01-30 12:12:25 +00:00
|
|
|
return id, err
|
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
// Figure out our address to that peer. This also
|
|
|
|
// ensures that it is reachable
|
2017-02-08 17:04:08 +00:00
|
|
|
var addrSerial api.MultiaddrSerial
|
2017-02-02 22:52:06 +00:00
|
|
|
err = c.rpcClient.Call(pid, "Cluster",
|
|
|
|
"RemoteMultiaddrForPeer", c.host.ID(), &addrSerial)
|
|
|
|
if err != nil {
|
2017-01-30 12:12:25 +00:00
|
|
|
logger.Error(err)
|
2017-02-08 17:04:08 +00:00
|
|
|
id := api.ID{ID: pid, Error: err.Error()}
|
2017-02-02 22:52:06 +00:00
|
|
|
c.peerManager.rmPeer(pid, false)
|
2017-01-30 12:12:25 +00:00
|
|
|
return id, err
|
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
// Log the new peer in the log so everyone gets it.
|
|
|
|
err = c.consensus.LogAddPeer(remoteAddr)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
2017-02-08 17:04:08 +00:00
|
|
|
id := api.ID{ID: pid, Error: err.Error()}
|
2017-02-02 22:52:06 +00:00
|
|
|
c.peerManager.rmPeer(pid, false)
|
|
|
|
return id, err
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
// Send cluster peers to the new peer.
|
|
|
|
clusterPeers := append(c.peerManager.peersAddrs(),
|
|
|
|
addrSerial.ToMultiaddr())
|
|
|
|
err = c.rpcClient.Call(pid,
|
|
|
|
"Cluster",
|
|
|
|
"PeerManagerAddFromMultiaddrs",
|
2017-02-08 17:04:08 +00:00
|
|
|
api.MultiaddrsToSerial(clusterPeers),
|
2017-02-02 22:52:06 +00:00
|
|
|
&struct{}{})
|
2017-01-30 12:12:25 +00:00
|
|
|
if err != nil {
|
2017-02-02 22:52:06 +00:00
|
|
|
logger.Error(err)
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
id, err := c.getIDForPeer(pid)
|
|
|
|
return id, nil
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// PeerRemove removes a peer from this Cluster.
|
|
|
|
//
|
|
|
|
// The peer will be removed from the consensus peer set,
|
2017-02-02 22:52:06 +00:00
|
|
|
// it will be shut down after this happens.
|
|
|
|
func (c *Cluster) PeerRemove(pid peer.ID) error {
|
|
|
|
if !c.peerManager.isPeer(pid) {
|
|
|
|
return fmt.Errorf("%s is not a peer", pid.Pretty())
|
|
|
|
}
|
|
|
|
|
|
|
|
err := c.consensus.LogRmPeer(pid)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// This is a best effort. It may fail
|
|
|
|
// if that peer is down
|
|
|
|
err = c.rpcClient.Call(pid,
|
|
|
|
"Cluster",
|
|
|
|
"PeerManagerRmPeerShutdown",
|
|
|
|
pid,
|
|
|
|
&struct{}{})
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Join adds this peer to an existing cluster. The calling peer should
|
|
|
|
// be a single-peer cluster node. This is almost equivalent to calling
|
|
|
|
// PeerAdd on the destination cluster.
|
|
|
|
func (c *Cluster) Join(addr ma.Multiaddr) error {
|
|
|
|
logger.Debugf("Join(%s)", addr)
|
|
|
|
|
|
|
|
//if len(c.peerManager.peers()) > 1 {
|
|
|
|
// logger.Error(c.peerManager.peers())
|
|
|
|
// return errors.New("only single-node clusters can be joined")
|
|
|
|
//}
|
|
|
|
|
|
|
|
pid, _, err := multiaddrSplit(addr)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Bootstrap to myself
|
|
|
|
if pid == c.host.ID() {
|
|
|
|
return nil
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
2017-02-02 22:52:06 +00:00
|
|
|
|
|
|
|
// Add peer to peerstore so we can talk to it
|
|
|
|
c.peerManager.addPeer(addr)
|
|
|
|
|
|
|
|
// Note that PeerAdd() on the remote peer will
|
|
|
|
// figure out what our real address is (obviously not
|
|
|
|
// ClusterAddr).
|
2017-02-08 17:04:08 +00:00
|
|
|
var myID api.IDSerial
|
2017-02-02 22:52:06 +00:00
|
|
|
err = c.rpcClient.Call(pid,
|
|
|
|
"Cluster",
|
|
|
|
"PeerAdd",
|
2017-02-08 17:04:08 +00:00
|
|
|
api.MultiaddrToSerial(multiaddrJoin(c.config.ClusterAddr, c.host.ID())),
|
2017-02-02 22:52:06 +00:00
|
|
|
&myID)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return err
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
2017-02-02 22:52:06 +00:00
|
|
|
|
|
|
|
// wait for leader and for state to catch up
|
|
|
|
// then sync
|
|
|
|
err = c.consensus.WaitForSync()
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
c.StateSync()
|
|
|
|
|
|
|
|
logger.Infof("joined %s's cluster", addr)
|
2017-01-30 12:12:25 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-12-20 18:51:13 +00:00
|
|
|
// StateSync syncs the consensus state to the Pin Tracker, ensuring
|
|
|
|
// that every Cid that should be tracked is tracked. It returns
|
|
|
|
// PinInfo for Cids which were added or deleted.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) StateSync() ([]api.PinInfo, error) {
|
2016-12-09 19:54:46 +00:00
|
|
|
cState, err := c.consensus.State()
|
|
|
|
if err != nil {
|
2016-12-15 18:08:46 +00:00
|
|
|
return nil, err
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
2016-12-20 18:51:13 +00:00
|
|
|
|
2017-01-23 22:58:04 +00:00
|
|
|
logger.Debug("syncing state to tracker")
|
2017-02-13 15:46:53 +00:00
|
|
|
clusterPins := cState.List()
|
2016-12-20 18:51:13 +00:00
|
|
|
var changed []*cid.Cid
|
|
|
|
|
|
|
|
// Track items which are not tracked
|
2017-02-13 15:46:53 +00:00
|
|
|
for _, carg := range clusterPins {
|
|
|
|
if c.tracker.Status(carg.Cid).Status == api.TrackerStatusUnpinned {
|
|
|
|
changed = append(changed, carg.Cid)
|
|
|
|
go c.tracker.Track(carg)
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Untrack items which should not be tracked
|
2017-01-25 18:38:23 +00:00
|
|
|
for _, p := range c.tracker.StatusAll() {
|
2017-02-13 15:46:53 +00:00
|
|
|
if !cState.Has(p.Cid) {
|
2017-02-08 17:04:08 +00:00
|
|
|
changed = append(changed, p.Cid)
|
|
|
|
go c.tracker.Untrack(p.Cid)
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
var infos []api.PinInfo
|
2016-12-19 17:35:24 +00:00
|
|
|
for _, h := range changed {
|
2017-01-25 18:38:23 +00:00
|
|
|
infos = append(infos, c.tracker.Status(h))
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
return infos, nil
|
|
|
|
}
|
|
|
|
|
2017-01-25 18:38:23 +00:00
|
|
|
// StatusAll returns the GlobalPinInfo for all tracked Cids. If an error
|
|
|
|
// happens, the slice will contain as much information as could be fetched.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) StatusAll() ([]api.GlobalPinInfo, error) {
|
2017-01-25 18:38:23 +00:00
|
|
|
return c.globalPinInfoSlice("TrackerStatusAll")
|
2016-12-23 18:35:37 +00:00
|
|
|
}
|
|
|
|
|
2017-01-25 18:38:23 +00:00
|
|
|
// Status returns the GlobalPinInfo for a given Cid. If an error happens,
|
2016-12-23 18:35:37 +00:00
|
|
|
// the GlobalPinInfo should contain as much information as could be fetched.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) Status(h *cid.Cid) (api.GlobalPinInfo, error) {
|
2017-01-25 18:38:23 +00:00
|
|
|
return c.globalPinInfoCid("TrackerStatus", h)
|
2016-12-23 18:35:37 +00:00
|
|
|
}
|
|
|
|
|
2017-01-25 18:38:23 +00:00
|
|
|
// SyncAllLocal makes sure that the current state for all tracked items
|
|
|
|
// matches the state reported by the IPFS daemon.
|
2016-12-20 18:51:13 +00:00
|
|
|
//
|
2017-01-25 18:38:23 +00:00
|
|
|
// SyncAllLocal returns the list of PinInfo that where updated because of
|
|
|
|
// the operation, along with those in error states.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) SyncAllLocal() ([]api.PinInfo, error) {
|
2017-01-25 18:38:23 +00:00
|
|
|
syncedItems, err := c.tracker.SyncAll()
|
|
|
|
// Despite errors, tracker provides synced items that we can provide.
|
|
|
|
// They encapsulate the error.
|
2017-01-25 17:07:19 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error("tracker.Sync() returned with error: ", err)
|
|
|
|
logger.Error("Is the ipfs daemon running?")
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
2017-01-25 18:38:23 +00:00
|
|
|
return syncedItems, err
|
2016-12-15 18:08:46 +00:00
|
|
|
}
|
|
|
|
|
2017-01-25 18:38:23 +00:00
|
|
|
// SyncLocal performs a local sync operation for the given Cid. This will
|
|
|
|
// tell the tracker to verify the status of the Cid against the IPFS daemon.
|
|
|
|
// It returns the updated PinInfo for the Cid.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) SyncLocal(h *cid.Cid) (api.PinInfo, error) {
|
2016-12-20 18:51:13 +00:00
|
|
|
var err error
|
2017-01-25 18:38:23 +00:00
|
|
|
pInfo, err := c.tracker.Sync(h)
|
2017-01-25 17:07:19 +00:00
|
|
|
// Despite errors, trackers provides an updated PinInfo so
|
|
|
|
// we just log it.
|
|
|
|
if err != nil {
|
|
|
|
logger.Error("tracker.SyncCid() returned with error: ", err)
|
|
|
|
logger.Error("Is the ipfs daemon running?")
|
2016-12-15 18:08:46 +00:00
|
|
|
}
|
2017-01-25 18:38:23 +00:00
|
|
|
return pInfo, err
|
2016-12-15 18:08:46 +00:00
|
|
|
}
|
|
|
|
|
2017-01-26 18:59:31 +00:00
|
|
|
// SyncAll triggers LocalSync() operations in all cluster peers.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) SyncAll() ([]api.GlobalPinInfo, error) {
|
2017-01-25 18:38:23 +00:00
|
|
|
return c.globalPinInfoSlice("SyncAllLocal")
|
2016-12-15 18:08:46 +00:00
|
|
|
}
|
|
|
|
|
2017-01-25 18:38:23 +00:00
|
|
|
// Sync triggers a LocalSyncCid() operation for a given Cid
|
2017-01-26 18:59:31 +00:00
|
|
|
// in all cluster peers.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) Sync(h *cid.Cid) (api.GlobalPinInfo, error) {
|
2017-01-25 18:38:23 +00:00
|
|
|
return c.globalPinInfoCid("SyncLocal", h)
|
|
|
|
}
|
|
|
|
|
|
|
|
// RecoverLocal triggers a recover operation for a given Cid
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) RecoverLocal(h *cid.Cid) (api.PinInfo, error) {
|
2017-01-25 18:38:23 +00:00
|
|
|
return c.tracker.Recover(h)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Recover triggers a recover operation for a given Cid in all
|
2017-01-26 18:59:31 +00:00
|
|
|
// cluster peers.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) Recover(h *cid.Cid) (api.GlobalPinInfo, error) {
|
2017-01-25 18:38:23 +00:00
|
|
|
return c.globalPinInfoCid("TrackerRecover", h)
|
2016-12-15 18:08:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Pins returns the list of Cids managed by Cluster and which are part
|
|
|
|
// of the current global state. This is the source of truth as to which
|
|
|
|
// pins are managed, but does not indicate if the item is successfully pinned.
|
2017-02-13 15:46:53 +00:00
|
|
|
func (c *Cluster) Pins() []api.CidArg {
|
2016-12-15 18:08:46 +00:00
|
|
|
cState, err := c.consensus.State()
|
|
|
|
if err != nil {
|
2017-02-01 17:16:09 +00:00
|
|
|
logger.Error(err)
|
2017-02-13 15:46:53 +00:00
|
|
|
return []api.CidArg{}
|
2016-12-15 18:08:46 +00:00
|
|
|
}
|
2017-02-13 15:46:53 +00:00
|
|
|
|
|
|
|
return cState.List()
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Pin makes the cluster Pin a Cid. This implies adding the Cid
|
|
|
|
// to the IPFS Cluster peers shared-state. Depending on the cluster
|
2016-12-16 11:40:28 +00:00
|
|
|
// pinning strategy, the PinTracker may then request the IPFS daemon
|
2017-01-23 13:01:49 +00:00
|
|
|
// to pin the Cid.
|
2016-12-02 18:33:39 +00:00
|
|
|
//
|
|
|
|
// Pin returns an error if the operation could not be persisted
|
|
|
|
// to the global state. Pin does not reflect the success or failure
|
|
|
|
// of underlying IPFS daemon pinning operations.
|
|
|
|
func (c *Cluster) Pin(h *cid.Cid) error {
|
2016-12-15 13:19:41 +00:00
|
|
|
logger.Info("pinning:", h)
|
2017-02-13 15:46:53 +00:00
|
|
|
|
|
|
|
cidArg := api.CidArg{
|
|
|
|
Cid: h,
|
|
|
|
}
|
|
|
|
|
|
|
|
rpl := c.config.ReplicationFactor
|
|
|
|
switch {
|
|
|
|
case rpl == 0:
|
|
|
|
return errors.New("replication factor is 0")
|
|
|
|
case rpl < 0:
|
|
|
|
cidArg.Everywhere = true
|
|
|
|
case rpl > 0:
|
|
|
|
allocs, err := c.allocate(h)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
cidArg.Allocations = allocs
|
|
|
|
}
|
|
|
|
|
|
|
|
err := c.consensus.LogPin(cidArg)
|
2016-12-23 18:35:37 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Unpin makes the cluster Unpin a Cid. This implies adding the Cid
|
2017-01-23 13:01:49 +00:00
|
|
|
// to the IPFS Cluster peers shared-state.
|
2016-12-02 18:33:39 +00:00
|
|
|
//
|
|
|
|
// Unpin returns an error if the operation could not be persisted
|
|
|
|
// to the global state. Unpin does not reflect the success or failure
|
|
|
|
// of underlying IPFS daemon unpinning operations.
|
|
|
|
func (c *Cluster) Unpin(h *cid.Cid) error {
|
2017-01-25 18:38:23 +00:00
|
|
|
logger.Info("unpinning:", h)
|
2017-02-13 15:46:53 +00:00
|
|
|
|
|
|
|
carg := api.CidArg{
|
|
|
|
Cid: h,
|
|
|
|
}
|
|
|
|
|
|
|
|
err := c.consensus.LogUnpin(carg)
|
2016-12-23 18:35:37 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Version returns the current IPFS Cluster version
|
|
|
|
func (c *Cluster) Version() string {
|
|
|
|
return Version
|
|
|
|
}
|
|
|
|
|
2017-01-26 18:59:31 +00:00
|
|
|
// Peers returns the IDs of the members of this Cluster
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) Peers() []api.ID {
|
2017-01-30 12:12:25 +00:00
|
|
|
members := c.peerManager.peers()
|
2017-02-08 17:04:08 +00:00
|
|
|
peersSerial := make([]api.IDSerial, len(members), len(members))
|
|
|
|
peers := make([]api.ID, len(members), len(members))
|
2017-01-26 18:59:31 +00:00
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
errs := c.multiRPC(members, "Cluster", "ID", struct{}{},
|
|
|
|
copyIDSerialsToIfaces(peersSerial))
|
2017-01-26 18:59:31 +00:00
|
|
|
|
|
|
|
for i, err := range errs {
|
|
|
|
if err != nil {
|
|
|
|
peersSerial[i].ID = peer.IDB58Encode(members[i])
|
|
|
|
peersSerial[i].Error = err.Error()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, ps := range peersSerial {
|
|
|
|
peers[i] = ps.ToID()
|
|
|
|
}
|
|
|
|
return peers
|
|
|
|
}
|
|
|
|
|
2016-12-02 18:33:39 +00:00
|
|
|
// makeHost makes a libp2p-host
|
2016-12-15 18:08:46 +00:00
|
|
|
func makeHost(ctx context.Context, cfg *Config) (host.Host, error) {
|
2016-12-02 18:33:39 +00:00
|
|
|
ps := peerstore.NewPeerstore()
|
2017-01-23 17:38:59 +00:00
|
|
|
privateKey := cfg.PrivateKey
|
2016-12-02 18:33:39 +00:00
|
|
|
publicKey := privateKey.GetPublic()
|
|
|
|
|
2017-01-23 17:38:59 +00:00
|
|
|
if err := ps.AddPubKey(cfg.ID, publicKey); err != nil {
|
2016-12-02 18:33:39 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-01-23 17:38:59 +00:00
|
|
|
if err := ps.AddPrivKey(cfg.ID, privateKey); err != nil {
|
2016-12-02 18:33:39 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
network, err := swarm.NewNetwork(
|
|
|
|
ctx,
|
2017-01-24 15:19:23 +00:00
|
|
|
[]ma.Multiaddr{cfg.ClusterAddr},
|
2017-01-23 17:38:59 +00:00
|
|
|
cfg.ID,
|
2016-12-02 18:33:39 +00:00
|
|
|
ps,
|
|
|
|
nil,
|
|
|
|
)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
bhost := basichost.New(network)
|
|
|
|
return bhost, nil
|
|
|
|
}
|
2016-12-20 18:51:13 +00:00
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
// Perform an RPC request to multiple destinations
|
2016-12-23 18:35:37 +00:00
|
|
|
func (c *Cluster) multiRPC(dests []peer.ID, svcName, svcMethod string, args interface{}, reply []interface{}) []error {
|
|
|
|
if len(dests) != len(reply) {
|
2017-01-23 13:21:26 +00:00
|
|
|
panic("must have matching dests and replies")
|
2016-12-23 18:35:37 +00:00
|
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
errs := make([]error, len(dests), len(dests))
|
|
|
|
|
2016-12-28 15:25:24 +00:00
|
|
|
for i := range dests {
|
2016-12-23 18:35:37 +00:00
|
|
|
wg.Add(1)
|
|
|
|
go func(i int) {
|
|
|
|
defer wg.Done()
|
|
|
|
err := c.rpcClient.Call(
|
|
|
|
dests[i],
|
|
|
|
svcName,
|
|
|
|
svcMethod,
|
|
|
|
args,
|
|
|
|
reply[i])
|
|
|
|
errs[i] = err
|
|
|
|
}(i)
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
return errs
|
2016-12-20 18:51:13 +00:00
|
|
|
|
2016-12-23 18:35:37 +00:00
|
|
|
}
|
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) globalPinInfoCid(method string, h *cid.Cid) (api.GlobalPinInfo, error) {
|
|
|
|
pin := api.GlobalPinInfo{
|
2017-01-25 17:07:19 +00:00
|
|
|
Cid: h,
|
2017-02-08 17:04:08 +00:00
|
|
|
PeerMap: make(map[peer.ID]api.PinInfo),
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
members := c.peerManager.peers()
|
2017-02-08 17:04:08 +00:00
|
|
|
replies := make([]api.PinInfoSerial, len(members), len(members))
|
|
|
|
arg := api.CidArg{
|
|
|
|
Cid: h,
|
|
|
|
}
|
|
|
|
errs := c.multiRPC(members,
|
|
|
|
"Cluster",
|
|
|
|
method, arg.ToSerial(),
|
|
|
|
copyPinInfoSerialToIfaces(replies))
|
|
|
|
|
|
|
|
for i, rserial := range replies {
|
|
|
|
r := rserial.ToPinInfo()
|
|
|
|
if e := errs[i]; e != nil {
|
|
|
|
if r.Status == api.TrackerStatusBug {
|
|
|
|
// This error must come from not being able to contact that cluster member
|
|
|
|
logger.Errorf("%s: error in broadcast response from %s: %s ", c.host.ID(), members[i], e)
|
|
|
|
r = api.PinInfo{
|
|
|
|
Cid: r.Cid,
|
2017-01-23 23:52:42 +00:00
|
|
|
Peer: members[i],
|
2017-02-08 17:04:08 +00:00
|
|
|
Status: api.TrackerStatusClusterError,
|
2017-01-23 23:52:42 +00:00
|
|
|
TS: time.Now(),
|
|
|
|
Error: e.Error(),
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
r.Error = e.Error()
|
|
|
|
}
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
2017-01-25 17:07:19 +00:00
|
|
|
pin.PeerMap[members[i]] = r
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
2016-12-28 15:25:24 +00:00
|
|
|
|
2017-01-23 23:52:42 +00:00
|
|
|
return pin, nil
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) globalPinInfoSlice(method string) ([]api.GlobalPinInfo, error) {
|
|
|
|
var infos []api.GlobalPinInfo
|
|
|
|
fullMap := make(map[string]api.GlobalPinInfo)
|
2016-12-20 18:51:13 +00:00
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
members := c.peerManager.peers()
|
2017-02-08 17:04:08 +00:00
|
|
|
replies := make([][]api.PinInfoSerial, len(members), len(members))
|
|
|
|
errs := c.multiRPC(members,
|
|
|
|
"Cluster",
|
|
|
|
method, struct{}{},
|
|
|
|
copyPinInfoSerialSliceToIfaces(replies))
|
2016-12-20 18:51:13 +00:00
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
mergePins := func(pins []api.PinInfoSerial) {
|
|
|
|
for _, pserial := range pins {
|
|
|
|
p := pserial.ToPinInfo()
|
|
|
|
item, ok := fullMap[pserial.Cid]
|
2016-12-20 18:51:13 +00:00
|
|
|
if !ok {
|
2017-02-08 17:04:08 +00:00
|
|
|
fullMap[pserial.Cid] = api.GlobalPinInfo{
|
|
|
|
Cid: p.Cid,
|
|
|
|
PeerMap: map[peer.ID]api.PinInfo{
|
2016-12-20 18:51:13 +00:00
|
|
|
p.Peer: p,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
} else {
|
2017-01-25 17:07:19 +00:00
|
|
|
item.PeerMap[p.Peer] = p
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-01-24 00:09:27 +00:00
|
|
|
erroredPeers := make(map[peer.ID]string)
|
2016-12-23 18:35:37 +00:00
|
|
|
for i, r := range replies {
|
2017-01-25 17:07:19 +00:00
|
|
|
if e := errs[i]; e != nil { // This error must come from not being able to contact that cluster member
|
2017-01-23 19:29:05 +00:00
|
|
|
logger.Errorf("%s: error in broadcast response from %s: %s ", c.host.ID(), members[i], e)
|
2017-01-24 00:09:27 +00:00
|
|
|
erroredPeers[members[i]] = e.Error()
|
2017-01-23 23:52:42 +00:00
|
|
|
} else {
|
|
|
|
mergePins(r)
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-01-24 00:09:27 +00:00
|
|
|
// Merge any errors
|
|
|
|
for p, msg := range erroredPeers {
|
2017-02-08 17:04:08 +00:00
|
|
|
for cidStr := range fullMap {
|
|
|
|
c, _ := cid.Decode(cidStr)
|
|
|
|
fullMap[cidStr].PeerMap[p] = api.PinInfo{
|
|
|
|
Cid: c,
|
2017-01-24 00:09:27 +00:00
|
|
|
Peer: p,
|
2017-02-08 17:04:08 +00:00
|
|
|
Status: api.TrackerStatusClusterError,
|
2017-01-24 00:09:27 +00:00
|
|
|
TS: time.Now(),
|
|
|
|
Error: msg,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-12-20 18:51:13 +00:00
|
|
|
for _, v := range fullMap {
|
|
|
|
infos = append(infos, v)
|
|
|
|
}
|
|
|
|
|
2017-01-23 23:52:42 +00:00
|
|
|
return infos, nil
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
2017-01-23 19:29:05 +00:00
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) getIDForPeer(pid peer.ID) (api.ID, error) {
|
|
|
|
idSerial := api.ID{ID: pid}.ToSerial()
|
2017-02-02 22:52:06 +00:00
|
|
|
err := c.rpcClient.Call(
|
|
|
|
pid, "Cluster", "ID", struct{}{}, &idSerial)
|
|
|
|
id := idSerial.ToID()
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
id.Error = err.Error()
|
2017-01-23 19:29:05 +00:00
|
|
|
}
|
2017-02-02 22:52:06 +00:00
|
|
|
return id, err
|
2017-01-23 19:29:05 +00:00
|
|
|
}
|
2017-02-13 15:46:53 +00:00
|
|
|
|
|
|
|
// allocate finds peers to allocate a hash using the informer and the monitor
|
|
|
|
// it should only be used with a positive replication factor
|
|
|
|
func (c *Cluster) allocate(hash *cid.Cid) ([]peer.ID, error) {
|
|
|
|
if c.config.ReplicationFactor <= 0 {
|
|
|
|
return nil, errors.New("cannot decide allocation for replication factor <= 0")
|
|
|
|
}
|
|
|
|
|
|
|
|
// Figure out who is currently holding this
|
|
|
|
var currentlyAllocatedPeers []peer.ID
|
|
|
|
st, err := c.consensus.State()
|
|
|
|
if err != nil {
|
|
|
|
// no state we assume it is empty. If there was other
|
|
|
|
// problem, we would fail to commit anyway.
|
|
|
|
currentlyAllocatedPeers = []peer.ID{}
|
|
|
|
} else {
|
|
|
|
carg := st.Get(hash)
|
|
|
|
currentlyAllocatedPeers = carg.Allocations
|
|
|
|
}
|
|
|
|
|
|
|
|
// initialize a candidate metrics map with all current clusterPeers
|
|
|
|
// (albeit with invalid metrics)
|
|
|
|
clusterPeers := c.peerManager.peers()
|
|
|
|
metricsMap := make(map[peer.ID]api.Metric)
|
|
|
|
for _, cp := range clusterPeers {
|
|
|
|
metricsMap[cp] = api.Metric{Valid: false}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Request latest metrics logged by informers from the leader
|
|
|
|
metricName := c.informer.Name()
|
|
|
|
l, err := c.consensus.Leader()
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.New("cannot determine leading Monitor")
|
|
|
|
}
|
|
|
|
var metrics []api.Metric
|
|
|
|
err = c.rpcClient.Call(l,
|
|
|
|
"Cluster", "PeerMonitorLastMetrics",
|
|
|
|
metricName,
|
|
|
|
&metrics)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// put metrics in the metricsMap if they belong to a current clusterPeer
|
|
|
|
for _, m := range metrics {
|
|
|
|
_, ok := metricsMap[m.Peer]
|
|
|
|
if !ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
metricsMap[m.Peer] = m
|
|
|
|
}
|
|
|
|
|
|
|
|
// Remove any invalid metric. This will clear any cluster peers
|
|
|
|
// for which we did not receive metrics.
|
|
|
|
for p, m := range metricsMap {
|
|
|
|
if m.Discard() {
|
|
|
|
delete(metricsMap, p)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Move metrics from currentlyAllocatedPeers to a new map
|
|
|
|
currentlyAllocatedPeersMetrics := make(map[peer.ID]api.Metric)
|
|
|
|
for _, p := range currentlyAllocatedPeers {
|
|
|
|
m, ok := metricsMap[p]
|
|
|
|
if !ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
currentlyAllocatedPeersMetrics[p] = m
|
|
|
|
delete(metricsMap, p)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
// how many allocations do we need (note we will re-allocate if we did
|
|
|
|
// not receive good metrics for currently allocated peeers)
|
|
|
|
needed := c.config.ReplicationFactor - len(currentlyAllocatedPeersMetrics)
|
|
|
|
|
|
|
|
// if we are already good (note invalid metrics would trigger
|
|
|
|
// re-allocations as they are not included in currentAllocMetrics)
|
|
|
|
if needed <= 0 {
|
|
|
|
return nil, fmt.Errorf("CID is already correctly allocated to %s", currentlyAllocatedPeers)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Allocate is called with currentAllocMetrics which contains
|
|
|
|
// only currentlyAllocatedPeers when they have provided valid metrics.
|
|
|
|
candidateAllocs, err := c.allocator.Allocate(hash, currentlyAllocatedPeersMetrics, metricsMap)
|
|
|
|
if err != nil {
|
|
|
|
return nil, logError(err.Error())
|
|
|
|
}
|
|
|
|
|
|
|
|
// we don't have enough peers to pin
|
|
|
|
if len(candidateAllocs) < needed {
|
|
|
|
err = logError("cannot find enough allocations for this CID: needed: %d. Got: %s",
|
|
|
|
needed, candidateAllocs)
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// return as many as needed
|
|
|
|
return candidateAllocs[0:needed], nil
|
|
|
|
}
|