2016-12-28 15:25:24 +00:00
|
|
|
// Package ipfscluster implements a wrapper for the IPFS deamon which
|
|
|
|
// allows to orchestrate pinning operations among several IPFS nodes.
|
2016-12-02 18:33:39 +00:00
|
|
|
//
|
2016-12-28 15:25:24 +00:00
|
|
|
// IPFS Cluster uses a go-libp2p-raft to keep a shared state between
|
2017-01-26 18:59:31 +00:00
|
|
|
// the different cluster peers. It also uses LibP2P to enable
|
2016-12-28 15:25:24 +00:00
|
|
|
// communication between its different components, which perform different
|
|
|
|
// tasks like managing the underlying IPFS daemons, or providing APIs for
|
|
|
|
// external control.
|
2016-12-02 18:33:39 +00:00
|
|
|
package ipfscluster
|
|
|
|
|
|
|
|
import (
|
2017-03-14 15:37:29 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/api"
|
|
|
|
"github.com/ipfs/ipfs-cluster/state"
|
|
|
|
|
2017-01-25 11:14:39 +00:00
|
|
|
rpc "github.com/hsanjuan/go-libp2p-gorpc"
|
2016-12-16 11:40:28 +00:00
|
|
|
cid "github.com/ipfs/go-cid"
|
|
|
|
peer "github.com/libp2p/go-libp2p-peer"
|
2016-12-23 18:35:37 +00:00
|
|
|
protocol "github.com/libp2p/go-libp2p-protocol"
|
2017-03-14 15:37:29 +00:00
|
|
|
ma "github.com/multiformats/go-multiaddr"
|
2016-12-02 18:33:39 +00:00
|
|
|
)
|
|
|
|
|
2017-01-26 18:59:31 +00:00
|
|
|
// RPCProtocol is used to send libp2p messages between cluster peers
|
2017-01-24 15:19:23 +00:00
|
|
|
var RPCProtocol = protocol.ID("/ipfscluster/" + Version + "/rpc")
|
2016-12-23 18:35:37 +00:00
|
|
|
|
2016-12-29 17:38:09 +00:00
|
|
|
// Component represents a piece of ipfscluster. Cluster components
|
2016-12-08 16:24:38 +00:00
|
|
|
// usually run their own goroutines (a http server for example). They
|
2016-12-23 18:35:37 +00:00
|
|
|
// communicate with the main Cluster component and other components
|
|
|
|
// (both local and remote), using an instance of rpc.Client.
|
2016-12-29 17:38:09 +00:00
|
|
|
type Component interface {
|
2016-12-23 18:35:37 +00:00
|
|
|
SetClient(*rpc.Client)
|
2016-12-02 18:33:39 +00:00
|
|
|
Shutdown() error
|
|
|
|
}
|
|
|
|
|
2017-03-14 15:37:29 +00:00
|
|
|
// Consensus is a component which keeps a shared state in
|
|
|
|
// IPFS Cluster and triggers actions on updates to that state.
|
|
|
|
// Currently, Consensus needs to be able to elect/provide a
|
|
|
|
// Cluster Leader and the implementation is very tight to
|
|
|
|
// the Cluster main component.
|
|
|
|
type Consensus interface {
|
|
|
|
Component
|
|
|
|
// Returns a channel to signal that the consensus
|
|
|
|
// algoritm is ready
|
|
|
|
Ready() <-chan struct{}
|
|
|
|
// Logs a pin operation
|
|
|
|
LogPin(c api.Pin) error
|
|
|
|
// Logs an unpin operation
|
|
|
|
LogUnpin(c api.Pin) error
|
|
|
|
LogAddPeer(addr ma.Multiaddr) error
|
|
|
|
LogRmPeer(p peer.ID) error
|
|
|
|
State() (state.State, error)
|
|
|
|
// Provide a node which is responsible to perform
|
|
|
|
// specific tasks which must only run in 1 cluster peer
|
|
|
|
Leader() (peer.ID, error)
|
|
|
|
// Only returns when the consensus state has all log
|
|
|
|
// updates applied to it
|
|
|
|
WaitForSync() error
|
2017-11-01 12:25:28 +00:00
|
|
|
// Clean removes all consensus data
|
|
|
|
Clean() error
|
2017-03-14 15:37:29 +00:00
|
|
|
}
|
|
|
|
|
2016-12-15 18:08:46 +00:00
|
|
|
// API is a component which offers an API for Cluster. This is
|
2016-12-02 18:33:39 +00:00
|
|
|
// a base component.
|
2016-12-15 18:08:46 +00:00
|
|
|
type API interface {
|
2016-12-29 17:38:09 +00:00
|
|
|
Component
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// IPFSConnector is a component which allows cluster to interact with
|
2016-12-08 16:24:38 +00:00
|
|
|
// an IPFS daemon. This is a base component.
|
2016-12-02 18:33:39 +00:00
|
|
|
type IPFSConnector interface {
|
2016-12-29 17:38:09 +00:00
|
|
|
Component
|
2017-02-08 17:04:08 +00:00
|
|
|
ID() (api.IPFSID, error)
|
2016-12-02 18:33:39 +00:00
|
|
|
Pin(*cid.Cid) error
|
|
|
|
Unpin(*cid.Cid) error
|
2017-02-08 17:04:08 +00:00
|
|
|
PinLsCid(*cid.Cid) (api.IPFSPinStatus, error)
|
2017-02-09 15:29:17 +00:00
|
|
|
PinLs(typeFilter string) (map[string]api.IPFSPinStatus, error)
|
2017-03-23 18:34:33 +00:00
|
|
|
// ConnectSwarms make sure this peer's IPFS daemon is connected to
|
|
|
|
// other peers IPFS daemons.
|
2017-03-27 13:07:12 +00:00
|
|
|
ConnectSwarms() error
|
|
|
|
// ConfigKey returns the value for a configuration key.
|
|
|
|
// Subobjects are reached with keypaths as "Parent/Child/GrandChild...".
|
|
|
|
ConfigKey(keypath string) (interface{}, error)
|
2017-08-02 22:49:25 +00:00
|
|
|
// FreeSpace returns the amount of remaining space on the repo, calculated from
|
|
|
|
//"repo stat"
|
2017-10-26 11:45:33 +00:00
|
|
|
FreeSpace() (uint64, error)
|
2017-03-27 13:07:12 +00:00
|
|
|
// RepoSize returns the current repository size as expressed
|
|
|
|
// by "repo stat".
|
2017-10-26 11:45:33 +00:00
|
|
|
RepoSize() (uint64, error)
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Peered represents a component which needs to be aware of the peers
|
2016-12-08 16:24:38 +00:00
|
|
|
// in the Cluster and of any changes to the peer set.
|
2016-12-02 18:33:39 +00:00
|
|
|
type Peered interface {
|
|
|
|
AddPeer(p peer.ID)
|
|
|
|
RmPeer(p peer.ID)
|
2017-01-30 12:12:25 +00:00
|
|
|
//SetPeers(peers []peer.ID)
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2016-12-06 21:29:59 +00:00
|
|
|
// PinTracker represents a component which tracks the status of
|
|
|
|
// the pins in this cluster and ensures they are in sync with the
|
|
|
|
// IPFS daemon. This component should be thread safe.
|
|
|
|
type PinTracker interface {
|
2016-12-29 17:38:09 +00:00
|
|
|
Component
|
2016-12-19 17:35:24 +00:00
|
|
|
// Track tells the tracker that a Cid is now under its supervision
|
|
|
|
// The tracker may decide to perform an IPFS pin.
|
2017-03-08 15:57:27 +00:00
|
|
|
Track(api.Pin) error
|
2016-12-19 17:35:24 +00:00
|
|
|
// Untrack tells the tracker that a Cid is to be forgotten. The tracker
|
|
|
|
// may perform an IPFS unpin operation.
|
|
|
|
Untrack(*cid.Cid) error
|
2017-01-25 18:38:23 +00:00
|
|
|
// StatusAll returns the list of pins with their local status.
|
2017-02-08 17:04:08 +00:00
|
|
|
StatusAll() []api.PinInfo
|
2017-01-25 18:38:23 +00:00
|
|
|
// Status returns the local status of a given Cid.
|
2017-02-08 17:04:08 +00:00
|
|
|
Status(*cid.Cid) api.PinInfo
|
2017-01-25 18:38:23 +00:00
|
|
|
// SyncAll makes sure that all tracked Cids reflect the real IPFS status.
|
2017-01-25 17:07:19 +00:00
|
|
|
// It returns the list of pins which were updated by the call.
|
2017-02-08 17:04:08 +00:00
|
|
|
SyncAll() ([]api.PinInfo, error)
|
2017-01-25 18:38:23 +00:00
|
|
|
// Sync makes sure that the Cid status reflect the real IPFS status.
|
|
|
|
// It returns the local status of the Cid.
|
2017-02-08 17:04:08 +00:00
|
|
|
Sync(*cid.Cid) (api.PinInfo, error)
|
2016-12-20 18:51:13 +00:00
|
|
|
// Recover retriggers a Pin/Unpin operation in Cids with error status.
|
2017-02-08 17:04:08 +00:00
|
|
|
Recover(*cid.Cid) (api.PinInfo, error)
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
2017-02-13 15:46:53 +00:00
|
|
|
|
2017-02-15 14:16:16 +00:00
|
|
|
// Informer provides Metric information from a peer. The metrics produced by
|
2017-02-13 15:46:53 +00:00
|
|
|
// informers are then passed to a PinAllocator which will use them to
|
2017-02-15 14:16:16 +00:00
|
|
|
// determine where to pin content. The metric is agnostic to the rest of
|
|
|
|
// Cluster.
|
2017-02-13 15:46:53 +00:00
|
|
|
type Informer interface {
|
|
|
|
Component
|
|
|
|
Name() string
|
|
|
|
GetMetric() api.Metric
|
|
|
|
}
|
|
|
|
|
|
|
|
// PinAllocator decides where to pin certain content. In order to make such
|
|
|
|
// decision, it receives the pin arguments, the peers which are currently
|
|
|
|
// allocated to the content and metrics available for all peers which could
|
|
|
|
// allocate the content.
|
|
|
|
type PinAllocator interface {
|
|
|
|
Component
|
|
|
|
// Allocate returns the list of peers that should be assigned to
|
|
|
|
// Pin content in oder of preference (from the most preferred to the
|
2017-02-15 14:16:16 +00:00
|
|
|
// least). The "current" map contains valid metrics for peers
|
2017-02-13 15:46:53 +00:00
|
|
|
// which are currently pinning the content. The candidates map
|
2017-02-15 14:16:16 +00:00
|
|
|
// contains the metrics for all peers which are eligible for pinning
|
2017-02-13 15:46:53 +00:00
|
|
|
// the content.
|
|
|
|
Allocate(c *cid.Cid, current, candidates map[peer.ID]api.Metric) ([]peer.ID, error)
|
|
|
|
}
|
|
|
|
|
|
|
|
// PeerMonitor is a component in charge of monitoring the peers in the cluster
|
|
|
|
// and providing candidates to the PinAllocator when a pin request arrives.
|
|
|
|
type PeerMonitor interface {
|
|
|
|
Component
|
|
|
|
// LogMetric stores a metric. Metrics are pushed reguarly from each peer
|
|
|
|
// to the active PeerMonitor.
|
|
|
|
LogMetric(api.Metric)
|
|
|
|
// LastMetrics returns a map with the latest metrics of matching name
|
|
|
|
// for the current cluster peers.
|
|
|
|
LastMetrics(name string) []api.Metric
|
|
|
|
// Alerts delivers alerts generated when this peer monitor detects
|
|
|
|
// a problem (i.e. metrics not arriving as expected). Alerts are used to
|
|
|
|
// trigger rebalancing operations.
|
|
|
|
Alerts() <-chan api.Alert
|
|
|
|
}
|