9b9d76f92d
This commit introduces the new go-libp2p-gorpc streaming capabilities for Cluster. The main aim is to work towards heavily reducing memory usage when working with very large pinsets. As a side-effect, it takes the chance to revampt all types for all public methods so that pointers to static what should be static objects are not used anymore. This should heavily reduce heap allocations and GC activity. The main change is that state.List now returns a channel from which to read the pins, rather than pins being all loaded into a huge slice. Things reading pins have been all updated to iterate on the channel rather than on the slice. The full pinset is no longer fully loaded onto memory for things that run regularly like StateSync(). Additionally, the /allocations endpoint of the rest API no longer returns an array of pins, but rather streams json-encoded pin objects directly. This change has extended to the restapi client (which puts pins into a channel as they arrive) and to ipfs-cluster-ctl. There are still pending improvements like StatusAll() calls which should also stream responses, and specially BlockPut calls which should stream blocks directly into IPFS on a single call. These are coming up in future commits.
99 lines
2.2 KiB
Go
99 lines
2.2 KiB
Go
// Package numpin implements an ipfs-cluster informer which determines how many
|
|
// items this peer is pinning and returns it as api.Metric
|
|
package numpin
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
"github.com/ipfs/ipfs-cluster/api"
|
|
|
|
rpc "github.com/libp2p/go-libp2p-gorpc"
|
|
|
|
"go.opencensus.io/trace"
|
|
)
|
|
|
|
// MetricName specifies the name of our metric
|
|
var MetricName = "numpin"
|
|
|
|
// Informer is a simple object to implement the ipfscluster.Informer
|
|
// and Component interfaces
|
|
type Informer struct {
|
|
config *Config
|
|
rpcClient *rpc.Client
|
|
}
|
|
|
|
// NewInformer returns an initialized Informer.
|
|
func NewInformer(cfg *Config) (*Informer, error) {
|
|
err := cfg.Validate()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &Informer{
|
|
config: cfg,
|
|
}, nil
|
|
}
|
|
|
|
// SetClient provides us with an rpc.Client which allows
|
|
// contacting other components in the cluster.
|
|
func (npi *Informer) SetClient(c *rpc.Client) {
|
|
npi.rpcClient = c
|
|
}
|
|
|
|
// Shutdown is called on cluster shutdown. We just invalidate
|
|
// any metrics from this point.
|
|
func (npi *Informer) Shutdown(ctx context.Context) error {
|
|
_, span := trace.StartSpan(ctx, "informer/numpin/Shutdown")
|
|
defer span.End()
|
|
|
|
npi.rpcClient = nil
|
|
return nil
|
|
}
|
|
|
|
// Name returns the name of this informer
|
|
func (npi *Informer) Name() string {
|
|
return MetricName
|
|
}
|
|
|
|
// GetMetrics contacts the IPFSConnector component and requests the `pin ls`
|
|
// command. We return the number of pins in IPFS. It must always return at
|
|
// least one metric.
|
|
func (npi *Informer) GetMetrics(ctx context.Context) []api.Metric {
|
|
ctx, span := trace.StartSpan(ctx, "informer/numpin/GetMetric")
|
|
defer span.End()
|
|
|
|
if npi.rpcClient == nil {
|
|
return []api.Metric{
|
|
{
|
|
Valid: false,
|
|
},
|
|
}
|
|
}
|
|
|
|
pinMap := make(map[string]api.IPFSPinStatus)
|
|
|
|
// make use of the RPC API to obtain information
|
|
// about the number of pins in IPFS. See RPCAPI docs.
|
|
err := npi.rpcClient.CallContext(
|
|
ctx,
|
|
"", // Local call
|
|
"IPFSConnector", // Service name
|
|
"PinLs", // Method name
|
|
"recursive", // in arg
|
|
&pinMap, // out arg
|
|
)
|
|
|
|
valid := err == nil
|
|
|
|
m := api.Metric{
|
|
Name: MetricName,
|
|
Value: fmt.Sprintf("%d", len(pinMap)),
|
|
Valid: valid,
|
|
Partitionable: false,
|
|
}
|
|
|
|
m.SetTTL(npi.config.MetricTTL)
|
|
return []api.Metric{m}
|
|
}
|