cascade/ipfs-cluster
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into dependency-upgrades

Browse Source
This commit is contained in:
Hector Sanjuan 2021-10-27 15:55:34 +02:00
commit 3ddda1fb59
9 changed files with 91 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
allocator/balanced/config.go
View File

@ -8,8 +8,8 @@ import (
"github.com/kelseyhightower/envconfig"
)
const configKey = "metricsalloc"
const envConfigKey = "cluster_metricsalloc"
const configKey = "balanced"
const envConfigKey = "cluster_balanced"
// These are the default values for a Config.
var (

2
allocator/balanced/config_test.go
View File

@ -51,7 +51,7 @@ func TestDefault(t *testing.T) {
}
func TestApplyEnvVars(t *testing.T) {
os.Setenv("CLUSTER_METRICSALLOC_ALLOCATEBY", "a,b,c")
os.Setenv("CLUSTER_BALANCED_ALLOCATEBY", "a,b,c")
cfg := &Config{}
cfg.ApplyEnvVars()

74
api/pb/types.pb.go
View File

@ -91,6 +91,7 @@ type Pin struct {
MaxDepth int32 `protobuf:"zigzag32,4,opt,name=MaxDepth,proto3" json:"MaxDepth,omitempty"`
Reference []byte `protobuf:"bytes,5,opt,name=Reference,proto3" json:"Reference,omitempty"`
Options *PinOptions `protobuf:"bytes,6,opt,name=Options,proto3" json:"Options,omitempty"`
Timestamp uint64 `protobuf:"varint,7,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"`
}
func (x *Pin) Reset() {
@ -167,6 +168,13 @@ func (x *Pin) GetOptions() *PinOptions {
return nil
}
func (x *Pin) GetTimestamp() uint64 {
if x != nil {
return x.Timestamp
}
return 0
}
type PinOptions struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@ -274,7 +282,7 @@ var File_types_proto protoreflect.FileDescriptor
var file_types_proto_rawDesc = []byte{
0x0a, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x61,
0x70, 0x69, 0x2e, 0x70, 0x62, 0x22, 0xa1, 0x02, 0x0a, 0x03, 0x50, 0x69, 0x6e, 0x12, 0x10, 0x0a,
0x70, 0x69, 0x2e, 0x70, 0x62, 0x22, 0xbf, 0x02, 0x0a, 0x03, 0x50, 0x69, 0x6e, 0x12, 0x10, 0x0a,
0x03, 0x43, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x43, 0x69, 0x64, 0x12,
0x27, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x13, 0x2e,
0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x2e, 0x50, 0x69, 0x6e, 0x54, 0x79,
@ -287,37 +295,39 @@ var file_types_proto_rawDesc = []byte{
0x65, 0x6e, 0x63, 0x65, 0x12, 0x2c, 0x0a, 0x07, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18,
0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x50,
0x69, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x4f, 0x70, 0x74, 0x69, 0x6f,
0x6e, 0x73, 0x22, 0x55, 0x0a, 0x07, 0x50, 0x69, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,
0x07, 0x42, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x61,
0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61,
0x54, 0x79, 0x70, 0x65, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65,
0x72, 0x44, 0x41, 0x47, 0x54, 0x79, 0x70, 0x65, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x68,
0x61, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0x04, 0x22, 0xfb, 0x02, 0x0a, 0x0a, 0x50, 0x69,
0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x52, 0x65, 0x70, 0x6c,
0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x4d, 0x69, 0x6e,
0x18, 0x01, 0x20, 0x01, 0x28, 0x11, 0x52, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x4d, 0x69, 0x6e, 0x12, 0x32, 0x0a, 0x14,
0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f,
0x72, 0x4d, 0x61, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x11, 0x52, 0x14, 0x52, 0x65, 0x70, 0x6c,
0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x4d, 0x61, 0x78,
0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04,
0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x53, 0x69, 0x7a,
0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x53, 0x69,
0x7a, 0x65, 0x12, 0x3c, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06,
0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x69,
0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74,
0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
0x12, 0x1c, 0x0a, 0x09, 0x50, 0x69, 0x6e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x18, 0x07, 0x20,
0x01, 0x28, 0x0c, 0x52, 0x09, 0x50, 0x69, 0x6e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x1a,
0x0a, 0x08, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04,
0x52, 0x08, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x4f, 0x72,
0x69, 0x67, 0x69, 0x6e, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x4f, 0x72, 0x69,
0x67, 0x69, 0x6e, 0x73, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38,
0x01, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x42, 0x06, 0x5a, 0x04, 0x2e, 0x3b, 0x70, 0x62, 0x62,
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18,
0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
0x22, 0x55, 0x0a, 0x07, 0x50, 0x69, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x42,
0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x61, 0x74, 0x61,
0x54, 0x79, 0x70, 0x65, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x54, 0x79,
0x70, 0x65, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x44,
0x41, 0x47, 0x54, 0x79, 0x70, 0x65, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72,
0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0x04, 0x22, 0xfb, 0x02, 0x0a, 0x0a, 0x50, 0x69, 0x6e, 0x4f,
0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x4d, 0x69, 0x6e, 0x18, 0x01,
0x20, 0x01, 0x28, 0x11, 0x52, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f,
0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x4d, 0x69, 0x6e, 0x12, 0x32, 0x0a, 0x14, 0x52, 0x65,
0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x4d,
0x61, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x11, 0x52, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x4d, 0x61, 0x78, 0x12, 0x12,
0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61,
0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x18,
0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x53, 0x69, 0x7a, 0x65,
0x12, 0x3c, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x03,
0x28, 0x0b, 0x32, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x4f,
0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45,
0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1c,
0x0a, 0x09, 0x50, 0x69, 0x6e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28,
0x0c, 0x52, 0x09, 0x50, 0x69, 0x6e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x0a, 0x08,
0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08,
0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x4f, 0x72, 0x69, 0x67,
0x69, 0x6e, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x4f, 0x72, 0x69, 0x67, 0x69,
0x6e, 0x73, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e,
0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02,
0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a,
0x04, 0x08, 0x05, 0x10, 0x06, 0x42, 0x06, 0x5a, 0x04, 0x2e, 0x3b, 0x70, 0x62, 0x62, 0x06, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (

1
api/pb/types.proto
View File

@ -18,6 +18,7 @@ message Pin {
sint32 MaxDepth = 4;
bytes Reference = 5;
PinOptions Options = 6;
uint64 Timestamp = 7;
}
message PinOptions {

23
api/types.go
View File

@ -829,6 +829,9 @@ type Pin struct {
// it is the previous shard CID.
// When not needed the pointer is nil
Reference *cid.Cid `json:"reference" codec:"r,omitempty"`
// The time that the pin was submitted to the consensus layer.
Timestamp time.Time `json:"timestamp" codec:"i,omitempty"`
}
// String is a string representation of a Pin.
@ -863,6 +866,7 @@ func PinCid(c cid.Cid) *Pin {
Type: DataType,
Allocations: []peer.ID{},
MaxDepth: -1, // Recursive
Timestamp: time.Now(),
}
}
@ -912,6 +916,12 @@ func (pin *Pin) ProtoMarshal() ([]byte, error) {
expireAtProto = uint64(pin.ExpireAt.Unix())
}
var timestampProto uint64
// Only set the protobuf field with non-zero times.
if !(pin.Timestamp.IsZero() || pin.Timestamp.Equal(unixZero)) {
timestampProto = uint64(pin.Timestamp.Unix())
}
opts := &pb.PinOptions{
ReplicationFactorMin: int32(pin.ReplicationFactorMin),
ReplicationFactorMax: int32(pin.ReplicationFactorMax),
@ -931,6 +941,7 @@ func (pin *Pin) ProtoMarshal() ([]byte, error) {
Allocations: allocs,
MaxDepth: int32(pin.MaxDepth),
Options: opts,
Timestamp: timestampProto,
}
if ref := pin.Reference; ref != nil {
pbPin.Reference = ref.Bytes()
@ -975,15 +986,21 @@ func (pin *Pin) ProtoUnmarshal(data []byte) error {
pin.Reference = &ref
}
ts := pbPin.GetTimestamp()
if ts > 0 {
pin.Timestamp = time.Unix(int64(ts), 0)
}
opts := pbPin.GetOptions()
pin.ReplicationFactorMin = int(opts.GetReplicationFactorMin())
pin.ReplicationFactorMax = int(opts.GetReplicationFactorMax())
pin.Name = opts.GetName()
pin.ShardSize = opts.GetShardSize()
// pin.UserAllocations = opts.GetUserAllocations()
t := opts.GetExpireAt()
if t > 0 {
pin.ExpireAt = time.Unix(int64(t), 0)
exp := opts.GetExpireAt()
if exp > 0 {
pin.ExpireAt = time.Unix(int64(exp), 0)
}
pin.Metadata = opts.GetMetadata()
pinUpdate, err := cid.Cast(opts.GetPinUpdate())

5
cluster.go
View File

@ -1373,6 +1373,10 @@ func (c *Cluster) setupPin(ctx context.Context, pin, existing *api.Pin) error {
_, span := trace.StartSpan(ctx, "cluster/setupPin")
defer span.End()
// Set the Pin timestamp to now(). This is not an user-controllable
// "option".
pin.Timestamp = time.Now()
err := c.setupReplicationFactor(pin)
if err != nil {
return err
@ -1580,6 +1584,7 @@ func (c *Cluster) PinUpdate(ctx context.Context, from cid.Cid, to cid.Cid, opts
existing.Cid = to
existing.PinUpdate = from
existing.Timestamp = time.Now()
if opts.Name != "" {
existing.Name = opts.Name
}

12
cmd/ipfs-cluster-ctl/formatters.go
View File

@ -216,11 +216,17 @@ func textFormatPrintPin(obj *api.Pin) {
} else {
fmt.Printf(" yes")
}
expireAt := "Exp: ∞"
expireAt := "∞"
if !obj.ExpireAt.IsZero() {
expireAt = humanize.Time(obj.ExpireAt)
expireAt = obj.Timestamp.Format("2006-01-02 15:04:05")
}
fmt.Printf(" | %s\n", expireAt)
fmt.Printf(" | Exp: %s", expireAt)
added := "unknown"
if !obj.Timestamp.IsZero() {
added = obj.Timestamp.Format("2006-01-02 15:04:05")
}
fmt.Printf(" | Added: %s\n", added)
}
func textFormatPrintAddedOutput(obj *api.AddedOutput) {

7
cmd/ipfs-cluster-service/daemon.go
View File

@ -168,6 +168,13 @@ func createCluster(
informers = append(informers, tagsinf)
}
// For legacy compatibility we need to make the allocator
// automatically compatible with informers that have been loaded. For
// simplicity we assume that anyone that does not specify an allocator
// configuration (legacy configs), will be using "freespace"
if !cfgMgr.IsLoadedFromJSON(config.Allocator, cfgs.BalancedAlloc.ConfigKey()) {
cfgs.BalancedAlloc.AllocateBy = []string{"freespace"}
}
alloc, err := balanced.New(cfgs.BalancedAlloc)
checkErr("creating allocator", err)

6
pintracker/stateless/stateless.go
View File

@ -342,6 +342,7 @@ func (spt *Tracker) Status(ctx context.Context, c cid.Cid) *api.PinInfo {
}
// The pin IS in the state.
pinInfo.Name = gpin.Name
pinInfo.TS = gpin.Timestamp
// check if pin is a meta pin
if gpin.Type == api.MetaType {
@ -464,7 +465,7 @@ func (spt *Tracker) ipfsStatusAll(ctx context.Context) (map[cid.Cid]*api.PinInfo
PinInfoShort: api.PinInfoShort{
PeerName: spt.peerName,
Status: ips.ToTrackerStatus(),
TS: time.Now(),
TS: time.Now(), // to be set later
},
}
pins[c] = p
@ -524,7 +525,7 @@ func (spt *Tracker) localStatus(ctx context.Context, incExtra bool, filter api.T
Peer: spt.peerID,
PinInfoShort: api.PinInfoShort{
PeerName: spt.peerName,
TS: time.Now(),
TS: p.Timestamp,
},
}
@ -543,6 +544,7 @@ func (spt *Tracker) localStatus(ctx context.Context, incExtra bool, filter api.T
pininfos[p.Cid] = &pinInfo
case pinnedInIpfs: // always false unless filter matches TrackerStatusPinnned
ipfsInfo.Name = p.Name
ipfsInfo.TS = p.Timestamp
pininfos[p.Cid] = ipfsInfo
default:
// report as UNEXPECTEDLY_UNPINNED for this peer.