From e9857652f2e30b02403464122cca49ca65832ece Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 20 Oct 2021 11:38:12 +0200 Subject: [PATCH] Add a timestamp to Pins This adds a Timestamp field to the pin objects. This allows to track when they were pinned. This: * Allows the pin-tracker to actually show accurate information on when the pin entered the system for pins that are not part of ongoing operations (currently it shows time.Now()) * Adds support for reporting timestamp on a pinning services api. --- api/pb/types.pb.go | 74 ++++++++++++++++++------------- api/pb/types.proto | 1 + api/types.go | 23 ++++++++-- cluster.go | 5 +++ pintracker/stateless/stateless.go | 6 ++- 5 files changed, 72 insertions(+), 37 deletions(-) diff --git a/api/pb/types.pb.go b/api/pb/types.pb.go index cce94a63..35bca0ac 100644 --- a/api/pb/types.pb.go +++ b/api/pb/types.pb.go @@ -91,6 +91,7 @@ type Pin struct { MaxDepth int32 `protobuf:"zigzag32,4,opt,name=MaxDepth,proto3" json:"MaxDepth,omitempty"` Reference []byte `protobuf:"bytes,5,opt,name=Reference,proto3" json:"Reference,omitempty"` Options *PinOptions `protobuf:"bytes,6,opt,name=Options,proto3" json:"Options,omitempty"` + Timestamp uint64 `protobuf:"varint,7,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"` } func (x *Pin) Reset() { @@ -167,6 +168,13 @@ func (x *Pin) GetOptions() *PinOptions { return nil } +func (x *Pin) GetTimestamp() uint64 { + if x != nil { + return x.Timestamp + } + return 0 +} + type PinOptions struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -274,7 +282,7 @@ var File_types_proto protoreflect.FileDescriptor var file_types_proto_rawDesc = []byte{ 0x0a, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x61, - 0x70, 0x69, 0x2e, 0x70, 0x62, 0x22, 0xa1, 0x02, 0x0a, 0x03, 0x50, 0x69, 0x6e, 0x12, 0x10, 0x0a, + 0x70, 0x69, 0x2e, 0x70, 0x62, 0x22, 0xbf, 0x02, 0x0a, 0x03, 0x50, 0x69, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x43, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x43, 0x69, 0x64, 0x12, 0x27, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x13, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x2e, 0x50, 0x69, 0x6e, 0x54, 0x79, @@ -287,37 +295,39 @@ var file_types_proto_rawDesc = []byte{ 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2c, 0x0a, 0x07, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x22, 0x55, 0x0a, 0x07, 0x50, 0x69, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, - 0x07, 0x42, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x61, - 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, - 0x54, 0x79, 0x70, 0x65, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x44, 0x41, 0x47, 0x54, 0x79, 0x70, 0x65, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0x04, 0x22, 0xfb, 0x02, 0x0a, 0x0a, 0x50, 0x69, - 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x52, 0x65, 0x70, 0x6c, - 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x4d, 0x69, 0x6e, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x11, 0x52, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x4d, 0x69, 0x6e, 0x12, 0x32, 0x0a, 0x14, - 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, - 0x72, 0x4d, 0x61, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x11, 0x52, 0x14, 0x52, 0x65, 0x70, 0x6c, - 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x4d, 0x61, 0x78, - 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x53, 0x69, 0x7a, - 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x53, 0x69, - 0x7a, 0x65, 0x12, 0x3c, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x69, - 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x69, 0x6e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x09, 0x50, 0x69, 0x6e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x1a, - 0x0a, 0x08, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x08, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x4f, 0x72, - 0x69, 0x67, 0x69, 0x6e, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x4f, 0x72, 0x69, - 0x67, 0x69, 0x6e, 0x73, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x42, 0x06, 0x5a, 0x04, 0x2e, 0x3b, 0x70, 0x62, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x22, 0x55, 0x0a, 0x07, 0x50, 0x69, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x42, + 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x61, 0x74, 0x61, + 0x54, 0x79, 0x70, 0x65, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x54, 0x79, + 0x70, 0x65, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x44, + 0x41, 0x47, 0x54, 0x79, 0x70, 0x65, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0x04, 0x22, 0xfb, 0x02, 0x0a, 0x0a, 0x50, 0x69, 0x6e, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x4d, 0x69, 0x6e, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x11, 0x52, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x4d, 0x69, 0x6e, 0x12, 0x32, 0x0a, 0x14, 0x52, 0x65, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x4d, + 0x61, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x11, 0x52, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x4d, 0x61, 0x78, 0x12, 0x12, + 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, + 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x53, 0x69, 0x7a, 0x65, + 0x12, 0x3c, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1c, + 0x0a, 0x09, 0x50, 0x69, 0x6e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x09, 0x50, 0x69, 0x6e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x0a, 0x08, + 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, + 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x4f, 0x72, 0x69, 0x67, + 0x69, 0x6e, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x4f, 0x72, 0x69, 0x67, 0x69, + 0x6e, 0x73, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a, + 0x04, 0x08, 0x05, 0x10, 0x06, 0x42, 0x06, 0x5a, 0x04, 0x2e, 0x3b, 0x70, 0x62, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/api/pb/types.proto b/api/pb/types.proto index 9cde5075..9df95806 100644 --- a/api/pb/types.proto +++ b/api/pb/types.proto @@ -18,6 +18,7 @@ message Pin { sint32 MaxDepth = 4; bytes Reference = 5; PinOptions Options = 6; + uint64 Timestamp = 7; } message PinOptions { diff --git a/api/types.go b/api/types.go index 6daa300e..eb88e37f 100644 --- a/api/types.go +++ b/api/types.go @@ -832,6 +832,9 @@ type Pin struct { // it is the previous shard CID. // When not needed the pointer is nil Reference *cid.Cid `json:"reference" codec:"r,omitempty"` + + // The time that the pin was submitted to the consensus layer. + Timestamp time.Time `json:"timestamp" codec:"i,omitempty"` } // String is a string representation of a Pin. @@ -866,6 +869,7 @@ func PinCid(c cid.Cid) *Pin { Type: DataType, Allocations: []peer.ID{}, MaxDepth: -1, // Recursive + Timestamp: time.Now(), } } @@ -915,6 +919,12 @@ func (pin *Pin) ProtoMarshal() ([]byte, error) { expireAtProto = uint64(pin.ExpireAt.Unix()) } + var timestampProto uint64 + // Only set the protobuf field with non-zero times. + if !(pin.Timestamp.IsZero() || pin.Timestamp.Equal(unixZero)) { + timestampProto = uint64(pin.Timestamp.Unix()) + } + opts := &pb.PinOptions{ ReplicationFactorMin: int32(pin.ReplicationFactorMin), ReplicationFactorMax: int32(pin.ReplicationFactorMax), @@ -934,6 +944,7 @@ func (pin *Pin) ProtoMarshal() ([]byte, error) { Allocations: allocs, MaxDepth: int32(pin.MaxDepth), Options: opts, + Timestamp: timestampProto, } if ref := pin.Reference; ref != nil { pbPin.Reference = ref.Bytes() @@ -978,15 +989,21 @@ func (pin *Pin) ProtoUnmarshal(data []byte) error { pin.Reference = &ref } + ts := pbPin.GetTimestamp() + if ts > 0 { + pin.Timestamp = time.Unix(int64(ts), 0) + } + opts := pbPin.GetOptions() pin.ReplicationFactorMin = int(opts.GetReplicationFactorMin()) pin.ReplicationFactorMax = int(opts.GetReplicationFactorMax()) pin.Name = opts.GetName() pin.ShardSize = opts.GetShardSize() + // pin.UserAllocations = opts.GetUserAllocations() - t := opts.GetExpireAt() - if t > 0 { - pin.ExpireAt = time.Unix(int64(t), 0) + exp := opts.GetExpireAt() + if exp > 0 { + pin.ExpireAt = time.Unix(int64(exp), 0) } pin.Metadata = opts.GetMetadata() pinUpdate, err := cid.Cast(opts.GetPinUpdate()) diff --git a/cluster.go b/cluster.go index c9e9227d..d613ba00 100644 --- a/cluster.go +++ b/cluster.go @@ -1373,6 +1373,10 @@ func (c *Cluster) setupPin(ctx context.Context, pin, existing *api.Pin) error { _, span := trace.StartSpan(ctx, "cluster/setupPin") defer span.End() + // Set the Pin timestamp to now(). This is not an user-controllable + // "option". + pin.Timestamp = time.Now() + err := c.setupReplicationFactor(pin) if err != nil { return err @@ -1580,6 +1584,7 @@ func (c *Cluster) PinUpdate(ctx context.Context, from cid.Cid, to cid.Cid, opts existing.Cid = to existing.PinUpdate = from + existing.Timestamp = time.Now() if opts.Name != "" { existing.Name = opts.Name } diff --git a/pintracker/stateless/stateless.go b/pintracker/stateless/stateless.go index 445c6e5d..d280810c 100644 --- a/pintracker/stateless/stateless.go +++ b/pintracker/stateless/stateless.go @@ -342,6 +342,7 @@ func (spt *Tracker) Status(ctx context.Context, c cid.Cid) *api.PinInfo { } // The pin IS in the state. pinInfo.Name = gpin.Name + pinInfo.TS = gpin.Timestamp // check if pin is a meta pin if gpin.Type == api.MetaType { @@ -464,7 +465,7 @@ func (spt *Tracker) ipfsStatusAll(ctx context.Context) (map[cid.Cid]*api.PinInfo PinInfoShort: api.PinInfoShort{ PeerName: spt.peerName, Status: ips.ToTrackerStatus(), - TS: time.Now(), + TS: time.Now(), // to be set later }, } pins[c] = p @@ -524,7 +525,7 @@ func (spt *Tracker) localStatus(ctx context.Context, incExtra bool, filter api.T Peer: spt.peerID, PinInfoShort: api.PinInfoShort{ PeerName: spt.peerName, - TS: time.Now(), + TS: p.Timestamp, }, } @@ -543,6 +544,7 @@ func (spt *Tracker) localStatus(ctx context.Context, incExtra bool, filter api.T pininfos[p.Cid] = &pinInfo case pinnedInIpfs: // always false unless filter matches TrackerStatusPinnned ipfsInfo.Name = p.Name + ipfsInfo.TS = p.Timestamp pininfos[p.Cid] = ipfsInfo default: // report as UNEXPECTEDLY_UNPINNED for this peer.