From 7a05eeeb6058da339a03aa84f325fe2030ca2511 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 6 May 2022 00:37:59 +0200 Subject: [PATCH] Fix #1547: Store pin metadata in a sorted array This deprecates the Metadata protobuf map and starts serializing metadata as an array that is always sorted by the metadata key. This should resolve the issue that an state export file can be imported resulting in two different CRDT dags because the binary representation of the pin can arbitrarily change depending on how the keys in the map are listed, and thus the CID of a delta is impacted. So with this, a state export should result in exactly the same DAG regardless of where it is imported. --- api/pb/types.pb.go | 148 +++++++++++++++++++++++++++++++++++---------- api/pb/types.proto | 8 ++- api/types.go | 37 ++++++++++-- 3 files changed, 157 insertions(+), 36 deletions(-) diff --git a/api/pb/types.pb.go b/api/pb/types.pb.go index 2ea69e29..b35a3d93 100644 --- a/api/pb/types.pb.go +++ b/api/pb/types.pb.go @@ -175,14 +175,16 @@ type PinOptions struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ReplicationFactorMin int32 `protobuf:"zigzag32,1,opt,name=ReplicationFactorMin,proto3" json:"ReplicationFactorMin,omitempty"` - ReplicationFactorMax int32 `protobuf:"zigzag32,2,opt,name=ReplicationFactorMax,proto3" json:"ReplicationFactorMax,omitempty"` - Name string `protobuf:"bytes,3,opt,name=Name,proto3" json:"Name,omitempty"` - ShardSize uint64 `protobuf:"varint,4,opt,name=ShardSize,proto3" json:"ShardSize,omitempty"` - Metadata map[string]string `protobuf:"bytes,6,rep,name=Metadata,proto3" json:"Metadata,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - PinUpdate []byte `protobuf:"bytes,7,opt,name=PinUpdate,proto3" json:"PinUpdate,omitempty"` - ExpireAt uint64 `protobuf:"varint,8,opt,name=ExpireAt,proto3" json:"ExpireAt,omitempty"` - Origins [][]byte `protobuf:"bytes,9,rep,name=Origins,proto3" json:"Origins,omitempty"` + ReplicationFactorMin int32 `protobuf:"zigzag32,1,opt,name=ReplicationFactorMin,proto3" json:"ReplicationFactorMin,omitempty"` + ReplicationFactorMax int32 `protobuf:"zigzag32,2,opt,name=ReplicationFactorMax,proto3" json:"ReplicationFactorMax,omitempty"` + Name string `protobuf:"bytes,3,opt,name=Name,proto3" json:"Name,omitempty"` + ShardSize uint64 `protobuf:"varint,4,opt,name=ShardSize,proto3" json:"ShardSize,omitempty"` + // Deprecated: Do not use. + Metadata map[string]string `protobuf:"bytes,6,rep,name=Metadata,proto3" json:"Metadata,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + PinUpdate []byte `protobuf:"bytes,7,opt,name=PinUpdate,proto3" json:"PinUpdate,omitempty"` + ExpireAt uint64 `protobuf:"varint,8,opt,name=ExpireAt,proto3" json:"ExpireAt,omitempty"` + Origins [][]byte `protobuf:"bytes,9,rep,name=Origins,proto3" json:"Origins,omitempty"` + SortedMetadata []*Metadata `protobuf:"bytes,10,rep,name=SortedMetadata,proto3" json:"SortedMetadata,omitempty"` } func (x *PinOptions) Reset() { @@ -245,6 +247,7 @@ func (x *PinOptions) GetShardSize() uint64 { return 0 } +// Deprecated: Do not use. func (x *PinOptions) GetMetadata() map[string]string { if x != nil { return x.Metadata @@ -273,6 +276,68 @@ func (x *PinOptions) GetOrigins() [][]byte { return nil } +func (x *PinOptions) GetSortedMetadata() []*Metadata { + if x != nil { + return x.SortedMetadata + } + return nil +} + +type Metadata struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Key string `protobuf:"bytes,1,opt,name=Key,proto3" json:"Key,omitempty"` + Value string `protobuf:"bytes,2,opt,name=Value,proto3" json:"Value,omitempty"` +} + +func (x *Metadata) Reset() { + *x = Metadata{} + if protoimpl.UnsafeEnabled { + mi := &file_types_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Metadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Metadata) ProtoMessage() {} + +func (x *Metadata) ProtoReflect() protoreflect.Message { + mi := &file_types_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Metadata.ProtoReflect.Descriptor instead. +func (*Metadata) Descriptor() ([]byte, []int) { + return file_types_proto_rawDescGZIP(), []int{2} +} + +func (x *Metadata) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *Metadata) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + var File_types_proto protoreflect.FileDescriptor var file_types_proto_rawDesc = []byte{ @@ -297,7 +362,7 @@ var file_types_proto_rawDesc = []byte{ 0x54, 0x79, 0x70, 0x65, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x44, 0x41, 0x47, 0x54, 0x79, 0x70, 0x65, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0x04, 0x22, 0xfb, 0x02, 0x0a, 0x0a, 0x50, 0x69, 0x6e, 0x4f, + 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0x04, 0x22, 0xb9, 0x03, 0x0a, 0x0a, 0x50, 0x69, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x4d, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x11, 0x52, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, @@ -308,21 +373,28 @@ var file_types_proto_rawDesc = []byte{ 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x53, 0x69, 0x7a, 0x65, - 0x12, 0x3c, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x03, + 0x12, 0x40, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1c, - 0x0a, 0x09, 0x50, 0x69, 0x6e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x09, 0x50, 0x69, 0x6e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x0a, 0x08, - 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, - 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x4f, 0x72, 0x69, 0x67, - 0x69, 0x6e, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x4f, 0x72, 0x69, 0x67, 0x69, - 0x6e, 0x73, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a, - 0x04, 0x08, 0x05, 0x10, 0x06, 0x42, 0x06, 0x5a, 0x04, 0x2e, 0x3b, 0x70, 0x62, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x74, 0x72, 0x79, 0x42, 0x02, 0x18, 0x01, 0x52, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x69, 0x6e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x50, 0x69, 0x6e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x12, 0x1a, 0x0a, 0x08, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x08, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x12, 0x18, 0x0a, 0x07, + 0x4f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x4f, + 0x72, 0x69, 0x67, 0x69, 0x6e, 0x73, 0x12, 0x38, 0x0a, 0x0e, 0x53, 0x6f, 0x72, 0x74, 0x65, 0x64, + 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x52, 0x0e, 0x53, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a, 0x04, 0x08, + 0x05, 0x10, 0x06, 0x22, 0x32, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, + 0x10, 0x0a, 0x03, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x4b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x06, 0x5a, 0x04, 0x2e, 0x3b, 0x70, 0x62, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -338,22 +410,24 @@ func file_types_proto_rawDescGZIP() []byte { } var file_types_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_types_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_types_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_types_proto_goTypes = []interface{}{ (Pin_PinType)(0), // 0: api.pb.Pin.PinType (*Pin)(nil), // 1: api.pb.Pin (*PinOptions)(nil), // 2: api.pb.PinOptions - nil, // 3: api.pb.PinOptions.MetadataEntry + (*Metadata)(nil), // 3: api.pb.Metadata + nil, // 4: api.pb.PinOptions.MetadataEntry } var file_types_proto_depIdxs = []int32{ 0, // 0: api.pb.Pin.Type:type_name -> api.pb.Pin.PinType 2, // 1: api.pb.Pin.Options:type_name -> api.pb.PinOptions - 3, // 2: api.pb.PinOptions.Metadata:type_name -> api.pb.PinOptions.MetadataEntry - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 4, // 2: api.pb.PinOptions.Metadata:type_name -> api.pb.PinOptions.MetadataEntry + 3, // 3: api.pb.PinOptions.SortedMetadata:type_name -> api.pb.Metadata + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_types_proto_init() } @@ -386,6 +460,18 @@ func file_types_proto_init() { return nil } } + file_types_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Metadata); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -393,7 +479,7 @@ func file_types_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_types_proto_rawDesc, NumEnums: 1, - NumMessages: 3, + NumMessages: 4, NumExtensions: 0, NumServices: 0, }, diff --git a/api/pb/types.proto b/api/pb/types.proto index 9df95806..ec39438e 100644 --- a/api/pb/types.proto +++ b/api/pb/types.proto @@ -27,8 +27,14 @@ message PinOptions { string Name = 3; uint64 ShardSize = 4; reserved 5; // reserved for UserAllocations - map Metadata = 6; + map Metadata = 6 [deprecated = true]; bytes PinUpdate = 7; uint64 ExpireAt = 8; repeated bytes Origins = 9; + repeated Metadata SortedMetadata = 10; +} + +message Metadata { + string Key = 1; + string Value = 2; } \ No newline at end of file diff --git a/api/types.go b/api/types.go index 2a7589f1..5149ef91 100644 --- a/api/types.go +++ b/api/types.go @@ -1106,17 +1106,36 @@ func (pin Pin) ProtoMarshal() ([]byte, error) { timestampProto = uint64(pin.Timestamp.Unix()) } + // Our metadata needs to always be seralized in exactly the same way, + // and that is why we use an array sorted by key and deprecated using + // a protobuf map. + var sortedMetadata []*pb.Metadata + var metaKeys []string + for k := range pin.Metadata { + metaKeys = append(metaKeys, k) + } + sort.Strings(metaKeys) + + for _, k := range metaKeys { + metadata := &pb.Metadata{ + Key: k, + Value: pin.Metadata[k], + } + sortedMetadata = append(sortedMetadata, metadata) + } + opts := &pb.PinOptions{ ReplicationFactorMin: int32(pin.ReplicationFactorMin), ReplicationFactorMax: int32(pin.ReplicationFactorMax), Name: pin.Name, ShardSize: pin.ShardSize, - Metadata: pin.Metadata, - PinUpdate: pin.PinUpdate.Bytes(), - ExpireAt: expireAtProto, + // Metadata: pin.Metadata, + PinUpdate: pin.PinUpdate.Bytes(), + ExpireAt: expireAtProto, // Mode: pin.Mode, // UserAllocations: pin.UserAllocations, - Origins: origins, + Origins: origins, + SortedMetadata: sortedMetadata, } pbPin := &pb.Pin{ @@ -1186,7 +1205,17 @@ func (pin *Pin) ProtoUnmarshal(data []byte) error { if exp > 0 { pin.ExpireAt = time.Unix(int64(exp), 0) } + + // Use whatever metadata is available. pin.Metadata = opts.GetMetadata() + sortedMetadata := opts.GetSortedMetadata() + if len(sortedMetadata) > 0 && pin.Metadata == nil { + pin.Metadata = make(map[string]string, len(sortedMetadata)) + } + for _, md := range opts.GetSortedMetadata() { + pin.Metadata[md.Key] = md.Value + } + pinUpdate, err := CastCid(opts.GetPinUpdate()) if err == nil { pin.PinUpdate = pinUpdate