Fix #1547: Store pin metadata in a sorted array
This deprecates the Metadata protobuf map and starts serializing metadata as an array that is always sorted by the metadata key. This should resolve the issue that an state export file can be imported resulting in two different CRDT dags because the binary representation of the pin can arbitrarily change depending on how the keys in the map are listed, and thus the CID of a delta is impacted. So with this, a state export should result in exactly the same DAG regardless of where it is imported.
This commit is contained in:
Hector Sanjuan
parent
03740e5bf0
commit
7a05eeeb60
|
|
@ -179,10 +179,12 @@ type PinOptions struct {
|
|||
ReplicationFactorMax int32 `protobuf:"zigzag32,2,opt,name=ReplicationFactorMax,proto3" json:"ReplicationFactorMax,omitempty"`
|
||||
Name string `protobuf:"bytes,3,opt,name=Name,proto3" json:"Name,omitempty"`
|
||||
ShardSize uint64 `protobuf:"varint,4,opt,name=ShardSize,proto3" json:"ShardSize,omitempty"`
|
||||
// Deprecated: Do not use.
|
||||
Metadata map[string]string `protobuf:"bytes,6,rep,name=Metadata,proto3" json:"Metadata,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
|
||||
PinUpdate []byte `protobuf:"bytes,7,opt,name=PinUpdate,proto3" json:"PinUpdate,omitempty"`
|
||||
ExpireAt uint64 `protobuf:"varint,8,opt,name=ExpireAt,proto3" json:"ExpireAt,omitempty"`
|
||||
Origins [][]byte `protobuf:"bytes,9,rep,name=Origins,proto3" json:"Origins,omitempty"`
|
||||
SortedMetadata []*Metadata `protobuf:"bytes,10,rep,name=SortedMetadata,proto3" json:"SortedMetadata,omitempty"`
|
||||
}
|
||||
|
||||
func (x *PinOptions) Reset() {
|
||||
|
|
@ -245,6 +247,7 @@ func (x *PinOptions) GetShardSize() uint64 {
|
|||
return 0
|
||||
}
|
||||
|
||||
// Deprecated: Do not use.
|
||||
func (x *PinOptions) GetMetadata() map[string]string {
|
||||
if x != nil {
|
||||
return x.Metadata
|
||||
|
|
@ -273,6 +276,68 @@ func (x *PinOptions) GetOrigins() [][]byte {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (x *PinOptions) GetSortedMetadata() []*Metadata {
|
||||
if x != nil {
|
||||
return x.SortedMetadata
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Metadata struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Key string `protobuf:"bytes,1,opt,name=Key,proto3" json:"Key,omitempty"`
|
||||
Value string `protobuf:"bytes,2,opt,name=Value,proto3" json:"Value,omitempty"`
|
||||
}
|
||||
|
||||
func (x *Metadata) Reset() {
|
||||
*x = Metadata{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_types_proto_msgTypes[2]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *Metadata) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*Metadata) ProtoMessage() {}
|
||||
|
||||
func (x *Metadata) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_types_proto_msgTypes[2]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use Metadata.ProtoReflect.Descriptor instead.
|
||||
func (*Metadata) Descriptor() ([]byte, []int) {
|
||||
return file_types_proto_rawDescGZIP(), []int{2}
|
||||
}
|
||||
|
||||
func (x *Metadata) GetKey() string {
|
||||
if x != nil {
|
||||
return x.Key
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *Metadata) GetValue() string {
|
||||
if x != nil {
|
||||
return x.Value
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var File_types_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_types_proto_rawDesc = []byte{
|
||||
|
|
@ -297,7 +362,7 @@ var file_types_proto_rawDesc = []byte{
|
|||
0x54, 0x79, 0x70, 0x65, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x54, 0x79,
|
||||
0x70, 0x65, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x44,
|
||||
0x41, 0x47, 0x54, 0x79, 0x70, 0x65, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72,
|
||||
0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0x04, 0x22, 0xfb, 0x02, 0x0a, 0x0a, 0x50, 0x69, 0x6e, 0x4f,
|
||||
0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0x04, 0x22, 0xb9, 0x03, 0x0a, 0x0a, 0x50, 0x69, 0x6e, 0x4f,
|
||||
0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63,
|
||||
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x4d, 0x69, 0x6e, 0x18, 0x01,
|
||||
0x20, 0x01, 0x28, 0x11, 0x52, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f,
|
||||
|
|
@ -308,21 +373,28 @@ var file_types_proto_rawDesc = []byte{
|
|||
0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61,
|
||||
0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x18,
|
||||
0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x53, 0x69, 0x7a, 0x65,
|
||||
0x12, 0x3c, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x03,
|
||||
0x12, 0x40, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x03,
|
||||
0x28, 0x0b, 0x32, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x4f,
|
||||
0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45,
|
||||
0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1c,
|
||||
0x0a, 0x09, 0x50, 0x69, 0x6e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28,
|
||||
0x0c, 0x52, 0x09, 0x50, 0x69, 0x6e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x0a, 0x08,
|
||||
0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08,
|
||||
0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x4f, 0x72, 0x69, 0x67,
|
||||
0x69, 0x6e, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x4f, 0x72, 0x69, 0x67, 0x69,
|
||||
0x6e, 0x73, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e,
|
||||
0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
|
||||
0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02,
|
||||
0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a,
|
||||
0x04, 0x08, 0x05, 0x10, 0x06, 0x42, 0x06, 0x5a, 0x04, 0x2e, 0x3b, 0x70, 0x62, 0x62, 0x06, 0x70,
|
||||
0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x6e, 0x74, 0x72, 0x79, 0x42, 0x02, 0x18, 0x01, 0x52, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61,
|
||||
0x74, 0x61, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x69, 0x6e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x18,
|
||||
0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x50, 0x69, 0x6e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65,
|
||||
0x12, 0x1a, 0x0a, 0x08, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x18, 0x08, 0x20, 0x01,
|
||||
0x28, 0x04, 0x52, 0x08, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x12, 0x18, 0x0a, 0x07,
|
||||
0x4f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x4f,
|
||||
0x72, 0x69, 0x67, 0x69, 0x6e, 0x73, 0x12, 0x38, 0x0a, 0x0e, 0x53, 0x6f, 0x72, 0x74, 0x65, 0x64,
|
||||
0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10,
|
||||
0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
|
||||
0x52, 0x0e, 0x53, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
|
||||
0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72,
|
||||
0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
|
||||
0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01,
|
||||
0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a, 0x04, 0x08,
|
||||
0x05, 0x10, 0x06, 0x22, 0x32, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12,
|
||||
0x10, 0x0a, 0x03, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x4b, 0x65,
|
||||
0x79, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
|
||||
0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x06, 0x5a, 0x04, 0x2e, 0x3b, 0x70, 0x62, 0x62,
|
||||
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
@ -338,22 +410,24 @@ func file_types_proto_rawDescGZIP() []byte {
|
|||
}
|
||||
|
||||
var file_types_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
|
||||
var file_types_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
|
||||
var file_types_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
|
||||
var file_types_proto_goTypes = []interface{}{
|
||||
(Pin_PinType)(0), // 0: api.pb.Pin.PinType
|
||||
(*Pin)(nil), // 1: api.pb.Pin
|
||||
(*PinOptions)(nil), // 2: api.pb.PinOptions
|
||||
nil, // 3: api.pb.PinOptions.MetadataEntry
|
||||
(*Metadata)(nil), // 3: api.pb.Metadata
|
||||
nil, // 4: api.pb.PinOptions.MetadataEntry
|
||||
}
|
||||
var file_types_proto_depIdxs = []int32{
|
||||
0, // 0: api.pb.Pin.Type:type_name -> api.pb.Pin.PinType
|
||||
2, // 1: api.pb.Pin.Options:type_name -> api.pb.PinOptions
|
||||
3, // 2: api.pb.PinOptions.Metadata:type_name -> api.pb.PinOptions.MetadataEntry
|
||||
3, // [3:3] is the sub-list for method output_type
|
||||
3, // [3:3] is the sub-list for method input_type
|
||||
3, // [3:3] is the sub-list for extension type_name
|
||||
3, // [3:3] is the sub-list for extension extendee
|
||||
0, // [0:3] is the sub-list for field type_name
|
||||
4, // 2: api.pb.PinOptions.Metadata:type_name -> api.pb.PinOptions.MetadataEntry
|
||||
3, // 3: api.pb.PinOptions.SortedMetadata:type_name -> api.pb.Metadata
|
||||
4, // [4:4] is the sub-list for method output_type
|
||||
4, // [4:4] is the sub-list for method input_type
|
||||
4, // [4:4] is the sub-list for extension type_name
|
||||
4, // [4:4] is the sub-list for extension extendee
|
||||
0, // [0:4] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_types_proto_init() }
|
||||
|
|
@ -386,6 +460,18 @@ func file_types_proto_init() {
|
|||
return nil
|
||||
}
|
||||
}
|
||||
file_types_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*Metadata); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
|
|
@ -393,7 +479,7 @@ func file_types_proto_init() {
|
|||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: file_types_proto_rawDesc,
|
||||
NumEnums: 1,
|
||||
NumMessages: 3,
|
||||
NumMessages: 4,
|
||||
NumExtensions: 0,
|
||||
NumServices: 0,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -27,8 +27,14 @@ message PinOptions {
|
|||
string Name = 3;
|
||||
uint64 ShardSize = 4;
|
||||
reserved 5; // reserved for UserAllocations
|
||||
map<string, string> Metadata = 6;
|
||||
map<string, string> Metadata = 6 [deprecated = true];
|
||||
bytes PinUpdate = 7;
|
||||
uint64 ExpireAt = 8;
|
||||
repeated bytes Origins = 9;
|
||||
repeated Metadata SortedMetadata = 10;
|
||||
}
|
||||
|
||||
message Metadata {
|
||||
string Key = 1;
|
||||
string Value = 2;
|
||||
}
|
||||
31
api/types.go
31
api/types.go
|
|
@ -1106,17 +1106,36 @@ func (pin Pin) ProtoMarshal() ([]byte, error) {
|
|||
timestampProto = uint64(pin.Timestamp.Unix())
|
||||
}
|
||||
|
||||
// Our metadata needs to always be seralized in exactly the same way,
|
||||
// and that is why we use an array sorted by key and deprecated using
|
||||
// a protobuf map.
|
||||
var sortedMetadata []*pb.Metadata
|
||||
var metaKeys []string
|
||||
for k := range pin.Metadata {
|
||||
metaKeys = append(metaKeys, k)
|
||||
}
|
||||
sort.Strings(metaKeys)
|
||||
|
||||
for _, k := range metaKeys {
|
||||
metadata := &pb.Metadata{
|
||||
Key: k,
|
||||
Value: pin.Metadata[k],
|
||||
}
|
||||
sortedMetadata = append(sortedMetadata, metadata)
|
||||
}
|
||||
|
||||
opts := &pb.PinOptions{
|
||||
ReplicationFactorMin: int32(pin.ReplicationFactorMin),
|
||||
ReplicationFactorMax: int32(pin.ReplicationFactorMax),
|
||||
Name: pin.Name,
|
||||
ShardSize: pin.ShardSize,
|
||||
Metadata: pin.Metadata,
|
||||
// Metadata: pin.Metadata,
|
||||
PinUpdate: pin.PinUpdate.Bytes(),
|
||||
ExpireAt: expireAtProto,
|
||||
// Mode: pin.Mode,
|
||||
// UserAllocations: pin.UserAllocations,
|
||||
Origins: origins,
|
||||
SortedMetadata: sortedMetadata,
|
||||
}
|
||||
|
||||
pbPin := &pb.Pin{
|
||||
|
|
@ -1186,7 +1205,17 @@ func (pin *Pin) ProtoUnmarshal(data []byte) error {
|
|||
if exp > 0 {
|
||||
pin.ExpireAt = time.Unix(int64(exp), 0)
|
||||
}
|
||||
|
||||
// Use whatever metadata is available.
|
||||
pin.Metadata = opts.GetMetadata()
|
||||
sortedMetadata := opts.GetSortedMetadata()
|
||||
if len(sortedMetadata) > 0 && pin.Metadata == nil {
|
||||
pin.Metadata = make(map[string]string, len(sortedMetadata))
|
||||
}
|
||||
for _, md := range opts.GetSortedMetadata() {
|
||||
pin.Metadata[md.Key] = md.Value
|
||||
}
|
||||
|
||||
pinUpdate, err := CastCid(opts.GetPinUpdate())
|
||||
if err == nil {
|
||||
pin.PinUpdate = pinUpdate
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user