diff --git a/README.md b/README.md index 1030c056..647ddb93 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,8 @@ The following features can be generated: - `unmarshal`: generates a `func (p *YourProto) UnmarshalVT(data []byte)` that behaves similarly to calling `proto.Unmarshal(data, p)` on the message, except the unmarshalling is performed by unrolled codegen without using reflection and allocating as little memory as possible. If the receiver `p` is **not** fully zeroed-out, the unmarshal call will actually behave like `proto.Merge(data, p)`. This is because the `proto.Unmarshal` in the ProtoBuf API is implemented by resetting the destination message and then calling `proto.Merge` on it. To ensure proper `Unmarshal` semantics, ensure you've called `proto.Reset` on your message before calling `UnmarshalVT`, or that your message has been newly allocated. + - The `ignoreUnknownFields` option can be used to ignore unknown fields in protobuf messages and further reduce memory allocations. + - `unmarshal_unsafe` generates a `func (p *YourProto) UnmarshalVTUnsafe(data []byte)` that behaves like `UnmarshalVT`, except it unsafely casts slices of data to `bytes` and `string` fields instead of copying them to newly allocated arrays, so that it performs less allocations. **Data received from the wire has to be left untouched for the lifetime of the message.** Otherwise, the message's `bytes` and `string` fields can be corrupted. - `pool`: generates the following helper methods @@ -122,7 +124,14 @@ message Label { --go-vtproto_opt=pool=vitess.io/vitess/go/vt/proto/query.Row \ --go-vtproto_opt=pool=vitess.io/vitess/go/vt/proto/binlogdata.VStreamRowsResponse \ ``` -6. (Optional) if you want to selectively compile the generate `vtprotobuf` files, the `--vtproto_opt=buildTag=` can be used. + +6. (Optional) If you are handling messages containing unknown fields and don't intend to forward these messages to a tool that might expect these fields, you can ignore them using the `ignoreUnknownFields` option. + + - You can tag messages explicitly in the `.proto` files with `option (vtproto.ignore_unknown_fields)`. Take a look at the example using `option (vtproto.mempool)` above. + + - Alternatively, you can enumerate the objects with `--go-vtproto_opt=ignoreUnknownFields=.` flags passed via the CLI. Take a look at the example using `--go-vtproto_opt=pool=...` above. + +7. (Optional) if you want to selectively compile the generate `vtprotobuf` files, the `--vtproto_opt=buildTag=` can be used. When using this option, the generated code will only be compiled in if a build tag is provided. @@ -133,9 +142,9 @@ message Label { This can be done with type assertions before using `vtprotobuf` generated methods. The `grpc.Codec{}` object (discussed below) shows an example. -7. Compile the `.proto` files in your project. You should see `_vtproto.pb.go` files next to the `.pb.go` and `_grpc.pb.go` files that were already being generated. +8. Compile the `.proto` files in your project. You should see `_vtproto.pb.go` files next to the `.pb.go` and `_grpc.pb.go` files that were already being generated. -8. (Optional) Switch your RPC framework to use the optimized helpers (see following sections) +9. (Optional) Switch your RPC framework to use the optimized helpers (see following sections) ## `vtprotobuf` package and well-known types diff --git a/cmd/protoc-gen-go-vtproto/main.go b/cmd/protoc-gen-go-vtproto/main.go index 90ca1243..e7a2ccf4 100644 --- a/cmd/protoc-gen-go-vtproto/main.go +++ b/cmd/protoc-gen-go-vtproto/main.go @@ -24,8 +24,10 @@ func main() { f.BoolVar(&cfg.AllowEmpty, "allow-empty", false, "allow generation of empty files") cfg.Poolable = generator.NewObjectSet() cfg.PoolableExclude = generator.NewObjectSet() + cfg.IgnoreUnknownFields = generator.NewObjectSet() f.Var(&cfg.Poolable, "pool", "use memory pooling for this object") f.Var(&cfg.PoolableExclude, "pool-exclude", "do not use memory pooling for this object") + f.Var(&cfg.IgnoreUnknownFields, "ignoreUnknownFields", "ignore unknown fields instead of saving them") f.BoolVar(&cfg.Wrap, "wrap", false, "generate wrapper types") f.StringVar(&features, "features", "all", "list of features to generate (separated by '+')") f.StringVar(&cfg.BuildTag, "buildTag", "", "the go:build tag to set on generated files") diff --git a/features/clone/clone.go b/features/clone/clone.go index 187a5a81..9e5d73e5 100644 --- a/features/clone/clone.go +++ b/features/clone/clone.go @@ -154,7 +154,7 @@ func (p *clone) cloneField(lhsBase, rhsBase string, allFieldsNullable bool, fiel func (p *clone) generateCloneMethodsForMessage(proto3 bool, message *protogen.Message) { ccTypeName := message.GoIdent.GoName p.P(`func (m *`, ccTypeName, `) `, cloneName, `() *`, ccTypeName, ` {`) - p.body(!proto3, ccTypeName, message, true) + p.body(!proto3, ccTypeName, message) p.P(`}`) p.P() @@ -169,7 +169,7 @@ func (p *clone) generateCloneMethodsForMessage(proto3 bool, message *protogen.Me // body generates the code for the actual cloning logic of a structure containing the given fields. // In practice, those can be the fields of a message. // The object to be cloned is assumed to be called "m". -func (p *clone) body(allFieldsNullable bool, ccTypeName string, message *protogen.Message, cloneUnknownFields bool) { +func (p *clone) body(allFieldsNullable bool, ccTypeName string, message *protogen.Message) { // The method body for a message or a oneof wrapper always starts with a nil check. p.P(`if m == nil {`) // We use an explicitly typed nil to avoid returning the nil interface in the oneof wrapper @@ -220,7 +220,7 @@ func (p *clone) body(allFieldsNullable bool, ccTypeName string, message *protoge p.cloneField("r", "m", allFieldsNullable, field) } - if cloneUnknownFields && !p.Wrapper() { + if !p.Wrapper() && !p.ShouldIgnoreUnknownFields(message) { // Clone unknown fields, if any p.P(`if len(m.unknownFields) > 0 {`) p.P(`r.unknownFields = make([]byte, len(m.unknownFields))`) diff --git a/features/equal/equal.go b/features/equal/equal.go index 3b51584c..bbba98ba 100644 --- a/features/equal/equal.go +++ b/features/equal/equal.go @@ -113,7 +113,7 @@ func (p *equal) message(proto3 bool, message *protogen.Message) { } } - if p.Wrapper() { + if p.Wrapper() || p.ShouldIgnoreUnknownFields(message) { p.P(`return true`) } else { p.P(`return string(this.unknownFields) == string(that.unknownFields)`) diff --git a/features/marshal/marshalto.go b/features/marshal/marshalto.go index b65f9b5d..d0451d04 100644 --- a/features/marshal/marshalto.go +++ b/features/marshal/marshalto.go @@ -599,7 +599,7 @@ func (p *marshal) message(message *protogen.Message) { p.P(`var l int`) p.P(`_ = l`) - if !p.Wrapper() { + if !p.Wrapper() && !p.ShouldIgnoreUnknownFields(message) { p.P(`if m.unknownFields != nil {`) p.P(`i -= len(m.unknownFields)`) p.P(`copy(dAtA[i:], m.unknownFields)`) diff --git a/features/size/size.go b/features/size/size.go index 23560a27..d7ac60f3 100644 --- a/features/size/size.go +++ b/features/size/size.go @@ -320,7 +320,7 @@ func (p *size) message(message *protogen.Message) { } } } - if !p.Wrapper() { + if !p.Wrapper() && !p.ShouldIgnoreUnknownFields(message) { p.P(`n+=len(m.unknownFields)`) } p.P(`return n`) diff --git a/features/unmarshal/unmarshal.go b/features/unmarshal/unmarshal.go index 0f7bce02..87f6ab69 100644 --- a/features/unmarshal/unmarshal.go +++ b/features/unmarshal/unmarshal.go @@ -856,7 +856,7 @@ func (p *unmarshal) message(proto3 bool, message *protogen.Message) { p.P(`iNdEx += skippy`) p.P(`} else {`) } - if !p.Wrapper() { + if !p.Wrapper() && !p.ShouldIgnoreUnknownFields(message) { p.P(`m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...)`) } p.P(`iNdEx += skippy`) diff --git a/generator/generatedfile.go b/generator/generatedfile.go index 1d8b7410..07c3d920 100644 --- a/generator/generatedfile.go +++ b/generator/generatedfile.go @@ -41,6 +41,16 @@ func (b *GeneratedFile) ShouldPool(message *protogen.Message) bool { return false } +func (b *GeneratedFile) ShouldIgnoreUnknownFields(message *protogen.Message) bool { + if b.Config.IgnoreUnknownFields.Contains(message.GoIdent) { + return true + } + + ext := proto.GetExtension(message.Desc.Options(), vtproto.E_IgnoreUnknownFields) + ignoreUnknownFields, ok := ext.(bool) + return ok && ignoreUnknownFields +} + func (b *GeneratedFile) Alloc(vname string, message *protogen.Message, isQualifiedIdent bool) { ident := message.GoIdent.GoName if isQualifiedIdent { diff --git a/generator/generator.go b/generator/generator.go index cbd1600d..a575ed45 100644 --- a/generator/generator.go +++ b/generator/generator.go @@ -56,10 +56,12 @@ type Config struct { Poolable ObjectSet // PoolableExclude rules determines if pool feature disabled for particular message PoolableExclude ObjectSet - Wrap bool - WellKnownTypes bool - AllowEmpty bool - BuildTag string + // IgnoreUnknownFields contains messages for which unknown fields shall be ignored + IgnoreUnknownFields ObjectSet + Wrap bool + WellKnownTypes bool + AllowEmpty bool + BuildTag string } type Generator struct { diff --git a/include/github.com/planetscale/vtprotobuf/vtproto/ext.proto b/include/github.com/planetscale/vtprotobuf/vtproto/ext.proto index 6c1c628a..21d78109 100644 --- a/include/github.com/planetscale/vtprotobuf/vtproto/ext.proto +++ b/include/github.com/planetscale/vtprotobuf/vtproto/ext.proto @@ -9,6 +9,7 @@ option go_package = "github.com/planetscale/vtprotobuf/vtproto"; extend google.protobuf.MessageOptions { optional bool mempool = 64101; + optional bool ignore_unknown_fields = 64102; } extend google.protobuf.FieldOptions { @@ -19,4 +20,4 @@ extend google.protobuf.FieldOptions { // applying them to some of the fields in protobuf message Opts { optional bool unique = 1; -} \ No newline at end of file +} diff --git a/vtproto/ext.pb.go b/vtproto/ext.pb.go index 816fee1a..e296959e 100644 --- a/vtproto/ext.pb.go +++ b/vtproto/ext.pb.go @@ -79,6 +79,14 @@ var file_github_com_planetscale_vtprotobuf_vtproto_ext_proto_extTypes = []protoi Tag: "varint,64101,opt,name=mempool", Filename: "github.com/planetscale/vtprotobuf/vtproto/ext.proto", }, + { + ExtendedType: (*descriptorpb.MessageOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 64102, + Name: "vtproto.ignore_unknown_fields", + Tag: "varint,64102,opt,name=ignore_unknown_fields", + Filename: "github.com/planetscale/vtprotobuf/vtproto/ext.proto", + }, { ExtendedType: (*descriptorpb.FieldOptions)(nil), ExtensionType: (*Opts)(nil), @@ -93,12 +101,14 @@ var file_github_com_planetscale_vtprotobuf_vtproto_ext_proto_extTypes = []protoi var ( // optional bool mempool = 64101; E_Mempool = &file_github_com_planetscale_vtprotobuf_vtproto_ext_proto_extTypes[0] + // optional bool ignore_unknown_fields = 64102; + E_IgnoreUnknownFields = &file_github_com_planetscale_vtprotobuf_vtproto_ext_proto_extTypes[1] ) // Extension fields to descriptorpb.FieldOptions. var ( // optional vtproto.Opts options = 64150; - E_Options = &file_github_com_planetscale_vtprotobuf_vtproto_ext_proto_extTypes[1] + E_Options = &file_github_com_planetscale_vtprotobuf_vtproto_ext_proto_extTypes[2] ) var File_github_com_planetscale_vtprotobuf_vtproto_ext_proto protoreflect.FileDescriptor @@ -115,17 +125,22 @@ var file_github_com_planetscale_vtprotobuf_vtproto_ext_proto_rawDesc = []byte{ 0x3a, 0x3b, 0x0a, 0x07, 0x6d, 0x65, 0x6d, 0x70, 0x6f, 0x6f, 0x6c, 0x12, 0x1f, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0xe5, 0xf4, 0x03, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x6d, 0x65, 0x6d, 0x70, 0x6f, 0x6f, 0x6c, 0x3a, 0x48, 0x0a, - 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, - 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x96, 0xf5, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x0d, 0x2e, 0x76, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4f, 0x70, 0x74, 0x73, 0x52, 0x07, - 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x49, 0x0a, 0x13, 0x63, 0x6f, 0x6d, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x42, 0x07, - 0x56, 0x54, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x74, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x2f, - 0x76, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x76, 0x74, 0x70, 0x72, 0x6f, - 0x74, 0x6f, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x6d, 0x65, 0x6d, 0x70, 0x6f, 0x6f, 0x6c, 0x3a, 0x55, 0x0a, + 0x15, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x5f, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x1f, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0xe6, 0xf4, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x13, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x46, 0x69, + 0x65, 0x6c, 0x64, 0x73, 0x3a, 0x48, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, + 0x1d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x96, + 0xf5, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x76, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x4f, 0x70, 0x74, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x49, + 0x0a, 0x13, 0x63, 0x6f, 0x6d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x42, 0x07, 0x56, 0x54, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x29, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6c, 0x61, 0x6e, 0x65, + 0x74, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x2f, 0x76, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2f, 0x76, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, } var ( @@ -148,12 +163,13 @@ var file_github_com_planetscale_vtprotobuf_vtproto_ext_proto_goTypes = []interfa } var file_github_com_planetscale_vtprotobuf_vtproto_ext_proto_depIdxs = []int32{ 1, // 0: vtproto.mempool:extendee -> google.protobuf.MessageOptions - 2, // 1: vtproto.options:extendee -> google.protobuf.FieldOptions - 0, // 2: vtproto.options:type_name -> vtproto.Opts - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 2, // [2:3] is the sub-list for extension type_name - 0, // [0:2] is the sub-list for extension extendee + 1, // 1: vtproto.ignore_unknown_fields:extendee -> google.protobuf.MessageOptions + 2, // 2: vtproto.options:extendee -> google.protobuf.FieldOptions + 0, // 3: vtproto.options:type_name -> vtproto.Opts + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 3, // [3:4] is the sub-list for extension type_name + 0, // [0:3] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name } @@ -183,7 +199,7 @@ func file_github_com_planetscale_vtprotobuf_vtproto_ext_proto_init() { RawDescriptor: file_github_com_planetscale_vtprotobuf_vtproto_ext_proto_rawDesc, NumEnums: 0, NumMessages: 1, - NumExtensions: 2, + NumExtensions: 3, NumServices: 0, }, GoTypes: file_github_com_planetscale_vtprotobuf_vtproto_ext_proto_goTypes,