From e9a7bc02444ede1505b423440e7ef3b2468334a1 Mon Sep 17 00:00:00 2001 From: liron Date: Sun, 11 Jun 2017 12:28:15 +0300 Subject: [PATCH] Pluggable secret backend This commit extends SwarmKit secret management with pluggable secret backends support. The solution uses the existing docker plugin framework for loading plugins and the existing SwarmKit data backend for storing them. The approach is to add a new `driver` parameter to existing secrets, which defines whether the values are taken as is or fetched from one of the secret plugins. The loading of secrets is done using the standard docker plugin infrastructure, which is already accessible in SwarmKit and used in other flows (e.g., networking). The fetched values are evaluated before assigning them to worker nodes, so the payload is not stored in the raft store. Remarks: * I've added support for mocking the plugin subsystem when settings up the controlapi server. I preferred this approach over loading the full plugin subsystem in UT. Work still needed in this CR: - [ ] More unit tests (pending initial iteration) - [ ] Customized error handling (e.g., customize error string for Not Found) Work still needed to complete this feature: - [ ] Inject secrets as part of plugin initialization - [ ] CLI support in docker - [ ] Docs - [ ] Support scheduling plugins in swarm https://github.com/moby/moby/pull/33575 Signed-off-by: liron --- api/specs.pb.go | 295 +++++++++++++++----------- api/specs.proto | 3 + api/validation/secrets.go | 14 ++ cmd/swarmctl/secret/create.go | 11 +- cmd/swarmctl/secret/list.go | 10 +- manager/controlapi/secret.go | 18 +- manager/controlapi/secret_test.go | 10 + manager/controlapi/server.go | 1 - manager/dispatcher/assignments.go | 44 +++- manager/dispatcher/dispatcher.go | 7 +- manager/dispatcher/dispatcher_test.go | 161 +++++++++++++- manager/drivers/provider.go | 34 +++ manager/drivers/secrets.go | 55 +++++ manager/manager.go | 3 +- 14 files changed, 525 insertions(+), 141 deletions(-) create mode 100644 api/validation/secrets.go create mode 100644 manager/drivers/provider.go create mode 100644 manager/drivers/secrets.go diff --git a/api/specs.pb.go b/api/specs.pb.go index 8578cf3849..bb9b0db918 100644 --- a/api/specs.pb.go +++ b/api/specs.pb.go @@ -766,6 +766,8 @@ type SecretSpec struct { // The currently recognized values are: // - golang: Go templating Templating *Driver `protobuf:"bytes,3,opt,name=templating" json:"templating,omitempty"` + // Driver is the the secret driver that is used to store the specified secret + Driver *Driver `protobuf:"bytes,4,opt,name=driver" json:"driver,omitempty"` } func (m *SecretSpec) Reset() { *m = SecretSpec{} } @@ -1240,6 +1242,10 @@ func (m *SecretSpec) CopyFrom(src interface{}) { m.Templating = &Driver{} github_com_docker_swarmkit_api_deepcopy.Copy(m.Templating, o.Templating) } + if o.Driver != nil { + m.Driver = &Driver{} + github_com_docker_swarmkit_api_deepcopy.Copy(m.Driver, o.Driver) + } } func (m *ConfigSpec) Copy() *ConfigSpec { @@ -2257,6 +2263,16 @@ func (m *SecretSpec) MarshalTo(dAtA []byte) (int, error) { } i += n37 } + if m.Driver != nil { + dAtA[i] = 0x22 + i++ + i = encodeVarintSpecs(dAtA, i, uint64(m.Driver.Size())) + n38, err := m.Driver.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n38 + } return i, nil } @@ -2278,11 +2294,11 @@ func (m *ConfigSpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintSpecs(dAtA, i, uint64(m.Annotations.Size())) - n38, err := m.Annotations.MarshalTo(dAtA[i:]) + n39, err := m.Annotations.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n38 + i += n39 if len(m.Data) > 0 { dAtA[i] = 0x12 i++ @@ -2293,11 +2309,11 @@ func (m *ConfigSpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintSpecs(dAtA, i, uint64(m.Templating.Size())) - n39, err := m.Templating.MarshalTo(dAtA[i:]) + n40, err := m.Templating.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n39 + i += n40 } return i, nil } @@ -2729,6 +2745,10 @@ func (m *SecretSpec) Size() (n int) { l = m.Templating.Size() n += 1 + l + sovSpecs(uint64(l)) } + if m.Driver != nil { + l = m.Driver.Size() + n += 1 + l + sovSpecs(uint64(l)) + } return n } @@ -3022,6 +3042,7 @@ func (this *SecretSpec) String() string { `Annotations:` + strings.Replace(strings.Replace(this.Annotations.String(), "Annotations", "Annotations", 1), `&`, ``, 1) + `,`, `Data:` + fmt.Sprintf("%v", this.Data) + `,`, `Templating:` + strings.Replace(fmt.Sprintf("%v", this.Templating), "Driver", "Driver", 1) + `,`, + `Driver:` + strings.Replace(fmt.Sprintf("%v", this.Driver), "Driver", "Driver", 1) + `,`, `}`, }, "") return s @@ -5883,6 +5904,39 @@ func (m *SecretSpec) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Driver", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpecs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthSpecs + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Driver == nil { + m.Driver = &Driver{} + } + if err := m.Driver.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipSpecs(dAtA[iNdEx:]) @@ -6156,122 +6210,123 @@ var ( func init() { proto.RegisterFile("specs.proto", fileDescriptorSpecs) } var fileDescriptorSpecs = []byte{ - // 1867 bytes of a gzipped FileDescriptorProto + // 1880 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x57, 0xcf, 0x73, 0x1b, 0x49, - 0x15, 0xb6, 0x6c, 0x59, 0x3f, 0xde, 0xc8, 0x89, 0xd2, 0x24, 0x61, 0xa2, 0xb0, 0xb2, 0xa2, 0x0d, + 0x15, 0xb6, 0x6c, 0x59, 0x3f, 0xde, 0xc8, 0x89, 0xdc, 0x24, 0x61, 0xac, 0xb0, 0xb2, 0xa2, 0x0d, 0xc1, 0xcb, 0x16, 0x72, 0x61, 0xa8, 0x25, 0xbb, 0x61, 0x01, 0xc9, 0x12, 0x8e, 0x31, 0x76, 0x54, - 0x6d, 0x6f, 0x20, 0x27, 0x55, 0x7b, 0xa6, 0x3d, 0x9a, 0xf2, 0xa8, 0x7b, 0xe8, 0xe9, 0xd1, 0x96, - 0x6e, 0x1c, 0xb7, 0x72, 0xe5, 0xec, 0xe2, 0x40, 0xf1, 0xbf, 0xe4, 0x48, 0x71, 0xe2, 0xe4, 0x62, - 0xfd, 0x2f, 0x70, 0xe3, 0x02, 0xd5, 0x3d, 0x3d, 0xd2, 0x28, 0x19, 0x27, 0xa9, 0x22, 0x07, 0x6e, - 0xdd, 0xaf, 0xbf, 0xef, 0xcd, 0xeb, 0xd7, 0x5f, 0xf7, 0x7b, 0x03, 0x56, 0x14, 0x52, 0x27, 0xea, - 0x84, 0x82, 0x4b, 0x8e, 0x90, 0xcb, 0x9d, 0x73, 0x2a, 0x3a, 0xd1, 0xd7, 0x44, 0x4c, 0xce, 0x7d, - 0xd9, 0x99, 0xfe, 0xb8, 0x61, 0xc9, 0x59, 0x48, 0x0d, 0xa0, 0x71, 0xdb, 0xe3, 0x1e, 0xd7, 0xc3, - 0x6d, 0x35, 0x32, 0xd6, 0xa6, 0xc7, 0xb9, 0x17, 0xd0, 0x6d, 0x3d, 0x3b, 0x8d, 0xcf, 0xb6, 0xdd, - 0x58, 0x10, 0xe9, 0x73, 0x66, 0xd6, 0xef, 0xbd, 0xbe, 0x4e, 0xd8, 0x2c, 0x59, 0x6a, 0x5f, 0x14, - 0xa1, 0x72, 0xc4, 0x5d, 0x7a, 0x1c, 0x52, 0x07, 0xed, 0x81, 0x45, 0x18, 0xe3, 0x52, 0x73, 0x23, - 0xbb, 0xd0, 0x2a, 0x6c, 0x59, 0x3b, 0x9b, 0x9d, 0x37, 0x83, 0xea, 0x74, 0x17, 0xb0, 0x5e, 0xf1, - 0xd5, 0xe5, 0xe6, 0x0a, 0xce, 0x32, 0xd1, 0x2f, 0xa1, 0xe6, 0xd2, 0xc8, 0x17, 0xd4, 0x1d, 0x09, - 0x1e, 0x50, 0x7b, 0xb5, 0x55, 0xd8, 0xba, 0xb1, 0xf3, 0xbd, 0x3c, 0x4f, 0xea, 0xe3, 0x98, 0x07, - 0x14, 0x5b, 0x86, 0xa1, 0x26, 0x68, 0x0f, 0x60, 0x42, 0x27, 0xa7, 0x54, 0x44, 0x63, 0x3f, 0xb4, - 0xd7, 0x34, 0xfd, 0x07, 0xd7, 0xd1, 0x55, 0xec, 0x9d, 0xc3, 0x39, 0x1c, 0x67, 0xa8, 0xe8, 0x10, - 0x6a, 0x64, 0x4a, 0xfc, 0x80, 0x9c, 0xfa, 0x81, 0x2f, 0x67, 0x76, 0x51, 0xbb, 0xfa, 0xe4, 0xad, - 0xae, 0xba, 0x19, 0x02, 0x5e, 0xa2, 0xb7, 0x5d, 0x80, 0xc5, 0x87, 0xd0, 0x23, 0x28, 0x0f, 0x07, - 0x47, 0xfd, 0xfd, 0xa3, 0xbd, 0xfa, 0x4a, 0xe3, 0xde, 0xcb, 0x8b, 0xd6, 0x1d, 0xe5, 0x63, 0x01, - 0x18, 0x52, 0xe6, 0xfa, 0xcc, 0x43, 0x5b, 0x50, 0xe9, 0xee, 0xee, 0x0e, 0x86, 0x27, 0x83, 0x7e, - 0xbd, 0xd0, 0x68, 0xbc, 0xbc, 0x68, 0xdd, 0x5d, 0x06, 0x76, 0x1d, 0x87, 0x86, 0x92, 0xba, 0x8d, - 0xe2, 0x37, 0x7f, 0x69, 0xae, 0xb4, 0xbf, 0x29, 0x40, 0x2d, 0x1b, 0x04, 0x7a, 0x04, 0xa5, 0xee, - 0xee, 0xc9, 0xfe, 0xf3, 0x41, 0x7d, 0x65, 0x41, 0xcf, 0x22, 0xba, 0x8e, 0xf4, 0xa7, 0x14, 0x3d, - 0x84, 0xf5, 0x61, 0xf7, 0xab, 0xe3, 0x41, 0xbd, 0xb0, 0x08, 0x27, 0x0b, 0x1b, 0x92, 0x38, 0xd2, - 0xa8, 0x3e, 0xee, 0xee, 0x1f, 0xd5, 0x57, 0xf3, 0x51, 0x7d, 0x41, 0x7c, 0x66, 0x42, 0xf9, 0x73, - 0x11, 0xac, 0x63, 0x2a, 0xa6, 0xbe, 0xf3, 0x81, 0x25, 0xf2, 0x19, 0x14, 0x25, 0x89, 0xce, 0xb5, - 0x34, 0xac, 0x7c, 0x69, 0x9c, 0x90, 0xe8, 0x5c, 0x7d, 0xd4, 0xd0, 0x35, 0x5e, 0x29, 0x43, 0xd0, - 0x30, 0xf0, 0x1d, 0x22, 0xa9, 0xab, 0x95, 0x61, 0xed, 0x7c, 0x3f, 0x8f, 0x8d, 0xe7, 0x28, 0x13, - 0xff, 0xd3, 0x15, 0x9c, 0xa1, 0xa2, 0x27, 0x50, 0xf2, 0x02, 0x7e, 0x4a, 0x02, 0xad, 0x09, 0x6b, - 0xe7, 0x41, 0x9e, 0x93, 0x3d, 0x8d, 0x58, 0x38, 0x30, 0x14, 0xf4, 0x18, 0x4a, 0x71, 0xe8, 0x12, - 0x49, 0xed, 0x92, 0x26, 0xb7, 0xf2, 0xc8, 0x5f, 0x69, 0xc4, 0x2e, 0x67, 0x67, 0xbe, 0x87, 0x0d, - 0x1e, 0x1d, 0x40, 0x85, 0x51, 0xf9, 0x35, 0x17, 0xe7, 0x91, 0x5d, 0x6e, 0xad, 0x6d, 0x59, 0x3b, - 0x9f, 0xe6, 0x8a, 0x31, 0xc1, 0x74, 0xa5, 0x24, 0xce, 0x78, 0x42, 0x99, 0x4c, 0xdc, 0xf4, 0x56, - 0xed, 0x02, 0x9e, 0x3b, 0x40, 0x3f, 0x87, 0x0a, 0x65, 0x6e, 0xc8, 0x7d, 0x26, 0xed, 0xca, 0xf5, - 0x81, 0x0c, 0x0c, 0x46, 0x25, 0x13, 0xcf, 0x19, 0x8a, 0x2d, 0x78, 0x10, 0x9c, 0x12, 0xe7, 0xdc, - 0xae, 0xbe, 0xe7, 0x36, 0xe6, 0x8c, 0x5e, 0x09, 0x8a, 0x13, 0xee, 0xd2, 0xf6, 0x36, 0xdc, 0x7a, - 0x23, 0xd5, 0xa8, 0x01, 0x15, 0x93, 0xea, 0x44, 0x23, 0x45, 0x3c, 0x9f, 0xb7, 0x6f, 0xc2, 0xc6, - 0x52, 0x5a, 0xdb, 0x7f, 0x2f, 0x42, 0x25, 0x3d, 0x6b, 0xd4, 0x85, 0xaa, 0xc3, 0x99, 0x24, 0x3e, - 0xa3, 0xc2, 0xc8, 0x2b, 0xf7, 0x64, 0x76, 0x53, 0x90, 0x62, 0x3d, 0x5d, 0xc1, 0x0b, 0x16, 0xfa, - 0x35, 0x54, 0x05, 0x8d, 0x78, 0x2c, 0x1c, 0x1a, 0x19, 0x7d, 0x6d, 0xe5, 0x2b, 0x24, 0x01, 0x61, - 0xfa, 0x87, 0xd8, 0x17, 0x54, 0x65, 0x39, 0xc2, 0x0b, 0x2a, 0x7a, 0x02, 0x65, 0x41, 0x23, 0x49, - 0x84, 0x7c, 0x9b, 0x44, 0x70, 0x02, 0x19, 0xf2, 0xc0, 0x77, 0x66, 0x38, 0x65, 0xa0, 0x27, 0x50, - 0x0d, 0x03, 0xe2, 0x68, 0xaf, 0xf6, 0xba, 0xa6, 0x7f, 0x94, 0x47, 0x1f, 0xa6, 0x20, 0xbc, 0xc0, - 0xa3, 0xcf, 0x01, 0x02, 0xee, 0x8d, 0x5c, 0xe1, 0x4f, 0xa9, 0x30, 0x12, 0x6b, 0xe4, 0xb1, 0xfb, - 0x1a, 0x81, 0xab, 0x01, 0xf7, 0x92, 0x21, 0xda, 0xfb, 0x9f, 0xf4, 0x95, 0xd1, 0xd6, 0x01, 0x00, - 0x99, 0xaf, 0x1a, 0x75, 0x7d, 0xf2, 0x5e, 0xae, 0xcc, 0x89, 0x64, 0xe8, 0xe8, 0x01, 0xd4, 0xce, - 0xb8, 0x70, 0xe8, 0xc8, 0xdc, 0x9a, 0xaa, 0xd6, 0x84, 0xa5, 0x6d, 0x89, 0xbe, 0x50, 0x0f, 0xca, - 0x1e, 0x65, 0x54, 0xf8, 0x8e, 0x0d, 0xfa, 0x63, 0x8f, 0x72, 0x2f, 0x64, 0x02, 0xc1, 0x31, 0x93, - 0xfe, 0x84, 0x9a, 0x2f, 0xa5, 0xc4, 0x5e, 0x15, 0xca, 0x22, 0x59, 0x69, 0xff, 0x1e, 0xd0, 0x9b, - 0x58, 0x84, 0xa0, 0x78, 0xee, 0x33, 0x57, 0x0b, 0xab, 0x8a, 0xf5, 0x18, 0x75, 0xa0, 0x1c, 0x92, - 0x59, 0xc0, 0x89, 0x6b, 0xc4, 0x72, 0xbb, 0x93, 0xd4, 0xcb, 0x4e, 0x5a, 0x2f, 0x3b, 0x5d, 0x36, - 0xc3, 0x29, 0xa8, 0x7d, 0x00, 0x77, 0x72, 0xb7, 0x8c, 0x76, 0xa0, 0x36, 0x17, 0xe1, 0xc8, 0x37, - 0x1f, 0xe9, 0xdd, 0xbc, 0xba, 0xdc, 0xb4, 0xe6, 0x6a, 0xdd, 0xef, 0x63, 0x6b, 0x0e, 0xda, 0x77, - 0xdb, 0x7f, 0xaa, 0xc2, 0xc6, 0x92, 0x94, 0xd1, 0x6d, 0x58, 0xf7, 0x27, 0xc4, 0xa3, 0x26, 0xc6, - 0x64, 0x82, 0x06, 0x50, 0x0a, 0xc8, 0x29, 0x0d, 0x94, 0xa0, 0xd5, 0xa1, 0xfe, 0xe8, 0x9d, 0x77, - 0xa2, 0xf3, 0x5b, 0x8d, 0x1f, 0x30, 0x29, 0x66, 0xd8, 0x90, 0x91, 0x0d, 0x65, 0x87, 0x4f, 0x26, - 0x84, 0xa9, 0xa7, 0x73, 0x6d, 0xab, 0x8a, 0xd3, 0xa9, 0xca, 0x0c, 0x11, 0x5e, 0x64, 0x17, 0xb5, - 0x59, 0x8f, 0x51, 0x1d, 0xd6, 0x28, 0x9b, 0xda, 0xeb, 0xda, 0xa4, 0x86, 0xca, 0xe2, 0xfa, 0x89, - 0x22, 0xab, 0x58, 0x0d, 0x15, 0x2f, 0x8e, 0xa8, 0xb0, 0xcb, 0x49, 0x46, 0xd5, 0x18, 0xfd, 0x0c, - 0x4a, 0x13, 0x1e, 0x33, 0x19, 0xd9, 0x15, 0x1d, 0xec, 0xbd, 0xbc, 0x60, 0x0f, 0x15, 0xc2, 0x3c, - 0xed, 0x06, 0x8e, 0x06, 0x70, 0x2b, 0x92, 0x3c, 0x1c, 0x79, 0x82, 0x38, 0x74, 0x14, 0x52, 0xe1, - 0x73, 0xd7, 0x3c, 0x4d, 0xf7, 0xde, 0x38, 0x94, 0xbe, 0x69, 0x72, 0xf0, 0x4d, 0xc5, 0xd9, 0x53, - 0x94, 0xa1, 0x66, 0xa0, 0x21, 0xd4, 0xc2, 0x38, 0x08, 0x46, 0x3c, 0x4c, 0xaa, 0x54, 0xa2, 0xa7, - 0xf7, 0x48, 0xd9, 0x30, 0x0e, 0x82, 0x67, 0x09, 0x09, 0x5b, 0xe1, 0x62, 0x82, 0xee, 0x42, 0xc9, - 0x13, 0x3c, 0x0e, 0x23, 0xdb, 0xd2, 0xc9, 0x30, 0x33, 0xf4, 0x25, 0x94, 0x23, 0xea, 0x08, 0x2a, - 0x23, 0xbb, 0xa6, 0xb7, 0xfa, 0x71, 0xde, 0x47, 0x8e, 0x35, 0x04, 0xd3, 0x33, 0x2a, 0x28, 0x73, - 0x28, 0x4e, 0x39, 0xe8, 0x1e, 0xac, 0x49, 0x39, 0xb3, 0x37, 0x5a, 0x85, 0xad, 0x4a, 0xaf, 0x7c, - 0x75, 0xb9, 0xb9, 0x76, 0x72, 0xf2, 0x02, 0x2b, 0x9b, 0x7a, 0x41, 0xc7, 0x3c, 0x92, 0x8c, 0x4c, - 0xa8, 0x7d, 0x43, 0xe7, 0x76, 0x3e, 0x47, 0x2f, 0x00, 0x5c, 0x16, 0x8d, 0x1c, 0x7d, 0x65, 0xed, - 0x9b, 0x7a, 0x77, 0x9f, 0xbe, 0x7b, 0x77, 0xfd, 0xa3, 0x63, 0x53, 0x45, 0x36, 0xae, 0x2e, 0x37, - 0xab, 0xf3, 0x29, 0xae, 0xba, 0x2c, 0x4a, 0x86, 0xa8, 0x07, 0xd6, 0x98, 0x92, 0x40, 0x8e, 0x9d, - 0x31, 0x75, 0xce, 0xed, 0xfa, 0xf5, 0x65, 0xe1, 0xa9, 0x86, 0x19, 0x0f, 0x59, 0x92, 0x52, 0xb0, - 0x0a, 0x35, 0xb2, 0x6f, 0xe9, 0x5c, 0x25, 0x13, 0xf4, 0x11, 0x00, 0x0f, 0x29, 0x1b, 0x45, 0xd2, - 0xf5, 0x99, 0x8d, 0xd4, 0x96, 0x71, 0x55, 0x59, 0x8e, 0x95, 0x01, 0xdd, 0x57, 0x8f, 0x36, 0x71, - 0x47, 0x9c, 0x05, 0x33, 0xfb, 0x3b, 0x7a, 0xb5, 0xa2, 0x0c, 0xcf, 0x58, 0x30, 0x43, 0x9b, 0x60, - 0x69, 0x5d, 0x44, 0xbe, 0xc7, 0x48, 0x60, 0xdf, 0xd6, 0xf9, 0x00, 0x65, 0x3a, 0xd6, 0x16, 0x75, - 0x0e, 0x49, 0x36, 0x22, 0xfb, 0xce, 0xf5, 0xe7, 0x60, 0x82, 0x5d, 0x9c, 0x83, 0xe1, 0xa0, 0x5f, - 0x00, 0x84, 0xc2, 0x9f, 0xfa, 0x01, 0xf5, 0x68, 0x64, 0xdf, 0xd5, 0x9b, 0x6e, 0xe6, 0xbe, 0xd6, - 0x73, 0x14, 0xce, 0x30, 0x1a, 0x9f, 0x83, 0x95, 0xb9, 0x6d, 0xea, 0x96, 0x9c, 0xd3, 0x99, 0xb9, - 0xc0, 0x6a, 0xa8, 0x52, 0x32, 0x25, 0x41, 0x9c, 0x74, 0xc2, 0x55, 0x9c, 0x4c, 0xbe, 0x58, 0x7d, - 0x5c, 0x68, 0xec, 0x80, 0x95, 0x51, 0x1d, 0xfa, 0x18, 0x36, 0x04, 0xf5, 0xfc, 0x48, 0x8a, 0xd9, - 0x88, 0xc4, 0x72, 0x6c, 0xff, 0x4a, 0x13, 0x6a, 0xa9, 0xb1, 0x1b, 0xcb, 0x71, 0x63, 0x04, 0x8b, - 0xc3, 0x43, 0x2d, 0xb0, 0x94, 0x28, 0x22, 0x2a, 0xa6, 0x54, 0xa8, 0x6a, 0xab, 0x72, 0x9e, 0x35, - 0x29, 0xf1, 0x46, 0x94, 0x08, 0x67, 0xac, 0xdf, 0x8e, 0x2a, 0x36, 0x33, 0xf5, 0x18, 0xa4, 0x37, - 0xc4, 0x3c, 0x06, 0x66, 0xda, 0xfe, 0x57, 0x01, 0x6a, 0xd9, 0xa6, 0x01, 0xed, 0x26, 0xc5, 0x5e, - 0x6f, 0xe9, 0xc6, 0xce, 0xf6, 0xbb, 0x9a, 0x0c, 0x5d, 0x5a, 0x83, 0x58, 0x39, 0x3b, 0x54, 0xfd, - 0xbd, 0x26, 0xa3, 0x9f, 0xc2, 0x7a, 0xc8, 0x85, 0x4c, 0x9f, 0xb0, 0xfc, 0x04, 0x73, 0x91, 0x96, - 0xa2, 0x04, 0xdc, 0x1e, 0xc3, 0x8d, 0x65, 0x6f, 0xe8, 0x21, 0xac, 0x3d, 0xdf, 0x1f, 0xd6, 0x57, - 0x1a, 0xf7, 0x5f, 0x5e, 0xb4, 0xbe, 0xbb, 0xbc, 0xf8, 0xdc, 0x17, 0x32, 0x26, 0xc1, 0xfe, 0x10, - 0xfd, 0x10, 0xd6, 0xfb, 0x47, 0xc7, 0x18, 0xd7, 0x0b, 0x8d, 0xcd, 0x97, 0x17, 0xad, 0xfb, 0xcb, - 0x38, 0xb5, 0xc4, 0x63, 0xe6, 0x62, 0x7e, 0x3a, 0xef, 0x75, 0xff, 0xbd, 0x0a, 0x96, 0x79, 0xd9, - 0x3f, 0xf4, 0xef, 0xd0, 0x46, 0x52, 0xca, 0xd3, 0x2b, 0xbb, 0xfa, 0xce, 0x8a, 0x5e, 0x4b, 0x08, - 0xe6, 0x8c, 0x1f, 0x40, 0xcd, 0x0f, 0xa7, 0x9f, 0x8d, 0x28, 0x23, 0xa7, 0x81, 0x69, 0x7b, 0x2b, - 0xd8, 0x52, 0xb6, 0x41, 0x62, 0x52, 0xef, 0x85, 0xcf, 0x24, 0x15, 0xcc, 0x34, 0xb4, 0x15, 0x3c, - 0x9f, 0xa3, 0x2f, 0xa1, 0xe8, 0x87, 0x64, 0x62, 0xda, 0x90, 0xdc, 0x1d, 0xec, 0x0f, 0xbb, 0x87, - 0x46, 0x83, 0xbd, 0xca, 0xd5, 0xe5, 0x66, 0x51, 0x19, 0xb0, 0xa6, 0xa1, 0x66, 0xda, 0x09, 0xa8, - 0x2f, 0xe9, 0xb7, 0xbf, 0x82, 0x33, 0x16, 0xa5, 0x23, 0x9f, 0x79, 0x82, 0x46, 0x91, 0xae, 0x02, - 0x15, 0x9c, 0x4e, 0x51, 0x03, 0xca, 0xa6, 0x9f, 0xd0, 0x0d, 0x44, 0x55, 0xd5, 0x6a, 0x63, 0xe8, - 0x6d, 0x80, 0x95, 0x64, 0x63, 0x74, 0x26, 0xf8, 0xa4, 0xfd, 0x9f, 0x22, 0x58, 0xbb, 0x41, 0x1c, - 0x49, 0x53, 0x06, 0x3f, 0x58, 0xf2, 0x5f, 0xc0, 0x2d, 0xa2, 0x7f, 0xaf, 0x08, 0x53, 0x35, 0x45, - 0xb7, 0x69, 0xe6, 0x00, 0x1e, 0xe6, 0xba, 0x9b, 0x83, 0x93, 0x96, 0xae, 0x57, 0x52, 0x3e, 0xed, - 0x02, 0xae, 0x93, 0xd7, 0x56, 0xd0, 0x31, 0x6c, 0x70, 0xe1, 0x8c, 0x69, 0x24, 0x93, 0x4a, 0x64, - 0x7e, 0x47, 0x72, 0x7f, 0x54, 0x9f, 0x65, 0x81, 0xe6, 0x19, 0x4e, 0xa2, 0x5d, 0xf6, 0x81, 0x1e, - 0x43, 0x51, 0x90, 0xb3, 0xb4, 0xe5, 0xcc, 0xbd, 0x24, 0x98, 0x9c, 0xc9, 0x25, 0x17, 0x9a, 0x81, - 0x7e, 0x03, 0xe0, 0xfa, 0x51, 0x48, 0xa4, 0x33, 0xa6, 0xc2, 0x1c, 0x76, 0xee, 0x16, 0xfb, 0x73, - 0xd4, 0x92, 0x97, 0x0c, 0x1b, 0x1d, 0x40, 0xd5, 0x21, 0xa9, 0x5c, 0x4b, 0xd7, 0xff, 0xa3, 0xed, - 0x76, 0x8d, 0x8b, 0xba, 0x72, 0x71, 0x75, 0xb9, 0x59, 0x49, 0x2d, 0xb8, 0xe2, 0x10, 0x23, 0xdf, - 0x03, 0xd8, 0x50, 0xff, 0x6e, 0x23, 0x97, 0x9e, 0x91, 0x38, 0x90, 0x89, 0x4c, 0xae, 0x29, 0x2b, - 0xea, 0x47, 0xa0, 0x6f, 0x70, 0x26, 0xae, 0x9a, 0xcc, 0xd8, 0xd0, 0xef, 0xe0, 0x16, 0x65, 0x8e, - 0x98, 0x69, 0xb1, 0xa6, 0x11, 0x56, 0xae, 0xdf, 0xec, 0x60, 0x0e, 0x5e, 0xda, 0x6c, 0x9d, 0xbe, - 0x66, 0x6f, 0xff, 0xb5, 0x00, 0x90, 0x54, 0xea, 0x0f, 0x2b, 0x40, 0x04, 0x45, 0x97, 0x48, 0xa2, - 0x35, 0x57, 0xc3, 0x7a, 0x8c, 0xbe, 0x00, 0x90, 0x74, 0x12, 0x06, 0x44, 0xfa, 0xcc, 0x33, 0xb2, - 0x79, 0xdb, 0x73, 0x90, 0x41, 0xeb, 0x38, 0x93, 0x90, 0xff, 0xaf, 0xe3, 0xec, 0xd9, 0xaf, 0xbe, - 0x6d, 0xae, 0xfc, 0xe3, 0xdb, 0xe6, 0xca, 0x1f, 0xaf, 0x9a, 0x85, 0x57, 0x57, 0xcd, 0xc2, 0xdf, - 0xae, 0x9a, 0x85, 0x7f, 0x5e, 0x35, 0x0b, 0xa7, 0x25, 0xdd, 0xc3, 0xfd, 0xe4, 0xbf, 0x01, 0x00, - 0x00, 0xff, 0xff, 0x06, 0x93, 0x6e, 0xba, 0xfc, 0x12, 0x00, 0x00, + 0x6d, 0x6f, 0x20, 0x27, 0x55, 0x7b, 0xa6, 0x2d, 0x4d, 0x79, 0xd4, 0x3d, 0xf4, 0xf4, 0x68, 0x4b, + 0x37, 0x8e, 0x5b, 0xb9, 0x72, 0x76, 0x71, 0xa0, 0xf8, 0x5f, 0x72, 0xa4, 0x38, 0xc1, 0xc5, 0x45, + 0xfc, 0x2f, 0x70, 0xe3, 0x02, 0xd5, 0x3d, 0x3d, 0xd2, 0x28, 0x19, 0xc7, 0xa9, 0x22, 0x87, 0xbd, + 0x75, 0xbf, 0xfe, 0xbe, 0xd7, 0xdd, 0xaf, 0xbf, 0xee, 0xf7, 0x1a, 0xac, 0x30, 0xa0, 0x4e, 0xd8, + 0x0a, 0x04, 0x97, 0x1c, 0x21, 0x97, 0x3b, 0xe7, 0x54, 0xb4, 0xc2, 0xaf, 0x89, 0x18, 0x9f, 0x7b, + 0xb2, 0x35, 0xf9, 0x71, 0xcd, 0x92, 0xd3, 0x80, 0x1a, 0x40, 0xed, 0xce, 0x90, 0x0f, 0xb9, 0x6e, + 0x6e, 0xab, 0x96, 0xb1, 0xd6, 0x87, 0x9c, 0x0f, 0x7d, 0xba, 0xad, 0x7b, 0xa7, 0xd1, 0xd9, 0xb6, + 0x1b, 0x09, 0x22, 0x3d, 0xce, 0xcc, 0xf8, 0xc6, 0x9b, 0xe3, 0x84, 0x4d, 0xe3, 0xa1, 0xe6, 0x45, + 0x1e, 0x4a, 0x47, 0xdc, 0xa5, 0xc7, 0x01, 0x75, 0xd0, 0x1e, 0x58, 0x84, 0x31, 0x2e, 0x35, 0x37, + 0xb4, 0x73, 0x8d, 0xdc, 0x96, 0xb5, 0xb3, 0xd9, 0x7a, 0x7b, 0x51, 0xad, 0xf6, 0x1c, 0xd6, 0xc9, + 0xbf, 0xba, 0xdc, 0x5c, 0xc2, 0x69, 0x26, 0xfa, 0x25, 0x54, 0x5c, 0x1a, 0x7a, 0x82, 0xba, 0x03, + 0xc1, 0x7d, 0x6a, 0x2f, 0x37, 0x72, 0x5b, 0xb7, 0x76, 0xbe, 0x97, 0xe5, 0x49, 0x4d, 0x8e, 0xb9, + 0x4f, 0xb1, 0x65, 0x18, 0xaa, 0x83, 0xf6, 0x00, 0xc6, 0x74, 0x7c, 0x4a, 0x45, 0x38, 0xf2, 0x02, + 0x7b, 0x45, 0xd3, 0x7f, 0x70, 0x1d, 0x5d, 0xad, 0xbd, 0x75, 0x38, 0x83, 0xe3, 0x14, 0x15, 0x1d, + 0x42, 0x85, 0x4c, 0x88, 0xe7, 0x93, 0x53, 0xcf, 0xf7, 0xe4, 0xd4, 0xce, 0x6b, 0x57, 0x9f, 0xbc, + 0xd3, 0x55, 0x3b, 0x45, 0xc0, 0x0b, 0xf4, 0xa6, 0x0b, 0x30, 0x9f, 0x08, 0x3d, 0x82, 0x62, 0xbf, + 0x77, 0xd4, 0xdd, 0x3f, 0xda, 0xab, 0x2e, 0xd5, 0x36, 0x5e, 0x5e, 0x34, 0xee, 0x2a, 0x1f, 0x73, + 0x40, 0x9f, 0x32, 0xd7, 0x63, 0x43, 0xb4, 0x05, 0xa5, 0xf6, 0xee, 0x6e, 0xaf, 0x7f, 0xd2, 0xeb, + 0x56, 0x73, 0xb5, 0xda, 0xcb, 0x8b, 0xc6, 0xbd, 0x45, 0x60, 0xdb, 0x71, 0x68, 0x20, 0xa9, 0x5b, + 0xcb, 0x7f, 0xf3, 0x97, 0xfa, 0x52, 0xf3, 0x9b, 0x1c, 0x54, 0xd2, 0x8b, 0x40, 0x8f, 0xa0, 0xd0, + 0xde, 0x3d, 0xd9, 0x7f, 0xde, 0xab, 0x2e, 0xcd, 0xe9, 0x69, 0x44, 0xdb, 0x91, 0xde, 0x84, 0xa2, + 0x87, 0xb0, 0xda, 0x6f, 0x7f, 0x75, 0xdc, 0xab, 0xe6, 0xe6, 0xcb, 0x49, 0xc3, 0xfa, 0x24, 0x0a, + 0x35, 0xaa, 0x8b, 0xdb, 0xfb, 0x47, 0xd5, 0xe5, 0x6c, 0x54, 0x57, 0x10, 0x8f, 0x99, 0xa5, 0xfc, + 0x39, 0x0f, 0xd6, 0x31, 0x15, 0x13, 0xcf, 0xf9, 0xc0, 0x12, 0xf9, 0x0c, 0xf2, 0x92, 0x84, 0xe7, + 0x5a, 0x1a, 0x56, 0xb6, 0x34, 0x4e, 0x48, 0x78, 0xae, 0x26, 0x35, 0x74, 0x8d, 0x57, 0xca, 0x10, + 0x34, 0xf0, 0x3d, 0x87, 0x48, 0xea, 0x6a, 0x65, 0x58, 0x3b, 0xdf, 0xcf, 0x62, 0xe3, 0x19, 0xca, + 0xac, 0xff, 0xe9, 0x12, 0x4e, 0x51, 0xd1, 0x13, 0x28, 0x0c, 0x7d, 0x7e, 0x4a, 0x7c, 0xad, 0x09, + 0x6b, 0xe7, 0x41, 0x96, 0x93, 0x3d, 0x8d, 0x98, 0x3b, 0x30, 0x14, 0xf4, 0x18, 0x0a, 0x51, 0xe0, + 0x12, 0x49, 0xed, 0x82, 0x26, 0x37, 0xb2, 0xc8, 0x5f, 0x69, 0xc4, 0x2e, 0x67, 0x67, 0xde, 0x10, + 0x1b, 0x3c, 0x3a, 0x80, 0x12, 0xa3, 0xf2, 0x6b, 0x2e, 0xce, 0x43, 0xbb, 0xd8, 0x58, 0xd9, 0xb2, + 0x76, 0x3e, 0xcd, 0x14, 0x63, 0x8c, 0x69, 0x4b, 0x49, 0x9c, 0xd1, 0x98, 0x32, 0x19, 0xbb, 0xe9, + 0x2c, 0xdb, 0x39, 0x3c, 0x73, 0x80, 0x7e, 0x0e, 0x25, 0xca, 0xdc, 0x80, 0x7b, 0x4c, 0xda, 0xa5, + 0xeb, 0x17, 0xd2, 0x33, 0x18, 0x15, 0x4c, 0x3c, 0x63, 0x28, 0xb6, 0xe0, 0xbe, 0x7f, 0x4a, 0x9c, + 0x73, 0xbb, 0xfc, 0x9e, 0xdb, 0x98, 0x31, 0x3a, 0x05, 0xc8, 0x8f, 0xb9, 0x4b, 0x9b, 0xdb, 0xb0, + 0xfe, 0x56, 0xa8, 0x51, 0x0d, 0x4a, 0x26, 0xd4, 0xb1, 0x46, 0xf2, 0x78, 0xd6, 0x6f, 0xde, 0x86, + 0xb5, 0x85, 0xb0, 0x36, 0xff, 0x9e, 0x87, 0x52, 0x72, 0xd6, 0xa8, 0x0d, 0x65, 0x87, 0x33, 0x49, + 0x3c, 0x46, 0x85, 0x91, 0x57, 0xe6, 0xc9, 0xec, 0x26, 0x20, 0xc5, 0x7a, 0xba, 0x84, 0xe7, 0x2c, + 0xf4, 0x6b, 0x28, 0x0b, 0x1a, 0xf2, 0x48, 0x38, 0x34, 0x34, 0xfa, 0xda, 0xca, 0x56, 0x48, 0x0c, + 0xc2, 0xf4, 0x0f, 0x91, 0x27, 0xa8, 0x8a, 0x72, 0x88, 0xe7, 0x54, 0xf4, 0x04, 0x8a, 0x82, 0x86, + 0x92, 0x08, 0xf9, 0x2e, 0x89, 0xe0, 0x18, 0xd2, 0xe7, 0xbe, 0xe7, 0x4c, 0x71, 0xc2, 0x40, 0x4f, + 0xa0, 0x1c, 0xf8, 0xc4, 0xd1, 0x5e, 0xed, 0x55, 0x4d, 0xff, 0x28, 0x8b, 0xde, 0x4f, 0x40, 0x78, + 0x8e, 0x47, 0x9f, 0x03, 0xf8, 0x7c, 0x38, 0x70, 0x85, 0x37, 0xa1, 0xc2, 0x48, 0xac, 0x96, 0xc5, + 0xee, 0x6a, 0x04, 0x2e, 0xfb, 0x7c, 0x18, 0x37, 0xd1, 0xde, 0xff, 0xa5, 0xaf, 0x94, 0xb6, 0x0e, + 0x00, 0xc8, 0x6c, 0xd4, 0xa8, 0xeb, 0x93, 0xf7, 0x72, 0x65, 0x4e, 0x24, 0x45, 0x47, 0x0f, 0xa0, + 0x72, 0xc6, 0x85, 0x43, 0x07, 0xe6, 0xd6, 0x94, 0xb5, 0x26, 0x2c, 0x6d, 0x8b, 0xf5, 0x85, 0x3a, + 0x50, 0x1c, 0x52, 0x46, 0x85, 0xe7, 0xd8, 0xa0, 0x27, 0x7b, 0x94, 0x79, 0x21, 0x63, 0x08, 0x8e, + 0x98, 0xf4, 0xc6, 0xd4, 0xcc, 0x94, 0x10, 0x3b, 0x65, 0x28, 0x8a, 0x78, 0xa4, 0xf9, 0x7b, 0x40, + 0x6f, 0x63, 0x11, 0x82, 0xfc, 0xb9, 0xc7, 0x5c, 0x2d, 0xac, 0x32, 0xd6, 0x6d, 0xd4, 0x82, 0x62, + 0x40, 0xa6, 0x3e, 0x27, 0xae, 0x11, 0xcb, 0x9d, 0x56, 0x9c, 0x2f, 0x5b, 0x49, 0xbe, 0x6c, 0xb5, + 0xd9, 0x14, 0x27, 0xa0, 0xe6, 0x01, 0xdc, 0xcd, 0xdc, 0x32, 0xda, 0x81, 0xca, 0x4c, 0x84, 0x03, + 0xcf, 0x4c, 0xd2, 0xb9, 0x7d, 0x75, 0xb9, 0x69, 0xcd, 0xd4, 0xba, 0xdf, 0xc5, 0xd6, 0x0c, 0xb4, + 0xef, 0x36, 0xff, 0x54, 0x86, 0xb5, 0x05, 0x29, 0xa3, 0x3b, 0xb0, 0xea, 0x8d, 0xc9, 0x90, 0x9a, + 0x35, 0xc6, 0x1d, 0xd4, 0x83, 0x82, 0x4f, 0x4e, 0xa9, 0xaf, 0x04, 0xad, 0x0e, 0xf5, 0x47, 0x37, + 0xde, 0x89, 0xd6, 0x6f, 0x35, 0xbe, 0xc7, 0xa4, 0x98, 0x62, 0x43, 0x46, 0x36, 0x14, 0x1d, 0x3e, + 0x1e, 0x13, 0xa6, 0x9e, 0xce, 0x95, 0xad, 0x32, 0x4e, 0xba, 0x2a, 0x32, 0x44, 0x0c, 0x43, 0x3b, + 0xaf, 0xcd, 0xba, 0x8d, 0xaa, 0xb0, 0x42, 0xd9, 0xc4, 0x5e, 0xd5, 0x26, 0xd5, 0x54, 0x16, 0xd7, + 0x8b, 0x15, 0x59, 0xc6, 0xaa, 0xa9, 0x78, 0x51, 0x48, 0x85, 0x5d, 0x8c, 0x23, 0xaa, 0xda, 0xe8, + 0x67, 0x50, 0x18, 0xf3, 0x88, 0xc9, 0xd0, 0x2e, 0xe9, 0xc5, 0x6e, 0x64, 0x2d, 0xf6, 0x50, 0x21, + 0xcc, 0xd3, 0x6e, 0xe0, 0xa8, 0x07, 0xeb, 0xa1, 0xe4, 0xc1, 0x60, 0x28, 0x88, 0x43, 0x07, 0x01, + 0x15, 0x1e, 0x77, 0xcd, 0xd3, 0xb4, 0xf1, 0xd6, 0xa1, 0x74, 0x4d, 0x91, 0x83, 0x6f, 0x2b, 0xce, + 0x9e, 0xa2, 0xf4, 0x35, 0x03, 0xf5, 0xa1, 0x12, 0x44, 0xbe, 0x3f, 0xe0, 0x41, 0x9c, 0xa5, 0x62, + 0x3d, 0xbd, 0x47, 0xc8, 0xfa, 0x91, 0xef, 0x3f, 0x8b, 0x49, 0xd8, 0x0a, 0xe6, 0x1d, 0x74, 0x0f, + 0x0a, 0x43, 0xc1, 0xa3, 0x20, 0xb4, 0x2d, 0x1d, 0x0c, 0xd3, 0x43, 0x5f, 0x42, 0x31, 0xa4, 0x8e, + 0xa0, 0x32, 0xb4, 0x2b, 0x7a, 0xab, 0x1f, 0x67, 0x4d, 0x72, 0xac, 0x21, 0x98, 0x9e, 0x51, 0x41, + 0x99, 0x43, 0x71, 0xc2, 0x41, 0x1b, 0xb0, 0x22, 0xe5, 0xd4, 0x5e, 0x6b, 0xe4, 0xb6, 0x4a, 0x9d, + 0xe2, 0xd5, 0xe5, 0xe6, 0xca, 0xc9, 0xc9, 0x0b, 0xac, 0x6c, 0xea, 0x05, 0x1d, 0xf1, 0x50, 0x32, + 0x32, 0xa6, 0xf6, 0x2d, 0x1d, 0xdb, 0x59, 0x1f, 0xbd, 0x00, 0x70, 0x59, 0x38, 0x70, 0xf4, 0x95, + 0xb5, 0x6f, 0xeb, 0xdd, 0x7d, 0x7a, 0xf3, 0xee, 0xba, 0x47, 0xc7, 0x26, 0x8b, 0xac, 0x5d, 0x5d, + 0x6e, 0x96, 0x67, 0x5d, 0x5c, 0x76, 0x59, 0x18, 0x37, 0x51, 0x07, 0xac, 0x11, 0x25, 0xbe, 0x1c, + 0x39, 0x23, 0xea, 0x9c, 0xdb, 0xd5, 0xeb, 0xd3, 0xc2, 0x53, 0x0d, 0x33, 0x1e, 0xd2, 0x24, 0xa5, + 0x60, 0xb5, 0xd4, 0xd0, 0x5e, 0xd7, 0xb1, 0x8a, 0x3b, 0xe8, 0x23, 0x00, 0x1e, 0x50, 0x36, 0x08, + 0xa5, 0xeb, 0x31, 0x1b, 0xa9, 0x2d, 0xe3, 0xb2, 0xb2, 0x1c, 0x2b, 0x03, 0xba, 0xaf, 0x1e, 0x6d, + 0xe2, 0x0e, 0x38, 0xf3, 0xa7, 0xf6, 0x77, 0xf4, 0x68, 0x49, 0x19, 0x9e, 0x31, 0x7f, 0x8a, 0x36, + 0xc1, 0xd2, 0xba, 0x08, 0xbd, 0x21, 0x23, 0xbe, 0x7d, 0x47, 0xc7, 0x03, 0x94, 0xe9, 0x58, 0x5b, + 0xd4, 0x39, 0xc4, 0xd1, 0x08, 0xed, 0xbb, 0xd7, 0x9f, 0x83, 0x59, 0xec, 0xfc, 0x1c, 0x0c, 0x07, + 0xfd, 0x02, 0x20, 0x10, 0xde, 0xc4, 0xf3, 0xe9, 0x90, 0x86, 0xf6, 0x3d, 0xbd, 0xe9, 0x7a, 0xe6, + 0x6b, 0x3d, 0x43, 0xe1, 0x14, 0xa3, 0xf6, 0x39, 0x58, 0xa9, 0xdb, 0xa6, 0x6e, 0xc9, 0x39, 0x9d, + 0x9a, 0x0b, 0xac, 0x9a, 0x2a, 0x24, 0x13, 0xe2, 0x47, 0x71, 0x25, 0x5c, 0xc6, 0x71, 0xe7, 0x8b, + 0xe5, 0xc7, 0xb9, 0xda, 0x0e, 0x58, 0x29, 0xd5, 0xa1, 0x8f, 0x61, 0x4d, 0xd0, 0xa1, 0x17, 0x4a, + 0x31, 0x1d, 0x90, 0x48, 0x8e, 0xec, 0x5f, 0x69, 0x42, 0x25, 0x31, 0xb6, 0x23, 0x39, 0xaa, 0x0d, + 0x60, 0x7e, 0x78, 0xa8, 0x01, 0x96, 0x12, 0x45, 0x48, 0xc5, 0x84, 0x0a, 0x95, 0x6d, 0x55, 0xcc, + 0xd3, 0x26, 0x25, 0xde, 0x90, 0x12, 0xe1, 0x8c, 0xf4, 0xdb, 0x51, 0xc6, 0xa6, 0xa7, 0x1e, 0x83, + 0xe4, 0x86, 0x98, 0xc7, 0xc0, 0x74, 0x9b, 0xff, 0xce, 0x41, 0x25, 0x5d, 0x34, 0xa0, 0xdd, 0x38, + 0xd9, 0xeb, 0x2d, 0xdd, 0xda, 0xd9, 0xbe, 0xa9, 0xc8, 0xd0, 0xa9, 0xd5, 0x8f, 0x94, 0xb3, 0x43, + 0x55, 0xdf, 0x6b, 0x32, 0xfa, 0x29, 0xac, 0x06, 0x5c, 0xc8, 0xe4, 0x09, 0xcb, 0x0e, 0x30, 0x17, + 0x49, 0x2a, 0x8a, 0xc1, 0xcd, 0x11, 0xdc, 0x5a, 0xf4, 0x86, 0x1e, 0xc2, 0xca, 0xf3, 0xfd, 0x7e, + 0x75, 0xa9, 0x76, 0xff, 0xe5, 0x45, 0xe3, 0xbb, 0x8b, 0x83, 0xcf, 0x3d, 0x21, 0x23, 0xe2, 0xef, + 0xf7, 0xd1, 0x0f, 0x61, 0xb5, 0x7b, 0x74, 0x8c, 0x71, 0x35, 0x57, 0xdb, 0x7c, 0x79, 0xd1, 0xb8, + 0xbf, 0x88, 0x53, 0x43, 0x3c, 0x62, 0x2e, 0xe6, 0xa7, 0xb3, 0x5a, 0xf7, 0x3f, 0xcb, 0x60, 0x99, + 0x97, 0xfd, 0x43, 0x7f, 0x87, 0xd6, 0xe2, 0x54, 0x9e, 0x5c, 0xd9, 0xe5, 0x1b, 0x33, 0x7a, 0x25, + 0x26, 0x98, 0x33, 0x7e, 0x00, 0x15, 0x2f, 0x98, 0x7c, 0x36, 0xa0, 0x8c, 0x9c, 0xfa, 0xa6, 0xec, + 0x2d, 0x61, 0x4b, 0xd9, 0x7a, 0xb1, 0x49, 0xbd, 0x17, 0x1e, 0x93, 0x54, 0x30, 0x53, 0xd0, 0x96, + 0xf0, 0xac, 0x8f, 0xbe, 0x84, 0xbc, 0x17, 0x90, 0xb1, 0x29, 0x43, 0x32, 0x77, 0xb0, 0xdf, 0x6f, + 0x1f, 0x1a, 0x0d, 0x76, 0x4a, 0x57, 0x97, 0x9b, 0x79, 0x65, 0xc0, 0x9a, 0x86, 0xea, 0x49, 0x25, + 0xa0, 0x66, 0xd2, 0x6f, 0x7f, 0x09, 0xa7, 0x2c, 0x4a, 0x47, 0x1e, 0x1b, 0x0a, 0x1a, 0x86, 0x3a, + 0x0b, 0x94, 0x70, 0xd2, 0x45, 0x35, 0x28, 0x9a, 0x7a, 0x42, 0x17, 0x10, 0x65, 0x95, 0xab, 0x8d, + 0xa1, 0xb3, 0x06, 0x56, 0x1c, 0x8d, 0xc1, 0x99, 0xe0, 0xe3, 0xe6, 0x7f, 0xf3, 0x60, 0xed, 0xfa, + 0x51, 0x28, 0x4d, 0x1a, 0xfc, 0x60, 0xc1, 0x7f, 0x01, 0xeb, 0x44, 0x7f, 0xaf, 0x08, 0x53, 0x39, + 0x45, 0x97, 0x69, 0xe6, 0x00, 0x1e, 0x66, 0xba, 0x9b, 0x81, 0xe3, 0x92, 0xae, 0x53, 0x50, 0x3e, + 0xed, 0x1c, 0xae, 0x92, 0x37, 0x46, 0xd0, 0x31, 0xac, 0x71, 0xe1, 0x8c, 0x68, 0x28, 0xe3, 0x4c, + 0x64, 0xbe, 0x23, 0x99, 0x1f, 0xd5, 0x67, 0x69, 0xa0, 0x79, 0x86, 0xe3, 0xd5, 0x2e, 0xfa, 0x40, + 0x8f, 0x21, 0x2f, 0xc8, 0x59, 0x52, 0x72, 0x66, 0x5e, 0x12, 0x4c, 0xce, 0xe4, 0x82, 0x0b, 0xcd, + 0x40, 0xbf, 0x01, 0x70, 0xbd, 0x30, 0x20, 0xd2, 0x19, 0x51, 0x61, 0x0e, 0x3b, 0x73, 0x8b, 0xdd, + 0x19, 0x6a, 0xc1, 0x4b, 0x8a, 0x8d, 0x0e, 0xa0, 0xec, 0x90, 0x44, 0xae, 0x85, 0xeb, 0xff, 0x68, + 0xbb, 0x6d, 0xe3, 0xa2, 0xaa, 0x5c, 0x5c, 0x5d, 0x6e, 0x96, 0x12, 0x0b, 0x2e, 0x39, 0xc4, 0xc8, + 0xf7, 0x00, 0xd6, 0xd4, 0xdf, 0x6d, 0xe0, 0xd2, 0x33, 0x12, 0xf9, 0x32, 0x96, 0xc9, 0x35, 0x69, + 0x45, 0x7d, 0x04, 0xba, 0x06, 0x67, 0xd6, 0x55, 0x91, 0x29, 0x1b, 0xfa, 0x1d, 0xac, 0x53, 0xe6, + 0x88, 0xa9, 0x16, 0x6b, 0xb2, 0xc2, 0xd2, 0xf5, 0x9b, 0xed, 0xcd, 0xc0, 0x0b, 0x9b, 0xad, 0xd2, + 0x37, 0xec, 0xcd, 0x7f, 0xe6, 0x00, 0xe2, 0x4c, 0xfd, 0x61, 0x05, 0x88, 0x20, 0xef, 0x12, 0x49, + 0xb4, 0xe6, 0x2a, 0x58, 0xb7, 0xd1, 0x17, 0x00, 0x92, 0x8e, 0x03, 0x9f, 0x48, 0x8f, 0x0d, 0x8d, + 0x6c, 0xde, 0xf5, 0x1c, 0xa4, 0xd0, 0x68, 0x07, 0x0a, 0xe6, 0x63, 0x90, 0xbf, 0x91, 0x67, 0x90, + 0xcd, 0xbf, 0xe6, 0x00, 0xe2, 0x6d, 0x7e, 0xab, 0xf7, 0xd6, 0xb1, 0x5f, 0xbd, 0xae, 0x2f, 0xfd, + 0xe3, 0x75, 0x7d, 0xe9, 0x8f, 0x57, 0xf5, 0xdc, 0xab, 0xab, 0x7a, 0xee, 0x6f, 0x57, 0xf5, 0xdc, + 0xbf, 0xae, 0xea, 0xb9, 0xd3, 0x82, 0xae, 0xfb, 0x7e, 0xf2, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xc1, 0xd8, 0x19, 0x9e, 0x30, 0x13, 0x00, 0x00, } diff --git a/api/specs.proto b/api/specs.proto index 13b52d4105..c14ebeb6d3 100644 --- a/api/specs.proto +++ b/api/specs.proto @@ -393,6 +393,9 @@ message SecretSpec { // The currently recognized values are: // - golang: Go templating Driver templating = 3; + + // Driver is the the secret driver that is used to store the specified secret + Driver driver = 4; } // ConfigSpec specifies user-provided configuration files. diff --git a/api/validation/secrets.go b/api/validation/secrets.go new file mode 100644 index 0000000000..e907b6b411 --- /dev/null +++ b/api/validation/secrets.go @@ -0,0 +1,14 @@ +package validation + +import "fmt" + +// MaxSecretSize is the maximum byte length of the `Secret.Spec.Data` field. +const MaxSecretSize = 500 * 1024 // 500KB + +// ValidateSecretPayload validates the secret payload size +func ValidateSecretPayload(data []byte) error { + if len(data) >= MaxSecretSize || len(data) < 1 { + return fmt.Errorf("secret data must be larger than 0 and less than %d bytes", MaxSecretSize) + } + return nil +} diff --git a/cmd/swarmctl/secret/create.go b/cmd/swarmctl/secret/create.go index 2731ccb60b..ad799a48be 100644 --- a/cmd/swarmctl/secret/create.go +++ b/cmd/swarmctl/secret/create.go @@ -24,8 +24,13 @@ var createCmd = &cobra.Command{ var ( secretData []byte err error + driver string ) + driver, err = flags.GetString("driver") + if err != nil { + return fmt.Errorf("Error reading secret driver %s", err.Error()) + } if flags.Changed("file") { filename, err := flags.GetString("file") if err != nil { @@ -35,7 +40,7 @@ var createCmd = &cobra.Command{ if err != nil { return fmt.Errorf("Error reading from file '%s': %s", filename, err.Error()) } - } else { + } else if driver == "" { secretData, err = ioutil.ReadAll(os.Stdin) if err != nil { return fmt.Errorf("Error reading content from STDIN: %s", err.Error()) @@ -51,6 +56,9 @@ var createCmd = &cobra.Command{ Annotations: api.Annotations{Name: args[0]}, Data: secretData, } + if driver != "" { + spec.Driver = &api.Driver{Name: driver} + } resp, err := client.CreateSecret(common.Context(cmd), &api.CreateSecretRequest{Spec: spec}) if err != nil { @@ -63,4 +71,5 @@ var createCmd = &cobra.Command{ func init() { createCmd.Flags().StringP("file", "f", "", "Rather than read the secret from STDIN, read from the given file") + createCmd.Flags().StringP("driver", "d", "", "Rather than read the secret from STDIN, read the value from an external secret driver") } diff --git a/cmd/swarmctl/secret/list.go b/cmd/swarmctl/secret/list.go index 74d781f897..8f2faf64f8 100644 --- a/cmd/swarmctl/secret/list.go +++ b/cmd/swarmctl/secret/list.go @@ -63,15 +63,21 @@ var ( // Ignore flushing errors - there's nothing we can do. _ = w.Flush() }() - common.PrintHeader(w, "ID", "Name", "Created") + common.PrintHeader(w, "ID", "Name", "Driver", "Created") output = func(s *api.Secret) { created, err := gogotypes.TimestampFromProto(s.Meta.CreatedAt) if err != nil { panic(err) } - fmt.Fprintf(w, "%s\t%s\t%s\n", + var driver string + if s.Spec.Driver != nil { + driver = s.Spec.Driver.Name + } + + fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", s.ID, s.Spec.Annotations.Name, + driver, humanize.Time(created), ) } diff --git a/manager/controlapi/secret.go b/manager/controlapi/secret.go index 5f125e9c60..23708146d6 100644 --- a/manager/controlapi/secret.go +++ b/manager/controlapi/secret.go @@ -6,6 +6,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/api/validation" "github.com/docker/swarmkit/identity" "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager/state/store" @@ -14,9 +15,6 @@ import ( "google.golang.org/grpc/codes" ) -// MaxSecretSize is the maximum byte length of the `Secret.Spec.Data` field. -const MaxSecretSize = 500 * 1024 // 500KB - // assumes spec is not nil func secretFromSecretSpec(spec *api.SecretSpec) *api.Secret { return &api.Secret{ @@ -56,7 +54,6 @@ func (s *Server) UpdateSecret(ctx context.Context, request *api.UpdateSecretRequ if request.SecretID == "" || request.SecretVersion == nil { return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } - var secret *api.Secret err := s.store.Update(func(tx store.Tx) error { secret = store.GetSecret(tx, request.SecretID) @@ -245,9 +242,16 @@ func validateSecretSpec(spec *api.SecretSpec) error { if err := validateConfigOrSecretAnnotations(spec.Annotations); err != nil { return err } - - if len(spec.Data) >= MaxSecretSize || len(spec.Data) < 1 { - return grpc.Errorf(codes.InvalidArgument, "secret data must be larger than 0 and less than %d bytes", MaxSecretSize) + // Check if secret driver is defined + if spec.Driver != nil { + // Ensure secret driver has a name + if spec.Driver.Name == "" { + return grpc.Errorf(codes.InvalidArgument, "secret driver must have a name") + } + return nil + } + if err := validation.ValidateSecretPayload(spec.Data); err != nil { + return grpc.Errorf(codes.InvalidArgument, "%s", err.Error()) } return nil } diff --git a/manager/controlapi/secret_test.go b/manager/controlapi/secret_test.go index 6566e347d4..cf66e39144 100644 --- a/manager/controlapi/secret_test.go +++ b/manager/controlapi/secret_test.go @@ -89,6 +89,16 @@ func TestValidateSecretSpec(t *testing.T) { err := validateSecretSpec(good) assert.NoError(t, err) } + + // Ensure secret driver has a name + spec := createSecretSpec("secret-driver", make([]byte, 1), nil) + spec.Driver = &api.Driver{} + err := validateSecretSpec(spec) + assert.Error(t, err) + assert.Equal(t, codes.InvalidArgument, grpc.Code(err), grpc.ErrorDesc(err)) + spec.Driver.Name = "secret-driver" + err = validateSecretSpec(spec) + assert.NoError(t, err) } func TestCreateSecret(t *testing.T) { diff --git a/manager/controlapi/server.go b/manager/controlapi/server.go index 3d49ef9430..c2490ba002 100644 --- a/manager/controlapi/server.go +++ b/manager/controlapi/server.go @@ -10,7 +10,6 @@ import ( ) var ( - errNotImplemented = errors.New("not implemented") errInvalidArgument = errors.New("invalid argument") ) diff --git a/manager/dispatcher/assignments.go b/manager/dispatcher/assignments.go index 871519efa2..3f17a6d20f 100644 --- a/manager/dispatcher/assignments.go +++ b/manager/dispatcher/assignments.go @@ -1,9 +1,13 @@ package dispatcher import ( + "fmt" + "github.com/Sirupsen/logrus" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/api/equality" + "github.com/docker/swarmkit/api/validation" + "github.com/docker/swarmkit/manager/drivers" "github.com/docker/swarmkit/manager/state/store" ) @@ -24,15 +28,16 @@ type typeAndID struct { } type assignmentSet struct { + dp *drivers.DriverProvider tasksMap map[string]*api.Task tasksUsingDependency map[typeAndID]map[string]struct{} changes map[typeAndID]*api.AssignmentChange - - log *logrus.Entry + log *logrus.Entry } -func newAssignmentSet(log *logrus.Entry) *assignmentSet { +func newAssignmentSet(log *logrus.Entry, dp *drivers.DriverProvider) *assignmentSet { return &assignmentSet{ + dp: dp, changes: make(map[typeAndID]*api.AssignmentChange), tasksMap: make(map[string]*api.Task), tasksUsingDependency: make(map[typeAndID]map[string]struct{}), @@ -53,12 +58,13 @@ func (a *assignmentSet) addTaskDependencies(readTx store.ReadTx, t *api.Task) { if len(a.tasksUsingDependency[mapKey]) == 0 { a.tasksUsingDependency[mapKey] = make(map[string]struct{}) - secret := store.GetSecret(readTx, secretID) - if secret == nil { + secret, err := a.secret(readTx, secretID) + if err != nil { a.log.WithFields(logrus.Fields{ "secret.id": secretID, "secret.name": secretRef.SecretName, - }).Debug("secret not found") + "error": err, + }).Error("failed to fetch secret") continue } @@ -245,3 +251,29 @@ func (a *assignmentSet) message() api.AssignmentsMessage { return message } + +// secret populates the secret value from raft store. For external secrets, the value is populated +// from the secret driver. +func (a *assignmentSet) secret(readTx store.ReadTx, secretID string) (*api.Secret, error) { + secret := store.GetSecret(readTx, secretID) + if secret == nil { + return nil, fmt.Errorf("secret not found") + } + if secret.Spec.Driver == nil { + return secret, nil + } + d, err := a.dp.NewSecretDriver(secret.Spec.Driver) + if err != nil { + return nil, err + } + value, err := d.Get(&secret.Spec) + if err != nil { + return nil, err + } + if err := validation.ValidateSecretPayload(value); err != nil { + return nil, err + } + // Assign the secret + secret.Spec.Data = value + return secret, nil +} diff --git a/manager/dispatcher/dispatcher.go b/manager/dispatcher/dispatcher.go index 47899cc48a..8ec3ae27df 100644 --- a/manager/dispatcher/dispatcher.go +++ b/manager/dispatcher/dispatcher.go @@ -17,6 +17,7 @@ import ( "github.com/docker/swarmkit/api/equality" "github.com/docker/swarmkit/ca" "github.com/docker/swarmkit/log" + "github.com/docker/swarmkit/manager/drivers" "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/remotes" "github.com/docker/swarmkit/watch" @@ -125,6 +126,7 @@ type Dispatcher struct { ctx context.Context cancel context.CancelFunc clusterUpdateQueue *watch.Queue + dp *drivers.DriverProvider taskUpdates map[string]*api.TaskStatus // indexed by task ID taskUpdatesLock sync.Mutex @@ -142,8 +144,9 @@ type Dispatcher struct { } // New returns Dispatcher with cluster interface(usually raft.Node). -func New(cluster Cluster, c *Config) *Dispatcher { +func New(cluster Cluster, c *Config, dp *drivers.DriverProvider) *Dispatcher { d := &Dispatcher{ + dp: dp, nodes: newNodeStore(c.HeartbeatPeriod, c.HeartbeatEpsilon, c.GracePeriodMultiplier, c.RateLimitPeriod), downNodes: newNodeStore(defaultNodeDownPeriod, 0, 1, 0), store: cluster.MemoryStore(), @@ -836,7 +839,7 @@ func (d *Dispatcher) Assignments(r *api.AssignmentsRequest, stream api.Dispatche var ( sequence int64 appliesTo string - assignments = newAssignmentSet(log) + assignments = newAssignmentSet(log, d.dp) ) sendMessage := func(msg api.AssignmentsMessage, assignmentType api.AssignmentsMessage_Type) error { diff --git a/manager/dispatcher/dispatcher_test.go b/manager/dispatcher/dispatcher_test.go index 7639ecf632..14df088ba0 100644 --- a/manager/dispatcher/dispatcher_test.go +++ b/manager/dispatcher/dispatcher_test.go @@ -2,9 +2,13 @@ package dispatcher import ( "crypto/tls" + "encoding/json" "errors" "fmt" + "io/ioutil" "net" + "net/http" + "net/http/httptest" "sync" "testing" "time" @@ -15,11 +19,14 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "github.com/docker/docker/pkg/plugingetter" + "github.com/docker/docker/pkg/plugins" "github.com/docker/go-events" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" cautils "github.com/docker/swarmkit/ca/testutils" "github.com/docker/swarmkit/identity" + "github.com/docker/swarmkit/manager/drivers" "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/testutils" digest "github.com/opencontainers/go-digest" @@ -36,6 +43,7 @@ type grpcDispatcher struct { conns []*grpc.ClientConn testCA *cautils.TestCA testCluster *testCluster + PluginGetter *mockPluginGetter } func (gd *grpcDispatcher) Close() { @@ -45,6 +53,7 @@ func (gd *grpcDispatcher) Close() { } gd.dispatcherServer.Stop() gd.grpcServer.Stop() + gd.PluginGetter.Close() gd.testCA.Stop() } @@ -145,7 +154,8 @@ func startDispatcher(c *Config) (*grpcDispatcher, error) { s := grpc.NewServer(serverOpts...) tc := newTestCluster(l.Addr().String(), tca.MemoryStore) - d := New(tc, c) + driverGetter := &mockPluginGetter{} + d := New(tc, c, drivers.New(driverGetter)) authorize := func(ctx context.Context, roles []string) error { _, err := ca.AuthorizeForwardedRoleAndOrg(ctx, roles, []string{ca.ManagerRole}, tca.Organization, nil) @@ -203,6 +213,7 @@ func startDispatcher(c *Config) (*grpcDispatcher, error) { grpcServer: s, testCA: tca, testCluster: tc, + PluginGetter: driverGetter, }, nil } @@ -406,6 +417,84 @@ func TestAssignmentsErrorsIfNoSessionID(t *testing.T) { assert.Equal(t, grpc.Code(err), codes.InvalidArgument) } +func TestAssignmentsSecretDriver(t *testing.T) { + t.Parallel() + + const ( + secretDriver = "secret-driver" + existingSecretName = "existing-secret" + secretValue = "custom-secret-value" + ) + + responses := map[string]*drivers.SecretsProviderResponse{ + existingSecretName: {Value: secretValue}, + } + + mux := http.NewServeMux() + mux.HandleFunc(drivers.SecretsProviderAPI, func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + body, err := ioutil.ReadAll(r.Body) + var request drivers.SecretsProviderRequest + assert.NoError(t, err) + assert.NoError(t, json.Unmarshal(body, &request)) + response := responses[request.Name] + assert.NotNil(t, response) + resp, err := json.Marshal(response) + assert.NoError(t, err) + w.Write(resp) + }) + + gd, err := startDispatcher(DefaultConfig()) + assert.NoError(t, err) + assert.NoError(t, gd.PluginGetter.SetupPlugin(secretDriver, mux)) + defer gd.Close() + + expectedSessionID, nodeID := getSessionAndNodeID(t, gd.Clients[0]) + + secret := &api.Secret{ + ID: "driverSecret", + Spec: api.SecretSpec{ + Annotations: api.Annotations{Name: existingSecretName}, + Driver: &api.Driver{Name: secretDriver}, + }, + } + config := &api.Config{ + ID: "config", + Spec: api.ConfigSpec{ + Data: []byte("config"), + }, + } + spec := taskSpecFromDependencies(secret, config) + task := &api.Task{ + NodeID: nodeID, + ID: "secretTask", + Status: api.TaskStatus{State: api.TaskStateReady}, + DesiredState: api.TaskStateNew, + Spec: spec, + } + + err = gd.Store.Update(func(tx store.Tx) error { + assert.NoError(t, store.CreateSecret(tx, secret)) + assert.NoError(t, store.CreateConfig(tx, config)) + assert.NoError(t, store.CreateTask(tx, task)) + return nil + }) + assert.NoError(t, err) + + stream, err := gd.Clients[0].Assignments(context.Background(), &api.AssignmentsRequest{SessionID: expectedSessionID}) + assert.NoError(t, err) + defer stream.CloseSend() + + resp, err := stream.Recv() + assert.NoError(t, err) + + _, _, secretChanges := splitChanges(resp.Changes) + assert.Len(t, secretChanges, 1) + for _, s := range secretChanges { + assert.Equal(t, secretValue, string(s.Spec.Data)) + } +} + // Assignments will send down any existing node tasks > ASSIGNED, and any secrets // for said tasks that are <= RUNNING (if the secrets exist) func TestAssignmentsInitialNodeTasks(t *testing.T) { @@ -419,6 +508,7 @@ func TestAssignmentsInitialNodeTasks(t *testing.T) { // create the relevant secrets and tasks secrets, configs, tasks := makeTasksAndDependencies(t, nodeID) + err = gd.Store.Update(func(tx store.Tx) error { for _, secret := range secrets { assert.NoError(t, store.CreateSecret(tx, secret)) @@ -1550,3 +1640,72 @@ func TestClusterUpdatesSendMessages(t *testing.T) { require.Equal(t, expected, msg) } } + +// mockPluginGetter enables mocking the server plugin getter with customized plugins +type mockPluginGetter struct { + addr string + server *httptest.Server + name string + plugin plugingetter.CompatPlugin +} + +// SetupPlugin setup a new plugin - the same plugin wil always return in all calls +func (m *mockPluginGetter) SetupPlugin(name string, handler http.Handler) error { + m.server = httptest.NewServer(handler) + client, err := plugins.NewClient(m.server.URL, nil) + if err != nil { + return err + } + m.plugin = NewMockPlugin(m.name, client) + m.name = name + return nil +} + +// Close closes the mock plugin getter +func (m *mockPluginGetter) Close() { + if m.server == nil { + return + } + m.server.Close() +} + +func (m *mockPluginGetter) Get(name, capability string, mode int) (plugingetter.CompatPlugin, error) { + if name != m.name { + return nil, fmt.Errorf("plugin with name %s not defined", name) + } + return m.plugin, nil +} +func (m *mockPluginGetter) GetAllByCap(capability string) ([]plugingetter.CompatPlugin, error) { + return nil, nil +} +func (m *mockPluginGetter) GetAllManagedPluginsByCap(capability string) []plugingetter.CompatPlugin { + return nil +} +func (m *mockPluginGetter) Handle(capability string, callback func(string, *plugins.Client)) { + return +} + +// MockPlugin mocks a v2 docker plugin +type MockPlugin struct { + client *plugins.Client + name string +} + +// NewMockPlugin creates a new v2 plugin fake (returns the specified client and name for all calls) +func NewMockPlugin(name string, client *plugins.Client) *MockPlugin { + return &MockPlugin{name: name, client: client} +} + +func (m *MockPlugin) Client() *plugins.Client { + return m.client +} +func (m *MockPlugin) Name() string { + return m.name +} +func (m *MockPlugin) BasePath() string { + return "" + +} +func (m *MockPlugin) IsV1() bool { + return false +} diff --git a/manager/drivers/provider.go b/manager/drivers/provider.go new file mode 100644 index 0000000000..0d9be6119d --- /dev/null +++ b/manager/drivers/provider.go @@ -0,0 +1,34 @@ +package drivers + +import ( + "fmt" + + "github.com/docker/docker/pkg/plugingetter" + "github.com/docker/swarmkit/api" +) + +// DriverProvider provides external drivers +type DriverProvider struct { + pluginGetter plugingetter.PluginGetter +} + +// New returns a new driver provider +func New(pluginGetter plugingetter.PluginGetter) *DriverProvider { + return &DriverProvider{pluginGetter: pluginGetter} +} + +// NewSecretDriver creates a new driver for fetching secrets +func (m *DriverProvider) NewSecretDriver(driver *api.Driver) (*SecretDriver, error) { + if m.pluginGetter == nil { + return nil, fmt.Errorf("plugin getter is nil") + } + if driver == nil && driver.Name == "" { + return nil, fmt.Errorf("driver specification is nil") + } + // Search for the specified plugin + plugin, err := m.pluginGetter.Get(driver.Name, SecretsProviderCapability, plugingetter.Lookup) + if err != nil { + return nil, err + } + return NewSecretDriver(plugin), nil +} diff --git a/manager/drivers/secrets.go b/manager/drivers/secrets.go new file mode 100644 index 0000000000..9c8ccc7d01 --- /dev/null +++ b/manager/drivers/secrets.go @@ -0,0 +1,55 @@ +package drivers + +import ( + "fmt" + + "github.com/docker/docker/pkg/plugingetter" + "github.com/docker/swarmkit/api" +) + +const ( + // SecretsProviderAPI is the endpoint for fetching secrets from plugins + SecretsProviderAPI = "/SecretProvider.GetSecret" + + // SecretsProviderCapability is the secrets provider plugin capability identification + SecretsProviderCapability = "secretprovider" +) + +// SecretDriver provides secrets from different stores +type SecretDriver struct { + plugin plugingetter.CompatPlugin +} + +// NewSecretDriver creates a new driver that provides third party secrets +func NewSecretDriver(plugin plugingetter.CompatPlugin) *SecretDriver { + return &SecretDriver{plugin: plugin} +} + +// Get gets a secret from the secret provider +func (d *SecretDriver) Get(spec *api.SecretSpec) ([]byte, error) { + if spec == nil { + return nil, fmt.Errorf("spec is nil") + } + var secretResp SecretsProviderResponse + secretReq := &SecretsProviderRequest{Name: spec.Annotations.Name} + err := d.plugin.Client().Call(SecretsProviderAPI, secretReq, &secretResp) + if err != nil { + return nil, err + } + if secretResp.Err != "" { + return nil, fmt.Errorf(secretResp.Err) + } + // Assign the secret value + return []byte(secretResp.Value), nil +} + +// SecretsProviderRequest is the secrets provider request. +type SecretsProviderRequest struct { + Name string `json:"name"` // Name is the name of the secret plugin +} + +// SecretsProviderResponse is the secrets provider response. +type SecretsProviderResponse struct { + Value string `json:"value"` // Value is the value of the secret + Err string `json:"err"` // Err is the error response of the plugin +} diff --git a/manager/manager.go b/manager/manager.go index d17e8ec231..4771c9dcbe 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -26,6 +26,7 @@ import ( "github.com/docker/swarmkit/manager/allocator/networkallocator" "github.com/docker/swarmkit/manager/controlapi" "github.com/docker/swarmkit/manager/dispatcher" + "github.com/docker/swarmkit/manager/drivers" "github.com/docker/swarmkit/manager/health" "github.com/docker/swarmkit/manager/keymanager" "github.com/docker/swarmkit/manager/logbroker" @@ -218,7 +219,7 @@ func New(config *Config) (*Manager, error) { m := &Manager{ config: *config, caserver: ca.NewServer(raftNode.MemoryStore(), config.SecurityConfig, config.RootCAPaths), - dispatcher: dispatcher.New(raftNode, dispatcher.DefaultConfig()), + dispatcher: dispatcher.New(raftNode, dispatcher.DefaultConfig(), drivers.New(config.PluginGetter)), logbroker: logbroker.New(raftNode.MemoryStore()), server: grpc.NewServer(opts...), localserver: grpc.NewServer(opts...),