diff --git a/changelog/fragments/1784582418-secure-output-api-key-storage.yaml b/changelog/fragments/1784582418-secure-output-api-key-storage.yaml new file mode 100644 index 0000000000..85ae99a6cc --- /dev/null +++ b/changelog/fragments/1784582418-secure-output-api-key-storage.yaml @@ -0,0 +1,14 @@ +kind: security + +summary: Store output API key secrets in .fleet-secrets instead of .fleet-agents + +description: | + Fleet Server now writes output API key material to .fleet-secrets via the Fleet + secrets API (POST /_fleet/secret) and stores only a $co.elastic.secret{id} reference + in .fleet-agents-7. Previously, the raw id:secret string was stored in plaintext in + outputs.{name}.api_key, which the kibana_system role could read and use to escalate + privileges to SIEM data stream write access. Retired secrets are deleted from + .fleet-secrets on ACK. Existing plaintext values continue to work transparently + until the next key rotation. + +component: fleet-server diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go index baceacad26..b6973c57c9 100644 --- a/internal/pkg/api/handleAck.go +++ b/internal/pkg/api/handleAck.go @@ -28,6 +28,7 @@ import ( "github.com/elastic/fleet-server/v7/internal/pkg/logger/ecs" "github.com/elastic/fleet-server/v7/internal/pkg/model" "github.com/elastic/fleet-server/v7/internal/pkg/policy" + "github.com/elastic/fleet-server/v7/internal/pkg/secret" "github.com/elastic/fleet-server/v7/internal/pkg/smap" ) @@ -507,6 +508,7 @@ func updateAPIKey(ctx context.Context, } } invalidateAPIKeys(ctx, zlog, bulk, toRetireAPIKeyIDs, apiKeyID) + deleteRetiredSecrets(ctx, zlog, bulk, toRetireAPIKeyIDs) } return nil @@ -585,6 +587,17 @@ func (ack *AckT) handleUnenroll(ctx context.Context, zlog zerolog.Logger, agent zlog.Info().Any("fleet.policy.apiKeyIDsToRetire", apiKeys).Msg("handleUnenroll invalidate API keys") ack.invalidateAPIKeys(ctx, zlog, apiKeys, "") + // Delete secrets for all output API keys: retired ones (carried in apiKeys via SecretID) + // and the current active ones (secret ID embedded in output.APIKey as a reference string). + allSecretItems := make([]model.ToRetireAPIKeyIdsItems, 0, len(apiKeys)+len(agent.Outputs)) + allSecretItems = append(allSecretItems, apiKeys...) + for _, output := range agent.Outputs { + if secretID, ok := secret.ParseSecretReference(output.APIKey); ok { + allSecretItems = append(allSecretItems, model.ToRetireAPIKeyIdsItems{SecretID: secretID}) + } + } + deleteRetiredSecrets(ctx, zlog, ack.bulk, allSecretItems) + now := time.Now().UTC().Format(time.RFC3339) doc := bulk.UpdateFields{ dl.FieldActive: false, @@ -744,3 +757,14 @@ func invalidateAPIKeys(ctx context.Context, zlog zerolog.Logger, bulk bulk.Bulk, } } } + +func deleteRetiredSecrets(ctx context.Context, zlog zerolog.Logger, bulk bulk.Bulk, toRetireAPIKeyIDs []model.ToRetireAPIKeyIdsItems) { + for _, k := range toRetireAPIKeyIDs { + if k.SecretID == "" { + continue + } + if err := bulk.DeleteSecret(ctx, k.SecretID); err != nil { + zlog.Warn().Err(err).Str("secret_id", k.SecretID).Msg("Failed to delete retired output API key secret") + } + } +} diff --git a/internal/pkg/api/handleAck_test.go b/internal/pkg/api/handleAck_test.go index 72434570ba..6ad8fe5408 100644 --- a/internal/pkg/api/handleAck_test.go +++ b/internal/pkg/api/handleAck_test.go @@ -834,3 +834,47 @@ func TestValidateAckRequest(t *testing.T) { }) } } + +func TestDeleteRetiredSecrets(t *testing.T) { + logger := testlog.SetLogger(t) + _ = logger + + t.Run("deletes secrets for items with a SecretID", func(t *testing.T) { + bulker := ftesting.NewMockBulk() + bulker.On("DeleteSecret", mock.Anything, "secret-1").Return(nil).Once() + bulker.On("DeleteSecret", mock.Anything, "secret-2").Return(nil).Once() + + items := []model.ToRetireAPIKeyIdsItems{ + {ID: "key-1", SecretID: "secret-1"}, + {ID: "key-2", SecretID: "secret-2"}, + {ID: "key-3"}, // no SecretID — plaintext key, no cleanup needed + } + + deleteRetiredSecrets(context.Background(), testlog.SetLogger(t), bulker, items) + bulker.AssertExpectations(t) + bulker.AssertNotCalled(t, "DeleteSecret", mock.Anything, "") + }) + + t.Run("logs warning on delete failure but does not return error", func(t *testing.T) { + bulker := ftesting.NewMockBulk() + bulker.On("DeleteSecret", mock.Anything, "secret-bad").Return(errors.New("delete failed")).Once() + + items := []model.ToRetireAPIKeyIdsItems{ + {ID: "key-1", SecretID: "secret-bad"}, + } + + // Should not panic or propagate the error. + deleteRetiredSecrets(context.Background(), testlog.SetLogger(t), bulker, items) + bulker.AssertExpectations(t) + }) + + t.Run("no-op when all items lack a SecretID", func(t *testing.T) { + bulker := ftesting.NewMockBulk() + items := []model.ToRetireAPIKeyIdsItems{ + {ID: "key-1"}, + {ID: "key-2"}, + } + deleteRetiredSecrets(context.Background(), testlog.SetLogger(t), bulker, items) + bulker.AssertNotCalled(t, "DeleteSecret", mock.Anything, mock.Anything) + }) +} diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go index 9705c21065..776e16c75e 100644 --- a/internal/pkg/bulk/engine.go +++ b/internal/pkg/bulk/engine.go @@ -84,6 +84,8 @@ type Bulk interface { RemoteOutputConfigChanged(zlog zerolog.Logger, name string, newCfg map[string]any) bool ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error) + WriteSecret(ctx context.Context, value string) (string, error) + DeleteSecret(ctx context.Context, secretID string) error } const kModBulk = "bulk" @@ -335,6 +337,14 @@ func (b *Bulker) ReadSecrets(ctx context.Context, secretIds []string) (map[strin return result, nil } +func (b *Bulker) WriteSecret(ctx context.Context, value string) (string, error) { + return WriteSecret(ctx, b.Client(), value) +} + +func (b *Bulker) DeleteSecret(ctx context.Context, secretID string) error { + return DeleteSecret(ctx, b.Client(), secretID) +} + // Stop timer, but don't stall on channel. // API doesn't not seem to work as specified. func stopTimer(t *time.Timer) { diff --git a/internal/pkg/bulk/secret.go b/internal/pkg/bulk/secret.go index acebe92130..9448493149 100644 --- a/internal/pkg/bulk/secret.go +++ b/internal/pkg/bulk/secret.go @@ -5,8 +5,11 @@ package bulk import ( + "bytes" "context" "encoding/json" + "fmt" + "io" "net/http" "github.com/elastic/go-elasticsearch/v8" @@ -37,6 +40,10 @@ func (c *ExtendedAPI) Read(ctx context.Context, secretID string) (*SecretRespons return nil, err } defer res.Body.Close() + if res.StatusCode >= 400 { + body, _ := io.ReadAll(res.Body) + return nil, fmt.Errorf("unexpected status %d from fleet secret read: %s", res.StatusCode, body) + } var secretResp SecretResponse err = json.NewDecoder(res.Body).Decode(&secretResp) @@ -46,6 +53,42 @@ func (c *ExtendedAPI) Read(ctx context.Context, secretID string) (*SecretRespons return &secretResp, nil } +// Write stores a secret value via the Fleet ES plugin secrets API and returns the assigned secret ID. +// POST /_fleet/secret +func (c *ExtendedAPI) Write(ctx context.Context, value string) (string, error) { + body, err := json.Marshal(struct { + Value string `json:"value"` + }{Value: value}) + if err != nil { + return "", err + } + + req, err := http.NewRequestWithContext(ctx, "POST", "/_fleet/secret", bytes.NewReader(body)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + res, err := c.Perform(req) + if err != nil { + return "", err + } + defer res.Body.Close() + if res.StatusCode >= 400 { + body, _ := io.ReadAll(res.Body) + return "", fmt.Errorf("unexpected status %d from fleet secret write: %s", res.StatusCode, body) + } + + var resp struct { + ID string `json:"id"` + } + if err := json.NewDecoder(res.Body).Decode(&resp); err != nil { + return "", err + } + return resp.ID, nil +} + type SecretResponse struct { Value string } @@ -63,3 +106,38 @@ func ReadSecret(ctx context.Context, client *elasticsearch.Client, secretID stri } return (*res).Value, err } + +func WriteSecret(ctx context.Context, client *elasticsearch.Client, value string) (string, error) { + span, ctx := apm.StartSpan(ctx, "writeSecret", "elasticsearch") + defer span.End() + es := ExtendedClient{Client: client, Custom: &ExtendedAPI{client}} + return es.Custom.Write(ctx, value) +} + +// Delete removes a secret from the Fleet secrets store. +// DELETE /_fleet/secret/secretId +func (c *ExtendedAPI) Delete(ctx context.Context, secretID string) error { + req, err := http.NewRequestWithContext(ctx, "DELETE", "/_fleet/secret/"+secretID, nil) + if err != nil { + return err + } + req.Header.Set("Accept", "application/json") + + res, err := c.Perform(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode >= 400 { + body, _ := io.ReadAll(res.Body) + return fmt.Errorf("unexpected status %d from fleet secret delete: %s", res.StatusCode, body) + } + return nil +} + +func DeleteSecret(ctx context.Context, client *elasticsearch.Client, secretID string) error { + span, ctx := apm.StartSpan(ctx, "deleteSecret", "elasticsearch") + defer span.End() + es := ExtendedClient{Client: client, Custom: &ExtendedAPI{client}} + return es.Custom.Delete(ctx, secretID) +} diff --git a/internal/pkg/model/schema.go b/internal/pkg/model/schema.go index c0673a06b8..2aa9d9041f 100644 --- a/internal/pkg/model/schema.go +++ b/internal/pkg/model/schema.go @@ -606,6 +606,9 @@ type ToRetireAPIKeyIdsItems struct { // Date/time the API key was retired RetiredAt string `json:"retired_at,omitempty"` + + // Fleet secret ID for the retired API key, if the key was stored in .fleet-secrets + SecretID string `json:"secret_id,omitempty"` } // UnitsItems diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index 57e8f8f183..81780fb43a 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -22,6 +22,7 @@ import ( "github.com/elastic/fleet-server/v7/internal/pkg/dl" "github.com/elastic/fleet-server/v7/internal/pkg/logger/ecs" "github.com/elastic/fleet-server/v7/internal/pkg/model" + "github.com/elastic/fleet-server/v7/internal/pkg/secret" "github.com/elastic/fleet-server/v7/internal/pkg/smap" ) @@ -129,11 +130,15 @@ func (p *Output) prepareElasticsearch( } if !found { zlog.Info().Str(ecs.APIKeyID, agentOutput.APIKeyID).Str(ecs.PolicyOutputName, agentOutputName).Msg("Output removed, will retire API key") - toRetireAPIKeys = &model.ToRetireAPIKeyIdsItems{ + retiring := model.ToRetireAPIKeyIdsItems{ ID: agentOutput.APIKeyID, RetiredAt: time.Now().UTC().Format(time.RFC3339), Output: agentOutputName, } + if secretID, ok := secret.ParseSecretReference(agentOutput.APIKey); ok { + retiring.SecretID = secretID + } + toRetireAPIKeys = &retiring removedOutputName = agentOutputName break } @@ -307,8 +312,14 @@ func (p *Output) prepareElasticsearch( Str(ecs.DefaultOutputAPIKeyID, outputAPIKey.ID). Msg("Updating agent record to pick up default output key.") + secretID, err := bulker.WriteSecret(ctx, outputAPIKey.Agent()) + if err != nil { + return fmt.Errorf("failed writing output API key secret: %w", err) + } + apiKeyRef := secret.MakeSecretReference(secretID) + fields := map[string]any{ - dl.FieldPolicyOutputAPIKey: outputAPIKey.Agent(), + dl.FieldPolicyOutputAPIKey: apiKeyRef, dl.FieldPolicyOutputAPIKeyID: outputAPIKey.ID, dl.FieldPolicyOutputPermissionsHash: p.Role.Sha2, } @@ -317,11 +328,15 @@ func (p *Output) prepareElasticsearch( fields[dl.FiledType] = OutputTypeElasticsearch } if output.APIKeyID != "" { - fields[dl.FieldPolicyOutputToRetireAPIKeyIDs] = model.ToRetireAPIKeyIdsItems{ + retiring := model.ToRetireAPIKeyIdsItems{ ID: output.APIKeyID, RetiredAt: time.Now().UTC().Format(time.RFC3339), Output: p.Name, } + if secretID, ok := secret.ParseSecretReference(output.APIKey); ok { + retiring.SecretID = secretID + } + fields[dl.FieldPolicyOutputToRetireAPIKeyIDs] = retiring } // Using painless script to append the old keys to the history @@ -332,6 +347,9 @@ func (p *Output) prepareElasticsearch( if err = bulker.Update(ctx, dl.FleetAgents, agent.Id, body, bulk.WithRefresh(), bulk.WithRetryOnConflict(3)); err != nil { zlog.Error().Err(err).Msg("fail update agent record") + if delErr := bulker.DeleteSecret(ctx, secretID); delErr != nil { + zlog.Warn().Err(delErr).Str("secret_id", secretID).Msg("failed to delete orphaned output API key secret after agent update failure") + } return fmt.Errorf("fail update agent record: %w", err) } @@ -340,7 +358,7 @@ func (p *Output) prepareElasticsearch( // data is correct and in sync with ES, so it can be safely used after // this method returns. output.Type = OutputTypeElasticsearch - output.APIKey = outputAPIKey.Agent() + output.APIKey = apiKeyRef output.APIKeyID = outputAPIKey.ID output.PermissionsHash = p.Role.Sha2 // for the sake of consistency } @@ -362,7 +380,19 @@ func (p *Output) prepareElasticsearch( // in place to reduce number of agent policy allocation when sending the updated // agent policy to multiple agents. // See: https://github.com/elastic/fleet-server/issues/1301 - outputMap[p.Name]["api_key"] = output.APIKey + apiKey := output.APIKey + if secretID, ok := secret.ParseSecretReference(apiKey); ok { + resolved, err := bulker.ReadSecrets(ctx, []string{secretID}) + if err != nil { + return fmt.Errorf("failed resolving output API key secret: %w", err) + } + val, ok := resolved[secretID] + if !ok || val == "" { + return fmt.Errorf("output API key secret %q not found", secretID) + } + apiKey = val + } + outputMap[p.Name]["api_key"] = apiKey return nil } @@ -476,29 +506,29 @@ func renderUpdatePainlessScript(outputName string, fields map[string]any) ([]byt var source strings.Builder // prepare agent.elasticsearch_outputs[OUTPUT_NAME] - source.WriteString(fmt.Sprintf(` + fmt.Fprintf(&source, ` if (ctx._source['outputs']==null) {ctx._source['outputs']=new HashMap();} if (ctx._source['outputs']['%s']==null) {ctx._source['outputs']['%s']=new HashMap();} -`, outputName, outputName)) +`, outputName, outputName) for field := range fields { if field == dl.FieldPolicyOutputToRetireAPIKeyIDs { // dl.FieldPolicyOutputToRetireAPIKeyIDs is a special case. // It's an array that gets deleted when the keys are invalidated. // Thus, append the old API key ID, create the field if necessary. - source.WriteString(fmt.Sprintf(` + fmt.Fprintf(&source, ` if (ctx._source['outputs']['%s'].%s==null) {ctx._source['outputs']['%s'].%s=new ArrayList();} if (!ctx._source['outputs']['%s'].%s.contains(params.%s)) {ctx._source['outputs']['%s'].%s.add(params.%s);} -`, outputName, field, outputName, field, outputName, field, field, outputName, field, field)) +`, outputName, field, outputName, field, outputName, field, field, outputName, field, field) } else { // Update the other fields - source.WriteString(fmt.Sprintf(` + fmt.Fprintf(&source, ` ctx._source['outputs']['%s'].%s=params.%s;`, - outputName, field, field)) + outputName, field, field) } } diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go index b344ffcced..6157538171 100644 --- a/internal/pkg/policy/policy_output_test.go +++ b/internal/pkg/policy/policy_output_test.go @@ -259,6 +259,8 @@ func TestPolicyOutputESPrepare(t *testing.T) { bulker.On("APIKeyCreate", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return(&apiKey, nil).Once() + const secretID = "test-secret-id" + bulker.On("WriteSecret", mock.Anything, apiKey.Agent()).Return(secretID, nil).Once() output := Output{ Type: OutputTypeElasticsearch, @@ -281,10 +283,12 @@ func TestPolicyOutputESPrepare(t *testing.T) { key, ok := policyMap[output.Name]["api_key"].(string) gotOutput := testAgent.Outputs[output.Name] - require.True(t, ok, "unable to case api key") - assert.Equal(t, apiKey.Agent(), key) + // The policy map receives the resolved secret value, not the reference. + require.True(t, ok, "unable to cast api key") + assert.Equal(t, secretID+"_value", key) - assert.Equal(t, apiKey.Agent(), gotOutput.APIKey) + // The agent document stores the secret reference, not the plaintext key. + assert.Equal(t, "$co.elastic.secret{"+secretID+"}", gotOutput.APIKey) assert.Equal(t, apiKey.ID, gotOutput.APIKeyID) assert.Equal(t, output.Role.Sha2, gotOutput.PermissionsHash) assert.Equal(t, output.Type, gotOutput.Type) @@ -298,6 +302,67 @@ func TestPolicyOutputESPrepare(t *testing.T) { bulker.AssertExpectations(t) }) + + t.Run("Secret is deleted when agent document update fails", func(t *testing.T) { + logger := testlog.SetLogger(t) + bulker := ftesting.NewMockBulk() + apiKey := bulk.APIKey{ID: "abc", Key: "new-key"} + bulker.On("APIKeyCreate", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(&apiKey, nil).Once() + const secretID = "test-secret-id" + bulker.On("WriteSecret", mock.Anything, apiKey.Agent()).Return(secretID, nil).Once() + bulker.On("Update", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(errors.New("ES update failed")).Once() + bulker.On("DeleteSecret", mock.Anything, secretID).Return(nil).Once() + + output := Output{ + Type: OutputTypeElasticsearch, + Name: "test output", + Role: &RoleT{Sha2: "new-hash", Raw: TestPayload}, + } + policyMap := map[string]map[string]any{"test output": {}} + testAgent := &model.Agent{Outputs: map[string]*model.PolicyOutput{}} + + err := output.Prepare(context.Background(), logger, bulker, testAgent, policyMap) + require.Error(t, err) + bulker.AssertExpectations(t) + }) + + t.Run("Existing plaintext key is delivered without modification", func(t *testing.T) { + logger := testlog.SetLogger(t) + bulker := ftesting.NewMockBulk() + + apiKey := bulk.APIKey{ID: "existing-id", Key: "existing-key"} + hashPerm := "existing-hash" + output := Output{ + Type: OutputTypeElasticsearch, + Name: "test output", + Role: &RoleT{Sha2: hashPerm, Raw: TestPayload}, + } + policyMap := map[string]map[string]any{"test output": {}} + testAgent := &model.Agent{ + Outputs: map[string]*model.PolicyOutput{ + output.Name: { + APIKey: apiKey.Agent(), + APIKeyID: apiKey.ID, + PermissionsHash: hashPerm, + Type: OutputTypeElasticsearch, + }, + }, + } + + err := output.Prepare(context.Background(), logger, bulker, testAgent, policyMap) + require.NoError(t, err) + + // Plaintext key is passed through directly — WriteSecret is not called. + key, ok := policyMap[output.Name]["api_key"].(string) + require.True(t, ok) + assert.Equal(t, apiKey.Agent(), key) + bulker.AssertNotCalled(t, "WriteSecret", mock.Anything, mock.Anything) + bulker.AssertExpectations(t) + }) } func TestPolicyRemoteESOutputPrepareNoRole(t *testing.T) { @@ -474,6 +539,8 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) { } return doc.Message == "" && doc.State == client.UnitStateHealthy.String() }), mock.Anything).Return("", nil) + const secretID = "test-secret-id" + bulker.On("WriteSecret", mock.Anything, apiKey.Agent()).Return(secretID, nil).Once() output := Output{ Type: OutputTypeRemoteElasticsearch, @@ -499,10 +566,10 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) { key, ok := policyMap[output.Name]["api_key"].(string) gotOutput := testAgent.Outputs[output.Name] - require.True(t, ok, "unable to case api key") - assert.Equal(t, apiKey.Agent(), key) + require.True(t, ok, "unable to cast api key") + assert.Equal(t, secretID+"_value", key) - assert.Equal(t, apiKey.Agent(), gotOutput.APIKey) + assert.Equal(t, "$co.elastic.secret{"+secretID+"}", gotOutput.APIKey) assert.Equal(t, apiKey.ID, gotOutput.APIKeyID) assert.Equal(t, output.Role.Sha2, gotOutput.PermissionsHash) assert.Empty(t, gotOutput.ToRetireAPIKeyIds) diff --git a/internal/pkg/secret/secret.go b/internal/pkg/secret/secret.go index 08ccefa237..091ea1e9be 100644 --- a/internal/pkg/secret/secret.go +++ b/internal/pkg/secret/secret.go @@ -76,7 +76,7 @@ func ProcessInputsSecrets(data *model.PolicyData, secretValues map[string]string // the values of secret refs in inputs and input streams properties using the old format // for specifying secrets: : $co.elastic.secret{} func processInputsWithInlineSecrets(data *model.PolicyData, secretValues map[string]string) ([]map[string]any, []string) { - result := make([]map[string]any, 0) + result := make([]map[string]any, 0, len(data.Inputs)) keys := make([]string, 0) for i, input := range data.Inputs { replacedInput, ks := replaceInlineSecretRefsInMap(input, secretValues) @@ -92,7 +92,7 @@ func processInputsWithInlineSecrets(data *model.PolicyData, secretValues map[str // the values of secret refs in inputs and input streams properties using the new format // for specifying secrets: secrets...id: func processInputsWithPathSecrets(data *model.PolicyData, secretValues map[string]string) ([]map[string]any, []string) { - result := make([]map[string]any, 0) + result := make([]map[string]any, 0, len(data.Inputs)) keys := make([]string, 0) for i, inp := range data.Inputs { @@ -158,7 +158,7 @@ func replacePathSecretRefsInMap(m smap.Map, secretValues map[string]string) (map func replacePathSecretRefsInSlice(arr []any, secretValues map[string]string) ([]any, []string) { result := make([]any, len(arr)) - keys := make([]string, 0) + keys := make([]string, 0, len(arr)) for i, v := range arr { var r any @@ -346,8 +346,8 @@ func processMapWithPathSecrets(m smap.Map, secretValues map[string]string) []str secrets := m.GetMap(FieldSecrets) delete(m, FieldSecrets) - secretReferences := make([]model.SecretReferencesItems, 0) outputSecrets := getSecretIDAndPath(secrets) + secretReferences := make([]model.SecretReferencesItems, 0, len(outputSecrets)) keys := make([]string, 0, len(outputSecrets)) for _, secret := range outputSecrets { @@ -401,6 +401,20 @@ func ProcessMapSecrets(m smap.Map, secretValues map[string]string) ([]string, er return keys, nil } +// ParseSecretReference returns the secret ID if value is a $co.elastic.secret{id} reference, otherwise returns ("", false). +func ParseSecretReference(value string) (string, bool) { + matches := secretRegex.FindStringSubmatch(value) + if len(matches) > 1 { + return matches[1], true + } + return "", false +} + +// MakeSecretReference formats a secret ID as a $co.elastic.secret{id} reference string. +func MakeSecretReference(secretID string) string { + return "$co.elastic.secret{" + secretID + "}" +} + // replaceStringRef replaces values matching a secret ref regex, e.g. $co.elastic.secret{} -> // and does this for multiple matches // returns the resulting string value, and if any replacements were made diff --git a/internal/pkg/secret/secret_test.go b/internal/pkg/secret/secret_test.go index 824073ad63..e4c8ecc809 100644 --- a/internal/pkg/secret/secret_test.go +++ b/internal/pkg/secret/secret_test.go @@ -317,3 +317,33 @@ func TestProcessOutputSecret(t *testing.T) { }) } } + +func TestParseSecretReference(t *testing.T) { + tests := []struct { + name string + input string + wantID string + wantParsed bool + }{ + {"valid reference", "$co.elastic.secret{abc123}", "abc123", true}, + {"plaintext api key", "keyid:keysecret", "", false}, + {"empty string", "", "", false}, + {"partial match", "prefix$co.elastic.secret{abc}", "abc", true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + id, ok := ParseSecretReference(tc.input) + assert.Equal(t, tc.wantParsed, ok) + assert.Equal(t, tc.wantID, id) + }) + } +} + +func TestMakeSecretReference(t *testing.T) { + ref := MakeSecretReference("abc123") + assert.Equal(t, "$co.elastic.secret{abc123}", ref) + + id, ok := ParseSecretReference(ref) + assert.True(t, ok) + assert.Equal(t, "abc123", id) +} diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go index 75fc78123b..f890b7b16b 100644 --- a/internal/pkg/testing/bulk.go +++ b/internal/pkg/testing/bulk.go @@ -42,12 +42,14 @@ func (m *MockBulk) Update(ctx context.Context, index, id string, body []byte, op func (m *MockBulk) Read(ctx context.Context, index, id string, opts ...bulk.Opt) ([]byte, error) { args := m.Called(ctx, index, id, opts) - return args.Get(0).([]byte), args.Error(1) + v, _ := args.Get(0).([]byte) + return v, args.Error(1) } func (m *MockBulk) ReadRaw(ctx context.Context, index, id string, opts ...bulk.Opt) (*bulk.MgetResponseItem, error) { args := m.Called(ctx, index, id, opts) - return args.Get(0).(*bulk.MgetResponseItem), args.Error(1) + v, _ := args.Get(0).(*bulk.MgetResponseItem) + return v, args.Error(1) } func (m *MockBulk) Delete(ctx context.Context, index, id string, opts ...bulk.Opt) error { @@ -57,32 +59,38 @@ func (m *MockBulk) Delete(ctx context.Context, index, id string, opts ...bulk.Op func (m *MockBulk) MCreate(ctx context.Context, ops []bulk.MultiOp, opts ...bulk.Opt) ([]bulk.BulkIndexerResponseItem, error) { args := m.Called(ctx, ops, opts) - return args.Get(0).([]bulk.BulkIndexerResponseItem), args.Error(1) + v, _ := args.Get(0).([]bulk.BulkIndexerResponseItem) + return v, args.Error(1) } func (m *MockBulk) MIndex(ctx context.Context, ops []bulk.MultiOp, opts ...bulk.Opt) ([]bulk.BulkIndexerResponseItem, error) { args := m.Called(ctx, ops, opts) - return args.Get(0).([]bulk.BulkIndexerResponseItem), args.Error(1) + v, _ := args.Get(0).([]bulk.BulkIndexerResponseItem) + return v, args.Error(1) } func (m *MockBulk) MUpdate(ctx context.Context, ops []bulk.MultiOp, opts ...bulk.Opt) ([]bulk.BulkIndexerResponseItem, error) { args := m.Called(ctx, ops, opts) - return args.Get(0).([]bulk.BulkIndexerResponseItem), args.Error(1) + v, _ := args.Get(0).([]bulk.BulkIndexerResponseItem) + return v, args.Error(1) } func (m *MockBulk) MDelete(ctx context.Context, ops []bulk.MultiOp, opts ...bulk.Opt) ([]bulk.BulkIndexerResponseItem, error) { args := m.Called(ctx, ops, opts) - return args.Get(0).([]bulk.BulkIndexerResponseItem), args.Error(1) + v, _ := args.Get(0).([]bulk.BulkIndexerResponseItem) + return v, args.Error(1) } func (m *MockBulk) Search(ctx context.Context, index string, body []byte, opts ...bulk.Opt) (*es.ResultT, error) { args := m.Called(ctx, index, body, opts) - return args.Get(0).(*es.ResultT), args.Error(1) + v, _ := args.Get(0).(*es.ResultT) + return v, args.Error(1) } func (m *MockBulk) Client() *elasticsearch.Client { args := m.Called() - return args.Get(0).(*elasticsearch.Client) + v, _ := args.Get(0).(*elasticsearch.Client) + return v } func (m *MockBulk) GetBulker(outputName string) bulk.Bulk { @@ -90,22 +98,27 @@ func (m *MockBulk) GetBulker(outputName string) bulk.Bulk { if args.Get(0) == nil { return nil } - return args.Get(0).(bulk.Bulk) + v, _ := args.Get(0).(bulk.Bulk) + return v } func (m *MockBulk) GetBulkerMap() map[string]bulk.Bulk { args := m.Called() - return args.Get(0).(map[string]bulk.Bulk) + v, _ := args.Get(0).(map[string]bulk.Bulk) + return v } func (m *MockBulk) CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, outputName string, outputMap map[string]map[string]any) (bulk.Bulk, bool, error) { args := m.Called(ctx, zlog, outputName, outputMap) - return args.Get(0).(bulk.Bulk), args.Get(1).(bool), nil + v, _ := args.Get(0).(bulk.Bulk) + ok, _ := args.Get(1).(bool) + return v, ok, nil } func (m *MockBulk) CancelFn() context.CancelFunc { args := m.Called() - return args.Get(0).(context.CancelFunc) + v, _ := args.Get(0).(context.CancelFunc) + return v } func (m *MockBulk) ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error) { @@ -116,19 +129,32 @@ func (m *MockBulk) ReadSecrets(ctx context.Context, secretIds []string) (map[str return result, nil } +func (m *MockBulk) WriteSecret(ctx context.Context, value string) (string, error) { + args := m.Called(ctx, value) + return args.String(0), args.Error(1) +} + +func (m *MockBulk) DeleteSecret(ctx context.Context, secretID string) error { + args := m.Called(ctx, secretID) + return args.Error(0) +} + func (m *MockBulk) APIKeyCreate(ctx context.Context, name, ttl string, roles []byte, meta any) (*bulk.APIKey, error) { args := m.Called(ctx, name, ttl, roles, meta) - return args.Get(0).(*bulk.APIKey), args.Error(1) + v, _ := args.Get(0).(*bulk.APIKey) + return v, args.Error(1) } func (m *MockBulk) APIKeyRead(ctx context.Context, id string, _ bool) (*bulk.APIKeyMetadata, error) { args := m.Called(ctx, id) - return args.Get(0).(*bulk.APIKeyMetadata), args.Error(1) + v, _ := args.Get(0).(*bulk.APIKeyMetadata) + return v, args.Error(1) } func (m *MockBulk) APIKeyAuth(ctx context.Context, key bulk.APIKey) (*bulk.SecurityInfo, error) { args := m.Called(ctx, key) - return args.Get(0).(*bulk.SecurityInfo), args.Error(1) + v, _ := args.Get(0).(*bulk.SecurityInfo) + return v, args.Error(1) } func (m *MockBulk) APIKeyInvalidate(ctx context.Context, ids ...string) error { diff --git a/model/schema.json b/model/schema.json index 26a73f52c6..924c996760 100644 --- a/model/schema.json +++ b/model/schema.json @@ -430,6 +430,10 @@ "output": { "description": "Output name where the API Key belongs", "type": "string" + }, + "secret_id": { + "description": "Fleet secret ID for the retired API key, if the key was stored in .fleet-secrets", + "type": "string" } } }