From 7e3e209de118c80e6c405351937d058746cd20c1 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 20 Jul 2026 14:17:57 -0700 Subject: [PATCH 01/12] Store output API key secrets in .fleet-secrets instead of .fleet-agents Fixes a privilege escalation (elastic/security#12225) where the kibana_system role's read access to .fleet-agents-7 could be used to extract plaintext output API key secrets and escalate to SIEM write access. Changes: - On new key generation, write the raw key material to .fleet-secrets via POST /_fleet/secret and store only a $co.elastic.secret{id} reference in outputs.{name}.api_key in .fleet-agents-7 - At agent checkin, resolve the reference via GET /_fleet/secret/{id} before injecting api_key into the policy payload delivered to the agent - On ACK (key rotation), delete the retired secret from .fleet-secrets via DELETE /_fleet/secret/{id}; the secret ID is carried in a new secret_id field on ToRetireAPIKeyIdsItems - Plaintext values already stored in .fleet-agents-7 continue to work transparently (backward compat); migration to the new format happens naturally on the next key rotation Requires elastic/elasticsearch#154498 (write_fleet_secrets privilege for the elastic/fleet-server service account) to be merged and deployed first. Co-Authored-By: Claude Sonnet 4.6 --- internal/pkg/api/handleAck.go | 12 ++++++ internal/pkg/bulk/engine.go | 10 +++++ internal/pkg/bulk/secret.go | 64 ++++++++++++++++++++++++++++ internal/pkg/model/schema.go | 7 ++- internal/pkg/policy/policy_output.go | 27 ++++++++++-- internal/pkg/secret/secret.go | 14 ++++++ internal/pkg/testing/bulk.go | 10 +++++ model/schema.json | 4 ++ 8 files changed, 140 insertions(+), 8 deletions(-) diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go index baceacad26..ce3718213e 100644 --- a/internal/pkg/api/handleAck.go +++ b/internal/pkg/api/handleAck.go @@ -507,6 +507,7 @@ func updateAPIKey(ctx context.Context, } } invalidateAPIKeys(ctx, zlog, bulk, toRetireAPIKeyIDs, apiKeyID) + deleteRetiredSecrets(ctx, zlog, bulk, toRetireAPIKeyIDs) } return nil @@ -744,3 +745,14 @@ func invalidateAPIKeys(ctx context.Context, zlog zerolog.Logger, bulk bulk.Bulk, } } } + +func deleteRetiredSecrets(ctx context.Context, zlog zerolog.Logger, bulk bulk.Bulk, toRetireAPIKeyIDs []model.ToRetireAPIKeyIdsItems) { + for _, k := range toRetireAPIKeyIDs { + if k.SecretID == "" { + continue + } + if err := bulk.DeleteSecret(ctx, k.SecretID); err != nil { + zlog.Warn().Err(err).Str("secret_id", k.SecretID).Msg("Failed to delete retired output API key secret") + } + } +} diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go index 9705c21065..776e16c75e 100644 --- a/internal/pkg/bulk/engine.go +++ b/internal/pkg/bulk/engine.go @@ -84,6 +84,8 @@ type Bulk interface { RemoteOutputConfigChanged(zlog zerolog.Logger, name string, newCfg map[string]any) bool ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error) + WriteSecret(ctx context.Context, value string) (string, error) + DeleteSecret(ctx context.Context, secretID string) error } const kModBulk = "bulk" @@ -335,6 +337,14 @@ func (b *Bulker) ReadSecrets(ctx context.Context, secretIds []string) (map[strin return result, nil } +func (b *Bulker) WriteSecret(ctx context.Context, value string) (string, error) { + return WriteSecret(ctx, b.Client(), value) +} + +func (b *Bulker) DeleteSecret(ctx context.Context, secretID string) error { + return DeleteSecret(ctx, b.Client(), secretID) +} + // Stop timer, but don't stall on channel. // API doesn't not seem to work as specified. func stopTimer(t *time.Timer) { diff --git a/internal/pkg/bulk/secret.go b/internal/pkg/bulk/secret.go index acebe92130..ca8b0408b8 100644 --- a/internal/pkg/bulk/secret.go +++ b/internal/pkg/bulk/secret.go @@ -5,6 +5,7 @@ package bulk import ( + "bytes" "context" "encoding/json" "net/http" @@ -46,6 +47,38 @@ func (c *ExtendedAPI) Read(ctx context.Context, secretID string) (*SecretRespons return &secretResp, nil } +// Write stores a secret value via the Fleet ES plugin secrets API and returns the assigned secret ID. +// POST /_fleet/secret +func (c *ExtendedAPI) Write(ctx context.Context, value string) (string, error) { + body, err := json.Marshal(struct { + Value string `json:"value"` + }{Value: value}) + if err != nil { + return "", err + } + + req, err := http.NewRequestWithContext(ctx, "POST", "/_fleet/secret", bytes.NewReader(body)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + res, err := c.Perform(req) + if err != nil { + return "", err + } + defer res.Body.Close() + + var resp struct { + ID string `json:"id"` + } + if err := json.NewDecoder(res.Body).Decode(&resp); err != nil { + return "", err + } + return resp.ID, nil +} + type SecretResponse struct { Value string } @@ -63,3 +96,34 @@ func ReadSecret(ctx context.Context, client *elasticsearch.Client, secretID stri } return (*res).Value, err } + +func WriteSecret(ctx context.Context, client *elasticsearch.Client, value string) (string, error) { + span, ctx := apm.StartSpan(ctx, "writeSecret", "elasticsearch") + defer span.End() + es := ExtendedClient{Client: client, Custom: &ExtendedAPI{client}} + return es.Custom.Write(ctx, value) +} + +// Delete removes a secret from the Fleet secrets store. +// DELETE /_fleet/secret/secretId +func (c *ExtendedAPI) Delete(ctx context.Context, secretID string) error { + req, err := http.NewRequestWithContext(ctx, "DELETE", "/_fleet/secret/"+secretID, nil) + if err != nil { + return err + } + req.Header.Set("Accept", "application/json") + + res, err := c.Perform(req) + if err != nil { + return err + } + res.Body.Close() + return nil +} + +func DeleteSecret(ctx context.Context, client *elasticsearch.Client, secretID string) error { + span, ctx := apm.StartSpan(ctx, "deleteSecret", "elasticsearch") + defer span.End() + es := ExtendedClient{Client: client, Custom: &ExtendedAPI{client}} + return es.Custom.Delete(ctx, secretID) +} diff --git a/internal/pkg/model/schema.go b/internal/pkg/model/schema.go index c0673a06b8..b47693e11b 100644 --- a/internal/pkg/model/schema.go +++ b/internal/pkg/model/schema.go @@ -1,7 +1,3 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License 2.0; -// you may not use this file except in compliance with the Elastic License 2.0. - // Code generated by schema-generate. DO NOT EDIT. package model @@ -606,6 +602,9 @@ type ToRetireAPIKeyIdsItems struct { // Date/time the API key was retired RetiredAt string `json:"retired_at,omitempty"` + + // Fleet secret ID for the retired API key, if the key was stored in .fleet-secrets + SecretID string `json:"secret_id,omitempty"` } // UnitsItems diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index 57e8f8f183..2bc02a9849 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -22,6 +22,7 @@ import ( "github.com/elastic/fleet-server/v7/internal/pkg/dl" "github.com/elastic/fleet-server/v7/internal/pkg/logger/ecs" "github.com/elastic/fleet-server/v7/internal/pkg/model" + "github.com/elastic/fleet-server/v7/internal/pkg/secret" "github.com/elastic/fleet-server/v7/internal/pkg/smap" ) @@ -307,8 +308,14 @@ func (p *Output) prepareElasticsearch( Str(ecs.DefaultOutputAPIKeyID, outputAPIKey.ID). Msg("Updating agent record to pick up default output key.") + secretID, err := bulker.WriteSecret(ctx, outputAPIKey.Agent()) + if err != nil { + return fmt.Errorf("failed writing output API key secret: %w", err) + } + apiKeyRef := secret.MakeSecretReference(secretID) + fields := map[string]any{ - dl.FieldPolicyOutputAPIKey: outputAPIKey.Agent(), + dl.FieldPolicyOutputAPIKey: apiKeyRef, dl.FieldPolicyOutputAPIKeyID: outputAPIKey.ID, dl.FieldPolicyOutputPermissionsHash: p.Role.Sha2, } @@ -317,11 +324,15 @@ func (p *Output) prepareElasticsearch( fields[dl.FiledType] = OutputTypeElasticsearch } if output.APIKeyID != "" { - fields[dl.FieldPolicyOutputToRetireAPIKeyIDs] = model.ToRetireAPIKeyIdsItems{ + retiring := model.ToRetireAPIKeyIdsItems{ ID: output.APIKeyID, RetiredAt: time.Now().UTC().Format(time.RFC3339), Output: p.Name, } + if secretID, ok := secret.ParseSecretReference(output.APIKey); ok { + retiring.SecretID = secretID + } + fields[dl.FieldPolicyOutputToRetireAPIKeyIDs] = retiring } // Using painless script to append the old keys to the history @@ -340,7 +351,7 @@ func (p *Output) prepareElasticsearch( // data is correct and in sync with ES, so it can be safely used after // this method returns. output.Type = OutputTypeElasticsearch - output.APIKey = outputAPIKey.Agent() + output.APIKey = apiKeyRef output.APIKeyID = outputAPIKey.ID output.PermissionsHash = p.Role.Sha2 // for the sake of consistency } @@ -362,7 +373,15 @@ func (p *Output) prepareElasticsearch( // in place to reduce number of agent policy allocation when sending the updated // agent policy to multiple agents. // See: https://github.com/elastic/fleet-server/issues/1301 - outputMap[p.Name]["api_key"] = output.APIKey + apiKey := output.APIKey + if secretID, ok := secret.ParseSecretReference(apiKey); ok { + resolved, err := bulker.ReadSecrets(ctx, []string{secretID}) + if err != nil { + return fmt.Errorf("failed resolving output API key secret: %w", err) + } + apiKey = resolved[secretID] + } + outputMap[p.Name]["api_key"] = apiKey return nil } diff --git a/internal/pkg/secret/secret.go b/internal/pkg/secret/secret.go index 08ccefa237..8eac140b77 100644 --- a/internal/pkg/secret/secret.go +++ b/internal/pkg/secret/secret.go @@ -401,6 +401,20 @@ func ProcessMapSecrets(m smap.Map, secretValues map[string]string) ([]string, er return keys, nil } +// ParseSecretReference returns the secret ID if value is a $co.elastic.secret{id} reference, otherwise returns ("", false). +func ParseSecretReference(value string) (string, bool) { + matches := secretRegex.FindStringSubmatch(value) + if len(matches) > 1 { + return matches[1], true + } + return "", false +} + +// MakeSecretReference formats a secret ID as a $co.elastic.secret{id} reference string. +func MakeSecretReference(secretID string) string { + return "$co.elastic.secret{" + secretID + "}" +} + // replaceStringRef replaces values matching a secret ref regex, e.g. $co.elastic.secret{} -> // and does this for multiple matches // returns the resulting string value, and if any replacements were made diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go index 75fc78123b..cf67eec813 100644 --- a/internal/pkg/testing/bulk.go +++ b/internal/pkg/testing/bulk.go @@ -116,6 +116,16 @@ func (m *MockBulk) ReadSecrets(ctx context.Context, secretIds []string) (map[str return result, nil } +func (m *MockBulk) WriteSecret(ctx context.Context, value string) (string, error) { + args := m.Called(ctx, value) + return args.String(0), args.Error(1) +} + +func (m *MockBulk) DeleteSecret(ctx context.Context, secretID string) error { + args := m.Called(ctx, secretID) + return args.Error(0) +} + func (m *MockBulk) APIKeyCreate(ctx context.Context, name, ttl string, roles []byte, meta any) (*bulk.APIKey, error) { args := m.Called(ctx, name, ttl, roles, meta) return args.Get(0).(*bulk.APIKey), args.Error(1) diff --git a/model/schema.json b/model/schema.json index 26a73f52c6..924c996760 100644 --- a/model/schema.json +++ b/model/schema.json @@ -430,6 +430,10 @@ "output": { "description": "Output name where the API Key belongs", "type": "string" + }, + "secret_id": { + "description": "Fleet secret ID for the retired API key, if the key was stored in .fleet-secrets", + "type": "string" } } } From 6aa6fa4e614d00fc2b89989b4f72606bc6ea6b28 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 20 Jul 2026 14:20:29 -0700 Subject: [PATCH 02/12] Add changelog fragment for output API key secret storage change Co-Authored-By: Claude Sonnet 4.6 --- ...1784582418-secure-output-api-key-storage.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 changelog/fragments/1784582418-secure-output-api-key-storage.yaml diff --git a/changelog/fragments/1784582418-secure-output-api-key-storage.yaml b/changelog/fragments/1784582418-secure-output-api-key-storage.yaml new file mode 100644 index 0000000000..7d024f4bde --- /dev/null +++ b/changelog/fragments/1784582418-secure-output-api-key-storage.yaml @@ -0,0 +1,16 @@ +kind: security + +summary: Store output API key secrets in .fleet-secrets instead of .fleet-agents + +description: | + Fleet Server now writes output API key material to .fleet-secrets via the Fleet + secrets API (POST /_fleet/secret) and stores only a $co.elastic.secret{id} reference + in .fleet-agents-7. Previously, the raw id:secret string was stored in plaintext in + outputs.{name}.api_key, which the kibana_system role could read and use to escalate + privileges to SIEM data stream write access. Retired secrets are deleted from + .fleet-secrets on ACK. Existing plaintext values continue to work transparently + until the next key rotation. + +component: fleet-server + +issue: https://github.com/elastic/security/issues/12225 From 6a0a124f7fdc5f50a993903aeee26466fe569232 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 20 Jul 2026 14:24:16 -0700 Subject: [PATCH 03/12] Add tests for secret reference helpers and output key secret storage Co-Authored-By: Claude Sonnet 4.6 --- internal/pkg/api/handleAck_test.go | 44 +++++++++++++++++++ internal/pkg/policy/policy_output_test.go | 52 ++++++++++++++++++++--- internal/pkg/secret/secret_test.go | 30 +++++++++++++ 3 files changed, 120 insertions(+), 6 deletions(-) diff --git a/internal/pkg/api/handleAck_test.go b/internal/pkg/api/handleAck_test.go index 72434570ba..6ad8fe5408 100644 --- a/internal/pkg/api/handleAck_test.go +++ b/internal/pkg/api/handleAck_test.go @@ -834,3 +834,47 @@ func TestValidateAckRequest(t *testing.T) { }) } } + +func TestDeleteRetiredSecrets(t *testing.T) { + logger := testlog.SetLogger(t) + _ = logger + + t.Run("deletes secrets for items with a SecretID", func(t *testing.T) { + bulker := ftesting.NewMockBulk() + bulker.On("DeleteSecret", mock.Anything, "secret-1").Return(nil).Once() + bulker.On("DeleteSecret", mock.Anything, "secret-2").Return(nil).Once() + + items := []model.ToRetireAPIKeyIdsItems{ + {ID: "key-1", SecretID: "secret-1"}, + {ID: "key-2", SecretID: "secret-2"}, + {ID: "key-3"}, // no SecretID — plaintext key, no cleanup needed + } + + deleteRetiredSecrets(context.Background(), testlog.SetLogger(t), bulker, items) + bulker.AssertExpectations(t) + bulker.AssertNotCalled(t, "DeleteSecret", mock.Anything, "") + }) + + t.Run("logs warning on delete failure but does not return error", func(t *testing.T) { + bulker := ftesting.NewMockBulk() + bulker.On("DeleteSecret", mock.Anything, "secret-bad").Return(errors.New("delete failed")).Once() + + items := []model.ToRetireAPIKeyIdsItems{ + {ID: "key-1", SecretID: "secret-bad"}, + } + + // Should not panic or propagate the error. + deleteRetiredSecrets(context.Background(), testlog.SetLogger(t), bulker, items) + bulker.AssertExpectations(t) + }) + + t.Run("no-op when all items lack a SecretID", func(t *testing.T) { + bulker := ftesting.NewMockBulk() + items := []model.ToRetireAPIKeyIdsItems{ + {ID: "key-1"}, + {ID: "key-2"}, + } + deleteRetiredSecrets(context.Background(), testlog.SetLogger(t), bulker, items) + bulker.AssertNotCalled(t, "DeleteSecret", mock.Anything, mock.Anything) + }) +} diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go index b344ffcced..f6a3bf192f 100644 --- a/internal/pkg/policy/policy_output_test.go +++ b/internal/pkg/policy/policy_output_test.go @@ -259,6 +259,8 @@ func TestPolicyOutputESPrepare(t *testing.T) { bulker.On("APIKeyCreate", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return(&apiKey, nil).Once() + const secretID = "test-secret-id" + bulker.On("WriteSecret", mock.Anything, apiKey.Agent()).Return(secretID, nil).Once() output := Output{ Type: OutputTypeElasticsearch, @@ -281,10 +283,12 @@ func TestPolicyOutputESPrepare(t *testing.T) { key, ok := policyMap[output.Name]["api_key"].(string) gotOutput := testAgent.Outputs[output.Name] - require.True(t, ok, "unable to case api key") - assert.Equal(t, apiKey.Agent(), key) + // The policy map receives the resolved secret value, not the reference. + require.True(t, ok, "unable to cast api key") + assert.Equal(t, secretID+"_value", key) - assert.Equal(t, apiKey.Agent(), gotOutput.APIKey) + // The agent document stores the secret reference, not the plaintext key. + assert.Equal(t, "$co.elastic.secret{"+secretID+"}", gotOutput.APIKey) assert.Equal(t, apiKey.ID, gotOutput.APIKeyID) assert.Equal(t, output.Role.Sha2, gotOutput.PermissionsHash) assert.Equal(t, output.Type, gotOutput.Type) @@ -298,6 +302,40 @@ func TestPolicyOutputESPrepare(t *testing.T) { bulker.AssertExpectations(t) }) + + t.Run("Existing plaintext key is delivered without modification", func(t *testing.T) { + logger := testlog.SetLogger(t) + bulker := ftesting.NewMockBulk() + + apiKey := bulk.APIKey{ID: "existing-id", Key: "existing-key"} + hashPerm := "existing-hash" + output := Output{ + Type: OutputTypeElasticsearch, + Name: "test output", + Role: &RoleT{Sha2: hashPerm, Raw: TestPayload}, + } + policyMap := map[string]map[string]any{"test output": {}} + testAgent := &model.Agent{ + Outputs: map[string]*model.PolicyOutput{ + output.Name: { + APIKey: apiKey.Agent(), + APIKeyID: apiKey.ID, + PermissionsHash: hashPerm, + Type: OutputTypeElasticsearch, + }, + }, + } + + err := output.Prepare(context.Background(), logger, bulker, testAgent, policyMap) + require.NoError(t, err) + + // Plaintext key is passed through directly — WriteSecret is not called. + key, ok := policyMap[output.Name]["api_key"].(string) + require.True(t, ok) + assert.Equal(t, apiKey.Agent(), key) + bulker.AssertNotCalled(t, "WriteSecret", mock.Anything, mock.Anything) + bulker.AssertExpectations(t) + }) } func TestPolicyRemoteESOutputPrepareNoRole(t *testing.T) { @@ -474,6 +512,8 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) { } return doc.Message == "" && doc.State == client.UnitStateHealthy.String() }), mock.Anything).Return("", nil) + const secretID = "test-secret-id" + bulker.On("WriteSecret", mock.Anything, apiKey.Agent()).Return(secretID, nil).Once() output := Output{ Type: OutputTypeRemoteElasticsearch, @@ -499,10 +539,10 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) { key, ok := policyMap[output.Name]["api_key"].(string) gotOutput := testAgent.Outputs[output.Name] - require.True(t, ok, "unable to case api key") - assert.Equal(t, apiKey.Agent(), key) + require.True(t, ok, "unable to cast api key") + assert.Equal(t, secretID+"_value", key) - assert.Equal(t, apiKey.Agent(), gotOutput.APIKey) + assert.Equal(t, "$co.elastic.secret{"+secretID+"}", gotOutput.APIKey) assert.Equal(t, apiKey.ID, gotOutput.APIKeyID) assert.Equal(t, output.Role.Sha2, gotOutput.PermissionsHash) assert.Empty(t, gotOutput.ToRetireAPIKeyIds) diff --git a/internal/pkg/secret/secret_test.go b/internal/pkg/secret/secret_test.go index 824073ad63..e4c8ecc809 100644 --- a/internal/pkg/secret/secret_test.go +++ b/internal/pkg/secret/secret_test.go @@ -317,3 +317,33 @@ func TestProcessOutputSecret(t *testing.T) { }) } } + +func TestParseSecretReference(t *testing.T) { + tests := []struct { + name string + input string + wantID string + wantParsed bool + }{ + {"valid reference", "$co.elastic.secret{abc123}", "abc123", true}, + {"plaintext api key", "keyid:keysecret", "", false}, + {"empty string", "", "", false}, + {"partial match", "prefix$co.elastic.secret{abc}", "abc", true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + id, ok := ParseSecretReference(tc.input) + assert.Equal(t, tc.wantParsed, ok) + assert.Equal(t, tc.wantID, id) + }) + } +} + +func TestMakeSecretReference(t *testing.T) { + ref := MakeSecretReference("abc123") + assert.Equal(t, "$co.elastic.secret{abc123}", ref) + + id, ok := ParseSecretReference(ref) + assert.True(t, ok) + assert.Equal(t, "abc123", id) +} From 814ba63f0c2aff32066dda1afbdc7ceb2d37e02c Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 20 Jul 2026 14:28:05 -0700 Subject: [PATCH 04/12] Remove security issue reference from changelog fragment Co-Authored-By: Claude Sonnet 4.6 --- .../fragments/1784582418-secure-output-api-key-storage.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/changelog/fragments/1784582418-secure-output-api-key-storage.yaml b/changelog/fragments/1784582418-secure-output-api-key-storage.yaml index 7d024f4bde..85ae99a6cc 100644 --- a/changelog/fragments/1784582418-secure-output-api-key-storage.yaml +++ b/changelog/fragments/1784582418-secure-output-api-key-storage.yaml @@ -12,5 +12,3 @@ description: | until the next key rotation. component: fleet-server - -issue: https://github.com/elastic/security/issues/12225 From c9dcd89cc25f95c22cf252e7ac558dfcd6296bd5 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 20 Jul 2026 14:37:32 -0700 Subject: [PATCH 05/12] Restore copyright header in generated schema.go The go generate run that added SecretID to ToRetireAPIKeyIdsItems stripped the copyright header because schema-generate does not preserve it. Re-add the ELv2 header. Co-Authored-By: Claude Sonnet 4.6 --- internal/pkg/model/schema.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/pkg/model/schema.go b/internal/pkg/model/schema.go index b47693e11b..2aa9d9041f 100644 --- a/internal/pkg/model/schema.go +++ b/internal/pkg/model/schema.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + // Code generated by schema-generate. DO NOT EDIT. package model From 7d181728034f8fb9517cc50ea8a96a6ee469cdde Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 22 Jul 2026 11:37:39 -0700 Subject: [PATCH 06/12] Populate SecretID when retiring a removed output's API key When an output is removed from the policy, the retired API key entry was missing SecretID, so deleteRetiredSecrets would skip cleaning up the corresponding .fleet-secrets entry. Co-Authored-By: Claude Sonnet 4.6 --- internal/pkg/policy/policy_output.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index 2bc02a9849..0f9d7aab06 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -130,11 +130,15 @@ func (p *Output) prepareElasticsearch( } if !found { zlog.Info().Str(ecs.APIKeyID, agentOutput.APIKeyID).Str(ecs.PolicyOutputName, agentOutputName).Msg("Output removed, will retire API key") - toRetireAPIKeys = &model.ToRetireAPIKeyIdsItems{ + retiring := model.ToRetireAPIKeyIdsItems{ ID: agentOutput.APIKeyID, RetiredAt: time.Now().UTC().Format(time.RFC3339), Output: agentOutputName, } + if secretID, ok := secret.ParseSecretReference(agentOutput.APIKey); ok { + retiring.SecretID = secretID + } + toRetireAPIKeys = &retiring removedOutputName = agentOutputName break } From 6fdc164c96cbf53b16f58f351ea05634229ddd48 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 23 Jul 2026 06:30:31 -0700 Subject: [PATCH 07/12] fix: check HTTP status code in fleet secrets API calls c.Perform only errors on network failures, not 4xx/5xx responses. Without this check, a 403 (missing write_fleet_secrets privilege) or 500 would cause Write() to return an empty secret ID, producing a broken $co.elastic.secret{} reference silently stored in the agent doc. Co-Authored-By: Claude Sonnet 4.6 --- internal/pkg/bulk/secret.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/pkg/bulk/secret.go b/internal/pkg/bulk/secret.go index ca8b0408b8..9448493149 100644 --- a/internal/pkg/bulk/secret.go +++ b/internal/pkg/bulk/secret.go @@ -8,6 +8,8 @@ import ( "bytes" "context" "encoding/json" + "fmt" + "io" "net/http" "github.com/elastic/go-elasticsearch/v8" @@ -38,6 +40,10 @@ func (c *ExtendedAPI) Read(ctx context.Context, secretID string) (*SecretRespons return nil, err } defer res.Body.Close() + if res.StatusCode >= 400 { + body, _ := io.ReadAll(res.Body) + return nil, fmt.Errorf("unexpected status %d from fleet secret read: %s", res.StatusCode, body) + } var secretResp SecretResponse err = json.NewDecoder(res.Body).Decode(&secretResp) @@ -69,6 +75,10 @@ func (c *ExtendedAPI) Write(ctx context.Context, value string) (string, error) { return "", err } defer res.Body.Close() + if res.StatusCode >= 400 { + body, _ := io.ReadAll(res.Body) + return "", fmt.Errorf("unexpected status %d from fleet secret write: %s", res.StatusCode, body) + } var resp struct { ID string `json:"id"` @@ -117,7 +127,11 @@ func (c *ExtendedAPI) Delete(ctx context.Context, secretID string) error { if err != nil { return err } - res.Body.Close() + defer res.Body.Close() + if res.StatusCode >= 400 { + body, _ := io.ReadAll(res.Body) + return fmt.Errorf("unexpected status %d from fleet secret delete: %s", res.StatusCode, body) + } return nil } From af26f66761d5382547d025c367fb079f6a89e66a Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 23 Jul 2026 06:31:40 -0700 Subject: [PATCH 08/12] fix: error if output API key secret missing from resolved map ReadSecrets returns no error when a secret ID is absent from the response (e.g. deleted between write and read). The zero-value map lookup would silently inject an empty api_key into the agent policy, causing the agent to fail connecting to Elasticsearch. Co-Authored-By: Claude Sonnet 4.6 --- internal/pkg/policy/policy_output.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index 0f9d7aab06..578673b17b 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -383,7 +383,11 @@ func (p *Output) prepareElasticsearch( if err != nil { return fmt.Errorf("failed resolving output API key secret: %w", err) } - apiKey = resolved[secretID] + val, ok := resolved[secretID] + if !ok || val == "" { + return fmt.Errorf("output API key secret %q not found", secretID) + } + apiKey = val } outputMap[p.Name]["api_key"] = apiKey return nil From 0af4d4162dac0a0ad80988f4289a5ee5bcd7f642 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 23 Jul 2026 11:10:32 -0700 Subject: [PATCH 09/12] fix: delete output API key secrets on agent unenroll handleUnenroll was invalidating all output API keys but never deleting their corresponding secrets from .fleet-secrets. Add a deleteRetiredSecrets call that covers both retired keys (SecretID already set in ToRetireAPIKeyIds items) and the current active output key (secret ID parsed from the $co.elastic.secret{} reference in output.APIKey). Co-Authored-By: Claude Sonnet 4.6 --- internal/pkg/api/handleAck.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go index ce3718213e..b6973c57c9 100644 --- a/internal/pkg/api/handleAck.go +++ b/internal/pkg/api/handleAck.go @@ -28,6 +28,7 @@ import ( "github.com/elastic/fleet-server/v7/internal/pkg/logger/ecs" "github.com/elastic/fleet-server/v7/internal/pkg/model" "github.com/elastic/fleet-server/v7/internal/pkg/policy" + "github.com/elastic/fleet-server/v7/internal/pkg/secret" "github.com/elastic/fleet-server/v7/internal/pkg/smap" ) @@ -586,6 +587,17 @@ func (ack *AckT) handleUnenroll(ctx context.Context, zlog zerolog.Logger, agent zlog.Info().Any("fleet.policy.apiKeyIDsToRetire", apiKeys).Msg("handleUnenroll invalidate API keys") ack.invalidateAPIKeys(ctx, zlog, apiKeys, "") + // Delete secrets for all output API keys: retired ones (carried in apiKeys via SecretID) + // and the current active ones (secret ID embedded in output.APIKey as a reference string). + allSecretItems := make([]model.ToRetireAPIKeyIdsItems, 0, len(apiKeys)+len(agent.Outputs)) + allSecretItems = append(allSecretItems, apiKeys...) + for _, output := range agent.Outputs { + if secretID, ok := secret.ParseSecretReference(output.APIKey); ok { + allSecretItems = append(allSecretItems, model.ToRetireAPIKeyIdsItems{SecretID: secretID}) + } + } + deleteRetiredSecrets(ctx, zlog, ack.bulk, allSecretItems) + now := time.Now().UTC().Format(time.RFC3339) doc := bulk.UpdateFields{ dl.FieldActive: false, From 59e285c823d8a2fdbe0f8ba670b69921e94e6333 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 28 Jul 2026 21:47:00 -0700 Subject: [PATCH 10/12] fix: use two-value type assertions in MockBulk to satisfy errcheck The golangci-lint config has check-type-assertions: true and --whole-files is passed, so touching bulk.go surfaces all pre-existing single-value type assertions. Convert them to v, _ := args.Get(0).(T) throughout. Co-Authored-By: Claude Sonnet 4.6 --- internal/pkg/testing/bulk.go | 46 ++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go index cf67eec813..f890b7b16b 100644 --- a/internal/pkg/testing/bulk.go +++ b/internal/pkg/testing/bulk.go @@ -42,12 +42,14 @@ func (m *MockBulk) Update(ctx context.Context, index, id string, body []byte, op func (m *MockBulk) Read(ctx context.Context, index, id string, opts ...bulk.Opt) ([]byte, error) { args := m.Called(ctx, index, id, opts) - return args.Get(0).([]byte), args.Error(1) + v, _ := args.Get(0).([]byte) + return v, args.Error(1) } func (m *MockBulk) ReadRaw(ctx context.Context, index, id string, opts ...bulk.Opt) (*bulk.MgetResponseItem, error) { args := m.Called(ctx, index, id, opts) - return args.Get(0).(*bulk.MgetResponseItem), args.Error(1) + v, _ := args.Get(0).(*bulk.MgetResponseItem) + return v, args.Error(1) } func (m *MockBulk) Delete(ctx context.Context, index, id string, opts ...bulk.Opt) error { @@ -57,32 +59,38 @@ func (m *MockBulk) Delete(ctx context.Context, index, id string, opts ...bulk.Op func (m *MockBulk) MCreate(ctx context.Context, ops []bulk.MultiOp, opts ...bulk.Opt) ([]bulk.BulkIndexerResponseItem, error) { args := m.Called(ctx, ops, opts) - return args.Get(0).([]bulk.BulkIndexerResponseItem), args.Error(1) + v, _ := args.Get(0).([]bulk.BulkIndexerResponseItem) + return v, args.Error(1) } func (m *MockBulk) MIndex(ctx context.Context, ops []bulk.MultiOp, opts ...bulk.Opt) ([]bulk.BulkIndexerResponseItem, error) { args := m.Called(ctx, ops, opts) - return args.Get(0).([]bulk.BulkIndexerResponseItem), args.Error(1) + v, _ := args.Get(0).([]bulk.BulkIndexerResponseItem) + return v, args.Error(1) } func (m *MockBulk) MUpdate(ctx context.Context, ops []bulk.MultiOp, opts ...bulk.Opt) ([]bulk.BulkIndexerResponseItem, error) { args := m.Called(ctx, ops, opts) - return args.Get(0).([]bulk.BulkIndexerResponseItem), args.Error(1) + v, _ := args.Get(0).([]bulk.BulkIndexerResponseItem) + return v, args.Error(1) } func (m *MockBulk) MDelete(ctx context.Context, ops []bulk.MultiOp, opts ...bulk.Opt) ([]bulk.BulkIndexerResponseItem, error) { args := m.Called(ctx, ops, opts) - return args.Get(0).([]bulk.BulkIndexerResponseItem), args.Error(1) + v, _ := args.Get(0).([]bulk.BulkIndexerResponseItem) + return v, args.Error(1) } func (m *MockBulk) Search(ctx context.Context, index string, body []byte, opts ...bulk.Opt) (*es.ResultT, error) { args := m.Called(ctx, index, body, opts) - return args.Get(0).(*es.ResultT), args.Error(1) + v, _ := args.Get(0).(*es.ResultT) + return v, args.Error(1) } func (m *MockBulk) Client() *elasticsearch.Client { args := m.Called() - return args.Get(0).(*elasticsearch.Client) + v, _ := args.Get(0).(*elasticsearch.Client) + return v } func (m *MockBulk) GetBulker(outputName string) bulk.Bulk { @@ -90,22 +98,27 @@ func (m *MockBulk) GetBulker(outputName string) bulk.Bulk { if args.Get(0) == nil { return nil } - return args.Get(0).(bulk.Bulk) + v, _ := args.Get(0).(bulk.Bulk) + return v } func (m *MockBulk) GetBulkerMap() map[string]bulk.Bulk { args := m.Called() - return args.Get(0).(map[string]bulk.Bulk) + v, _ := args.Get(0).(map[string]bulk.Bulk) + return v } func (m *MockBulk) CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, outputName string, outputMap map[string]map[string]any) (bulk.Bulk, bool, error) { args := m.Called(ctx, zlog, outputName, outputMap) - return args.Get(0).(bulk.Bulk), args.Get(1).(bool), nil + v, _ := args.Get(0).(bulk.Bulk) + ok, _ := args.Get(1).(bool) + return v, ok, nil } func (m *MockBulk) CancelFn() context.CancelFunc { args := m.Called() - return args.Get(0).(context.CancelFunc) + v, _ := args.Get(0).(context.CancelFunc) + return v } func (m *MockBulk) ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error) { @@ -128,17 +141,20 @@ func (m *MockBulk) DeleteSecret(ctx context.Context, secretID string) error { func (m *MockBulk) APIKeyCreate(ctx context.Context, name, ttl string, roles []byte, meta any) (*bulk.APIKey, error) { args := m.Called(ctx, name, ttl, roles, meta) - return args.Get(0).(*bulk.APIKey), args.Error(1) + v, _ := args.Get(0).(*bulk.APIKey) + return v, args.Error(1) } func (m *MockBulk) APIKeyRead(ctx context.Context, id string, _ bool) (*bulk.APIKeyMetadata, error) { args := m.Called(ctx, id) - return args.Get(0).(*bulk.APIKeyMetadata), args.Error(1) + v, _ := args.Get(0).(*bulk.APIKeyMetadata) + return v, args.Error(1) } func (m *MockBulk) APIKeyAuth(ctx context.Context, key bulk.APIKey) (*bulk.SecurityInfo, error) { args := m.Called(ctx, key) - return args.Get(0).(*bulk.SecurityInfo), args.Error(1) + v, _ := args.Get(0).(*bulk.SecurityInfo) + return v, args.Error(1) } func (m *MockBulk) APIKeyInvalidate(ctx context.Context, ids ...string) error { From 527895762b6f2295fb2d7ee1085a63e1f5e84e3f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 28 Jul 2026 22:51:49 -0700 Subject: [PATCH 11/12] fix: address prealloc and staticcheck lint issues in touched files --whole-files causes golangci-lint to flag pre-existing issues in any file touched by the PR. Fix prealloc warnings in secret.go by adding capacity hints to slice literals, and replace WriteString(fmt.Sprintf) with fmt.Fprintf in policy_output.go as suggested by staticcheck QF1012. Co-Authored-By: Claude Sonnet 4.6 --- internal/pkg/policy/policy_output.go | 12 ++++++------ internal/pkg/secret/secret.go | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index 578673b17b..a00a5d163a 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -503,29 +503,29 @@ func renderUpdatePainlessScript(outputName string, fields map[string]any) ([]byt var source strings.Builder // prepare agent.elasticsearch_outputs[OUTPUT_NAME] - source.WriteString(fmt.Sprintf(` + fmt.Fprintf(&source, ` if (ctx._source['outputs']==null) {ctx._source['outputs']=new HashMap();} if (ctx._source['outputs']['%s']==null) {ctx._source['outputs']['%s']=new HashMap();} -`, outputName, outputName)) +`, outputName, outputName) for field := range fields { if field == dl.FieldPolicyOutputToRetireAPIKeyIDs { // dl.FieldPolicyOutputToRetireAPIKeyIDs is a special case. // It's an array that gets deleted when the keys are invalidated. // Thus, append the old API key ID, create the field if necessary. - source.WriteString(fmt.Sprintf(` + fmt.Fprintf(&source, ` if (ctx._source['outputs']['%s'].%s==null) {ctx._source['outputs']['%s'].%s=new ArrayList();} if (!ctx._source['outputs']['%s'].%s.contains(params.%s)) {ctx._source['outputs']['%s'].%s.add(params.%s);} -`, outputName, field, outputName, field, outputName, field, field, outputName, field, field)) +`, outputName, field, outputName, field, outputName, field, field, outputName, field, field) } else { // Update the other fields - source.WriteString(fmt.Sprintf(` + fmt.Fprintf(&source, ` ctx._source['outputs']['%s'].%s=params.%s;`, - outputName, field, field)) + outputName, field, field) } } diff --git a/internal/pkg/secret/secret.go b/internal/pkg/secret/secret.go index 8eac140b77..091ea1e9be 100644 --- a/internal/pkg/secret/secret.go +++ b/internal/pkg/secret/secret.go @@ -76,7 +76,7 @@ func ProcessInputsSecrets(data *model.PolicyData, secretValues map[string]string // the values of secret refs in inputs and input streams properties using the old format // for specifying secrets: : $co.elastic.secret{} func processInputsWithInlineSecrets(data *model.PolicyData, secretValues map[string]string) ([]map[string]any, []string) { - result := make([]map[string]any, 0) + result := make([]map[string]any, 0, len(data.Inputs)) keys := make([]string, 0) for i, input := range data.Inputs { replacedInput, ks := replaceInlineSecretRefsInMap(input, secretValues) @@ -92,7 +92,7 @@ func processInputsWithInlineSecrets(data *model.PolicyData, secretValues map[str // the values of secret refs in inputs and input streams properties using the new format // for specifying secrets: secrets...id: func processInputsWithPathSecrets(data *model.PolicyData, secretValues map[string]string) ([]map[string]any, []string) { - result := make([]map[string]any, 0) + result := make([]map[string]any, 0, len(data.Inputs)) keys := make([]string, 0) for i, inp := range data.Inputs { @@ -158,7 +158,7 @@ func replacePathSecretRefsInMap(m smap.Map, secretValues map[string]string) (map func replacePathSecretRefsInSlice(arr []any, secretValues map[string]string) ([]any, []string) { result := make([]any, len(arr)) - keys := make([]string, 0) + keys := make([]string, 0, len(arr)) for i, v := range arr { var r any @@ -346,8 +346,8 @@ func processMapWithPathSecrets(m smap.Map, secretValues map[string]string) []str secrets := m.GetMap(FieldSecrets) delete(m, FieldSecrets) - secretReferences := make([]model.SecretReferencesItems, 0) outputSecrets := getSecretIDAndPath(secrets) + secretReferences := make([]model.SecretReferencesItems, 0, len(outputSecrets)) keys := make([]string, 0, len(outputSecrets)) for _, secret := range outputSecrets { From 0818c0ebf69a2ff613e7f911dca95578b710ddf7 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 29 Jul 2026 09:37:14 -0700 Subject: [PATCH 12/12] fix: delete orphaned secret if .fleet-agents update fails after WriteSecret If WriteSecret succeeds but the subsequent bulker.Update to .fleet-agents fails, the written secret would previously be left in .fleet-secrets with no reference. Now a compensating DeleteSecret is issued on the update failure path; if that cleanup also fails it logs a warning and still returns the original update error. Co-Authored-By: Claude Sonnet 4.6 --- internal/pkg/policy/policy_output.go | 3 +++ internal/pkg/policy/policy_output_test.go | 27 +++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index a00a5d163a..81780fb43a 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -347,6 +347,9 @@ func (p *Output) prepareElasticsearch( if err = bulker.Update(ctx, dl.FleetAgents, agent.Id, body, bulk.WithRefresh(), bulk.WithRetryOnConflict(3)); err != nil { zlog.Error().Err(err).Msg("fail update agent record") + if delErr := bulker.DeleteSecret(ctx, secretID); delErr != nil { + zlog.Warn().Err(delErr).Str("secret_id", secretID).Msg("failed to delete orphaned output API key secret after agent update failure") + } return fmt.Errorf("fail update agent record: %w", err) } diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go index f6a3bf192f..6157538171 100644 --- a/internal/pkg/policy/policy_output_test.go +++ b/internal/pkg/policy/policy_output_test.go @@ -303,6 +303,33 @@ func TestPolicyOutputESPrepare(t *testing.T) { bulker.AssertExpectations(t) }) + t.Run("Secret is deleted when agent document update fails", func(t *testing.T) { + logger := testlog.SetLogger(t) + bulker := ftesting.NewMockBulk() + apiKey := bulk.APIKey{ID: "abc", Key: "new-key"} + bulker.On("APIKeyCreate", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(&apiKey, nil).Once() + const secretID = "test-secret-id" + bulker.On("WriteSecret", mock.Anything, apiKey.Agent()).Return(secretID, nil).Once() + bulker.On("Update", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(errors.New("ES update failed")).Once() + bulker.On("DeleteSecret", mock.Anything, secretID).Return(nil).Once() + + output := Output{ + Type: OutputTypeElasticsearch, + Name: "test output", + Role: &RoleT{Sha2: "new-hash", Raw: TestPayload}, + } + policyMap := map[string]map[string]any{"test output": {}} + testAgent := &model.Agent{Outputs: map[string]*model.PolicyOutput{}} + + err := output.Prepare(context.Background(), logger, bulker, testAgent, policyMap) + require.Error(t, err) + bulker.AssertExpectations(t) + }) + t.Run("Existing plaintext key is delivered without modification", func(t *testing.T) { logger := testlog.SetLogger(t) bulker := ftesting.NewMockBulk()