Skip to content

Commit 37656f8

Browse files
cristiangrecocristiangreco
authored
feat: Database_observability: expose exclude_schemas and exclude_databases settings (#5334)
### Brief description of Pull Request Correctly expose and wire setting to exclude schemas (mysql) and databases (postgres) from data collection. At the moment, this only applies to the `explain_plans` collectors. ### Pull Request Details <!-- Add a more detailed descripion of the Pull Request here, if needed. --> ### Issue(s) fixed by this Pull Request <!-- Uncomment the following line and fill in an issue number if you want a GitHub issue to be closed automatically when this PR gets merged. --> <!-- Fixes #issue_id --> ### Notes to the Reviewer <!-- Add any relevant notes for the reviewers and testers of this PR. --> ### PR Checklist <!-- Remove items that do not apply. For completed items, change [ ] to [x]. --> - [ ] Documentation added - [ ] Tests updated - [ ] Config converters updated
1 parent e6d2a12 commit 37656f8

File tree

7 files changed

+60
-58
lines changed

7 files changed

+60
-58
lines changed

‎docs/sources/reference/components/database_observability/database_observability.mysql.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ You can use the following arguments with `database_observability.mysql`:
3333
| `targets` | `list(map(string))` | List of targets to scrape. | | yes |
3434
| `disable_collectors` | `list(string)` | A list of collectors to disable from the default set. | | no |
3535
| `enable_collectors` | `list(string)` | A list of collectors to enable on top of the default set. | | no |
36+
| `exclude_schemas` | `list(string)` | A list of schemas to exclude from monitoring. | | no |
3637
| `allow_update_performance_schema_settings` | `boolean` | Whether to allow updates to `performance_schema` settings in any collector. Enable this in conjunction with other collector-specific settings where required. | `false` | no |
3738

3839
The following collectors are configurable:
@@ -133,7 +134,6 @@ The `azure` block supplies the identifying information for the database being mo
133134
| Name | Type | Description | Default | Required |
134135
| ------------------------------ | -------------- | ------------------------------------------------------------------------------- | ------- | -------- |
135136
| `collect_interval` | `duration` | How frequently to collect information from database. | `"1m"` | no |
136-
| `explain_plan_exclude_schemas` | `list(string)` | List of schemas to exclude from explain plan collection. | | no |
137137
| `initial_lookback` | `duration` | How far back to look for explain plan queries on the first collection interval. | `"24h"` | no |
138138
| `per_collect_ratio` | `float` | Ratio of explain plan queries to collect per collect interval. | `1.0` | no |
139139

‎docs/sources/reference/components/database_observability/database_observability.postgres.md‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ You can use the following arguments with `database_observability.postgres`:
3333
| `targets` | `list(map(string))` | List of targets to scrape. | | yes |
3434
| `disable_collectors` | `list(string)` | A list of collectors to disable from the default set. | | no |
3535
| `enable_collectors` | `list(string)` | A list of collectors to enable on top of the default set. | | no |
36+
| `exclude_databases` | `list(string)` | A list of databases to exclude from monitoring. | | no |
3637

3738
The following collectors are configurable:
3839

@@ -122,11 +123,10 @@ The `azure` block supplies the identifying information for the database being mo
122123

123124
### `explain_plans`
124125

125-
| Name | Type | Description | Default | Required |
126-
|--------------------------------|----------------|------------------------------------------------------|---------|----------|
127-
| `collect_interval` | `duration` | How frequently to collect information from database. | `"1m"` | no |
128-
| `per_collect_ratio` | `float64` | The ratio of queries to collect explain plans for. | `1.0` | no |
129-
| `explain_plan_exclude_schemas` | `list(string)` | Schemas to exclude from explain plans. | `[]` | no |
126+
| Name | Type | Description | Default | Required |
127+
|---------------------|----------------|------------------------------------------------------|---------|----------|
128+
| `collect_interval` | `duration` | How frequently to collect information from database. | `"1m"` | no |
129+
| `per_collect_ratio` | `float64` | The ratio of queries to collect explain plans for. | `1.0` | no |
130130

131131
### `health_check`
132132

‎internal/component/database_observability/mysql/collector/explain_plans.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -397,9 +397,9 @@ type ExplainPlansArguments struct {
397397
DB *sql.DB
398398
ScrapeInterval time.Duration
399399
PerScrapeRatio float64
400+
InitialLookback time.Time
400401
ExcludeSchemas []string
401402
EntryHandler loki.EntryHandler
402-
InitialLookback time.Time
403403
DBVersion string
404404

405405
Logger log.Logger
@@ -427,12 +427,12 @@ func NewExplainPlans(args ExplainPlansArguments) (*ExplainPlans, error) {
427427
dbConnection: args.DB,
428428
dbVersion: args.DBVersion,
429429
scrapeInterval: args.ScrapeInterval,
430+
perScrapeRatio: args.PerScrapeRatio,
431+
excludeSchemas: args.ExcludeSchemas,
432+
lastSeen: args.InitialLookback,
430433
queryCache: make(map[string]*queryInfo),
431434
queryDenylist: make(map[string]*queryInfo),
432-
excludeSchemas: args.ExcludeSchemas,
433-
perScrapeRatio: args.PerScrapeRatio,
434435
entryHandler: args.EntryHandler,
435-
lastSeen: args.InitialLookback,
436436
logger: log.With(args.Logger, "collector", ExplainPlansCollector),
437437
running: atomic.NewBool(false),
438438
}, nil

‎internal/component/database_observability/mysql/component.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ type Arguments struct {
6060
Targets []discovery.Target `alloy:"targets,attr"`
6161
EnableCollectors []string `alloy:"enable_collectors,attr,optional"`
6262
DisableCollectors []string `alloy:"disable_collectors,attr,optional"`
63+
ExcludeSchemas []string `alloy:"exclude_schemas,attr,optional"`
6364
AllowUpdatePerfSchemaSettings bool `alloy:"allow_update_performance_schema_settings,attr,optional"`
6465

6566
CloudProvider *CloudProvider `alloy:"cloud_provider,block,optional"`
@@ -110,10 +111,9 @@ type SetupActorsArguments struct {
110111
}
111112

112113
type ExplainPlansArguments struct {
113-
CollectInterval time.Duration `alloy:"collect_interval,attr,optional"`
114-
PerCollectRatio float64 `alloy:"per_collect_ratio,attr,optional"`
115-
InitialLookback time.Duration `alloy:"initial_lookback,attr,optional"`
116-
ExplainPlanExcludeSchemas []string `alloy:"explain_plan_exclude_schemas,attr,optional"`
114+
CollectInterval time.Duration `alloy:"collect_interval,attr,optional"`
115+
PerCollectRatio float64 `alloy:"per_collect_ratio,attr,optional"`
116+
InitialLookback time.Duration `alloy:"initial_lookback,attr,optional"`
117117
}
118118

119119
type LocksArguments struct {
@@ -133,6 +133,7 @@ type HealthCheckArguments struct {
133133
}
134134

135135
var DefaultArguments = Arguments{
136+
ExcludeSchemas: []string{},
136137
AllowUpdatePerfSchemaSettings: false,
137138

138139
QueryDetailsArguments: QueryDetailsArguments{
@@ -568,10 +569,11 @@ func (c *Component) startCollectors(serverID string, engineVersion string, parse
568569
DB: c.dbConnection,
569570
ScrapeInterval: c.args.ExplainPlansArguments.CollectInterval,
570571
PerScrapeRatio: c.args.ExplainPlansArguments.PerCollectRatio,
572+
ExcludeSchemas: c.args.ExcludeSchemas,
573+
InitialLookback: time.Now().Add(-c.args.ExplainPlansArguments.InitialLookback),
571574
Logger: c.opts.Logger,
572575
DBVersion: engineVersion,
573576
EntryHandler: entryHandler,
574-
InitialLookback: time.Now().Add(-c.args.ExplainPlansArguments.InitialLookback),
575577
})
576578
if err != nil {
577579
logStartError(collector.ExplainPlansCollector, "create", err)

‎internal/component/database_observability/postgres/collector/explain_plans.go‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -210,14 +210,13 @@ func newQueryInfo(datname, queryId, queryText string, calls int64, callsReset ti
210210
}
211211

212212
type ExplainPlansArguments struct {
213-
DB *sql.DB
214-
DSN string
215-
ScrapeInterval time.Duration
216-
PerScrapeRatio float64
217-
ExcludeSchemas []string
218-
EntryHandler loki.EntryHandler
219-
InitialLookback time.Time
220-
DBVersion string
213+
DB *sql.DB
214+
DSN string
215+
ScrapeInterval time.Duration
216+
PerScrapeRatio float64
217+
ExcludeDatabases []string
218+
EntryHandler loki.EntryHandler
219+
DBVersion string
221220

222221
Logger log.Logger
223222
}
@@ -231,7 +230,7 @@ type ExplainPlans struct {
231230
queryCache map[string]*queryInfo
232231
queryDenylist map[string]*queryInfo
233232
finishedQueryCache map[string]*queryInfo
234-
excludeSchemas []string
233+
excludeDatabases []string
235234
perScrapeRatio float64
236235
currentBatchSize int
237236
entryHandler loki.EntryHandler
@@ -247,11 +246,11 @@ func NewExplainPlan(args ExplainPlansArguments) (*ExplainPlans, error) {
247246
dbDSN: args.DSN,
248247
dbConnectionFactory: defaultDbConnectionFactory,
249248
scrapeInterval: args.ScrapeInterval,
249+
perScrapeRatio: args.PerScrapeRatio,
250+
excludeDatabases: args.ExcludeDatabases,
250251
queryCache: make(map[string]*queryInfo),
251252
queryDenylist: make(map[string]*queryInfo),
252253
finishedQueryCache: make(map[string]*queryInfo),
253-
excludeSchemas: args.ExcludeSchemas,
254-
perScrapeRatio: args.PerScrapeRatio,
255254
entryHandler: args.EntryHandler,
256255
logger: log.With(args.Logger, "collector", ExplainPlanCollector),
257256
running: atomic.NewBool(false),
@@ -380,7 +379,7 @@ func (c *ExplainPlans) populateQueryCache(ctx context.Context) error {
380379
return fmt.Errorf("failed to scan query for explain plan: %w", err)
381380
}
382381

383-
if slices.ContainsFunc(c.excludeSchemas, func(schema string) bool {
382+
if slices.ContainsFunc(c.excludeDatabases, func(schema string) bool {
384383
return strings.EqualFold(schema, datname)
385384
}) {
386385

‎internal/component/database_observability/postgres/collector/explain_plans_test.go‎

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2261,15 +2261,14 @@ func TestNewExplainPlan(t *testing.T) {
22612261
require.NoError(t, err)
22622262

22632263
args := ExplainPlansArguments{
2264-
DB: db,
2265-
DSN: "postgres://user:pass@localhost:5432/testdb",
2266-
ScrapeInterval: time.Minute,
2267-
PerScrapeRatio: 0.1,
2268-
ExcludeSchemas: []string{"information_schema", "pg_catalog"},
2269-
EntryHandler: entryHandler,
2270-
InitialLookback: time.Now().Add(-time.Hour),
2271-
DBVersion: pre17ver,
2272-
Logger: logger,
2264+
DB: db,
2265+
DSN: "postgres://user:pass@localhost:5432/testdb",
2266+
ScrapeInterval: time.Minute,
2267+
PerScrapeRatio: 0.1,
2268+
ExcludeDatabases: []string{"information_schema", "pg_catalog"},
2269+
EntryHandler: entryHandler,
2270+
DBVersion: pre17ver,
2271+
Logger: logger,
22732272
}
22742273

22752274
explainPlan, err := NewExplainPlan(args)
@@ -2281,7 +2280,7 @@ func TestNewExplainPlan(t *testing.T) {
22812280
assert.Equal(t, pre17semver, explainPlan.dbVersion)
22822281
assert.Equal(t, args.ScrapeInterval, explainPlan.scrapeInterval)
22832282
assert.Equal(t, args.PerScrapeRatio, explainPlan.perScrapeRatio)
2284-
assert.Equal(t, args.ExcludeSchemas, explainPlan.excludeSchemas)
2283+
assert.Equal(t, args.ExcludeDatabases, explainPlan.excludeDatabases)
22852284
assert.Equal(t, entryHandler, explainPlan.entryHandler)
22862285
assert.NotNil(t, explainPlan.queryCache)
22872286
assert.NotNil(t, explainPlan.queryDenylist)
@@ -2520,7 +2519,7 @@ func TestExplainPlan_PopulateQueryCache(t *testing.T) {
25202519
queryCache: make(map[string]*queryInfo),
25212520
queryDenylist: make(map[string]*queryInfo),
25222521
finishedQueryCache: make(map[string]*queryInfo),
2523-
excludeSchemas: []string{},
2522+
excludeDatabases: []string{},
25242523
perScrapeRatio: 1.0,
25252524
logger: logger,
25262525
entryHandler: lokiClient,
@@ -2560,7 +2559,7 @@ func TestExplainPlan_PopulateQueryCache(t *testing.T) {
25602559
queryCache: make(map[string]*queryInfo),
25612560
queryDenylist: make(map[string]*queryInfo),
25622561
finishedQueryCache: make(map[string]*queryInfo),
2563-
excludeSchemas: []string{"information_schema"},
2562+
excludeDatabases: []string{"information_schema"},
25642563
perScrapeRatio: 0.5,
25652564
logger: logger,
25662565
entryHandler: lokiClient,
@@ -2748,7 +2747,7 @@ func TestExplainPlanFetchExplainPlans(t *testing.T) {
27482747
queryCache: make(map[string]*queryInfo),
27492748
queryDenylist: make(map[string]*queryInfo),
27502749
finishedQueryCache: make(map[string]*queryInfo),
2751-
excludeSchemas: []string{},
2750+
excludeDatabases: []string{},
27522751
perScrapeRatio: 1.0,
27532752
logger: logger,
27542753
}
@@ -2794,7 +2793,7 @@ func TestExplainPlanFetchExplainPlans(t *testing.T) {
27942793
},
27952794
queryDenylist: map[string]*queryInfo{},
27962795
finishedQueryCache: map[string]*queryInfo{},
2797-
excludeSchemas: []string{},
2796+
excludeDatabases: []string{},
27982797
perScrapeRatio: 1.0,
27992798
logger: log.NewLogfmtLogger(log.NewSyncWriter(&logBuffer)),
28002799
entryHandler: lokiClient,
@@ -2849,7 +2848,7 @@ func TestExplainPlanFetchExplainPlans(t *testing.T) {
28492848
},
28502849
queryDenylist: map[string]*queryInfo{},
28512850
finishedQueryCache: map[string]*queryInfo{},
2852-
excludeSchemas: []string{},
2851+
excludeDatabases: []string{},
28532852
perScrapeRatio: 1.0,
28542853
logger: log.NewLogfmtLogger(log.NewSyncWriter(&logBuffer)),
28552854
entryHandler: lokiClient,
@@ -2902,7 +2901,7 @@ func TestExplainPlanFetchExplainPlans(t *testing.T) {
29022901
},
29032902
queryDenylist: map[string]*queryInfo{},
29042903
finishedQueryCache: map[string]*queryInfo{},
2905-
excludeSchemas: []string{},
2904+
excludeDatabases: []string{},
29062905
perScrapeRatio: 1.0,
29072906
logger: log.NewLogfmtLogger(log.NewSyncWriter(&logBuffer)),
29082907
currentBatchSize: 1,
@@ -2969,7 +2968,7 @@ func TestExplainPlanFetchExplainPlans(t *testing.T) {
29692968
},
29702969
queryDenylist: map[string]*queryInfo{},
29712970
finishedQueryCache: map[string]*queryInfo{},
2972-
excludeSchemas: []string{},
2971+
excludeDatabases: []string{},
29732972
perScrapeRatio: 1.0,
29742973
logger: log.NewLogfmtLogger(log.NewSyncWriter(&logBuffer)),
29752974
currentBatchSize: 1,
@@ -3036,7 +3035,7 @@ func TestExplainPlanFetchExplainPlans(t *testing.T) {
30363035
},
30373036
queryDenylist: map[string]*queryInfo{},
30383037
finishedQueryCache: map[string]*queryInfo{},
3039-
excludeSchemas: []string{},
3038+
excludeDatabases: []string{},
30403039
perScrapeRatio: 1.0,
30413040
logger: log.NewLogfmtLogger(log.NewSyncWriter(&logBuffer)),
30423041
currentBatchSize: 1,

‎internal/component/database_observability/postgres/component.go‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,13 @@ type Arguments struct {
6565
Targets []discovery.Target `alloy:"targets,attr"`
6666
EnableCollectors []string `alloy:"enable_collectors,attr,optional"`
6767
DisableCollectors []string `alloy:"disable_collectors,attr,optional"`
68+
ExcludeDatabases []string `alloy:"exclude_databases,attr,optional"`
6869

6970
CloudProvider *CloudProvider `alloy:"cloud_provider,block,optional"`
7071
QuerySampleArguments QuerySampleArguments `alloy:"query_samples,block,optional"`
7172
QueryTablesArguments QueryTablesArguments `alloy:"query_details,block,optional"`
7273
SchemaDetailsArguments SchemaDetailsArguments `alloy:"schema_details,block,optional"`
73-
ExplainPlanArguments ExplainPlanArguments `alloy:"explain_plans,block,optional"`
74+
ExplainPlansArguments ExplainPlansArguments `alloy:"explain_plans,block,optional"`
7475
HealthCheckArguments HealthCheckArguments `alloy:"health_check,block,optional"`
7576
}
7677

@@ -107,6 +108,7 @@ type SchemaDetailsArguments struct {
107108
}
108109

109110
var DefaultArguments = Arguments{
111+
ExcludeDatabases: []string{},
110112
QuerySampleArguments: QuerySampleArguments{
111113
CollectInterval: 15 * time.Second,
112114
DisableQueryRedaction: false,
@@ -121,7 +123,7 @@ var DefaultArguments = Arguments{
121123
CacheSize: 256,
122124
CacheTTL: 10 * time.Minute,
123125
},
124-
ExplainPlanArguments: ExplainPlanArguments{
126+
ExplainPlansArguments: ExplainPlansArguments{
125127
CollectInterval: 1 * time.Minute,
126128
PerCollectRatio: 1.0,
127129
},
@@ -130,10 +132,9 @@ var DefaultArguments = Arguments{
130132
},
131133
}
132134

133-
type ExplainPlanArguments struct {
134-
CollectInterval time.Duration `alloy:"collect_interval,attr,optional"`
135-
PerCollectRatio float64 `alloy:"per_collect_ratio,attr,optional"`
136-
ExplainPlanExcludeSchemas []string `alloy:"explain_plan_exclude_schemas,attr,optional"`
135+
type ExplainPlansArguments struct {
136+
CollectInterval time.Duration `alloy:"collect_interval,attr,optional"`
137+
PerCollectRatio float64 `alloy:"per_collect_ratio,attr,optional"`
137138
}
138139

139140
type HealthCheckArguments struct {
@@ -462,13 +463,14 @@ func (c *Component) startCollectors(systemID string, engineVersion string, cloud
462463

463464
if collectors[collector.ExplainPlanCollector] {
464465
epCollector, err := collector.NewExplainPlan(collector.ExplainPlansArguments{
465-
DB: c.dbConnection,
466-
DSN: string(c.args.DataSourceName),
467-
ScrapeInterval: c.args.ExplainPlanArguments.CollectInterval,
468-
PerScrapeRatio: c.args.ExplainPlanArguments.PerCollectRatio,
469-
Logger: c.opts.Logger,
470-
DBVersion: engineVersion,
471-
EntryHandler: entryHandler,
466+
DB: c.dbConnection,
467+
DSN: string(c.args.DataSourceName),
468+
ScrapeInterval: c.args.ExplainPlansArguments.CollectInterval,
469+
PerScrapeRatio: c.args.ExplainPlansArguments.PerCollectRatio,
470+
ExcludeDatabases: c.args.ExcludeDatabases,
471+
Logger: c.opts.Logger,
472+
DBVersion: engineVersion,
473+
EntryHandler: entryHandler,
472474
})
473475
if err != nil {
474476
logStartError(collector.ExplainPlanCollector, "create", err)

0 commit comments

Comments
 (0)