Skip to content

Commit 085f300

Browse files
cristiangrecocristiangreco
authored
fix: Database_observability: allow setting limit for mysql query_details (#5314)
### Brief description of Pull Request Allow limiting the number of recent queries collected in the mysql `query_details` collector. Default to 250 to match mysqld_exporter's behavior. ### Pull Request Details <!-- Add a more detailed descripion of the Pull Request here, if needed. --> ### Issue(s) fixed by this Pull Request <!-- Uncomment the following line and fill in an issue number if you want a GitHub issue to be closed automatically when this PR gets merged. --> <!-- Fixes #issue_id --> ### Notes to the Reviewer <!-- Add any relevant notes for the reviewers and testers of this PR. --> ### PR Checklist <!-- Remove items that do not apply. For completed items, change [ ] to [x]. --> - [x] Documentation added - [x] Tests updated - [ ] Config converters updated
1 parent 3090c4a commit 085f300

File tree

4 files changed

+32
-18
lines changed

4 files changed

+32
-18
lines changed

‎docs/sources/reference/components/database_observability/database_observability.mysql.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ The `azure` block supplies the identifying information for the database being mo
117117
| Name | Type | Description | Default | Required |
118118
|--------------------|------------|------------------------------------------------------|---------|----------|
119119
| `collect_interval` | `duration` | How frequently to collect information from database. | `"1m"` | no |
120+
| `statements_limit` | `integer` | Max number of recent queries to collect details for. | `250` | no |
120121

121122
### `schema_details`
122123

‎internal/component/database_observability/mysql/collector/query_details.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,14 @@ const selectQueryTablesSamples = `
3131
query_sample_text
3232
FROM performance_schema.events_statements_summary_by_digest
3333
WHERE last_seen > DATE_SUB(NOW(), INTERVAL 1 DAY)
34-
AND schema_name NOT IN ` + EXCLUDED_SCHEMAS
34+
AND schema_name NOT IN %s
35+
ORDER BY last_seen DESC
36+
LIMIT %d`
3537

3638
type QueryDetailsArguments struct {
3739
DB *sql.DB
3840
CollectInterval time.Duration
41+
StatementsLimit int
3942
EntryHandler loki.EntryHandler
4043

4144
Logger log.Logger
@@ -44,6 +47,7 @@ type QueryDetailsArguments struct {
4447
type QueryDetails struct {
4548
dbConnection *sql.DB
4649
collectInterval time.Duration
50+
statementsLimit int
4751
entryHandler loki.EntryHandler
4852
sqlParser parser.Parser
4953
normalizer *sqllexer.Normalizer
@@ -58,6 +62,7 @@ func NewQueryDetails(args QueryDetailsArguments) (*QueryDetails, error) {
5862
c := &QueryDetails{
5963
dbConnection: args.DB,
6064
collectInterval: args.CollectInterval,
65+
statementsLimit: args.StatementsLimit,
6166
entryHandler: args.EntryHandler,
6267
sqlParser: parser.NewTiDBSqlParser(),
6368
normalizer: sqllexer.NewNormalizer(sqllexer.WithCollectTables(true)),
@@ -115,7 +120,8 @@ func (c *QueryDetails) Stop() {
115120
}
116121

117122
func (c *QueryDetails) tablesFromEventsStatements(ctx context.Context) error {
118-
rs, err := c.dbConnection.QueryContext(ctx, selectQueryTablesSamples)
123+
query := fmt.Sprintf(selectQueryTablesSamples, EXCLUDED_SCHEMAS, c.statementsLimit)
124+
rs, err := c.dbConnection.QueryContext(ctx, query)
119125
if err != nil {
120126
return fmt.Errorf("failed to fetch summary table samples: %w", err)
121127
}

‎internal/component/database_observability/mysql/collector/query_details_test.go‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -371,13 +371,14 @@ func TestQueryTables(t *testing.T) {
371371
collector, err := NewQueryDetails(QueryDetailsArguments{
372372
DB: db,
373373
CollectInterval: time.Second,
374+
StatementsLimit: 250,
374375
EntryHandler: lokiClient,
375376
Logger: log.NewLogfmtLogger(os.Stderr),
376377
})
377378
require.NoError(t, err)
378379
require.NotNil(t, collector)
379380

380-
mock.ExpectQuery(selectQueryTablesSamples).WithoutArgs().RowsWillBeClosed().
381+
mock.ExpectQuery(fmt.Sprintf(selectQueryTablesSamples, EXCLUDED_SCHEMAS, 250)).WithoutArgs().RowsWillBeClosed().
381382
WillReturnRows(
382383
sqlmock.NewRows([]string{
383384
"digest",
@@ -431,21 +432,22 @@ func TestQueryTablesSQLDriverErrors(t *testing.T) {
431432
collector, err := NewQueryDetails(QueryDetailsArguments{
432433
DB: db,
433434
CollectInterval: time.Second,
435+
StatementsLimit: 250,
434436
EntryHandler: lokiClient,
435437
Logger: log.NewLogfmtLogger(os.Stderr),
436438
})
437439
require.NoError(t, err)
438440
require.NotNil(t, collector)
439441

440-
mock.ExpectQuery(selectQueryTablesSamples).WithoutArgs().RowsWillBeClosed().
442+
mock.ExpectQuery(fmt.Sprintf(selectQueryTablesSamples, EXCLUDED_SCHEMAS, 250)).WithoutArgs().RowsWillBeClosed().
441443
WillReturnRows(
442444
sqlmock.NewRows([]string{
443445
"digest", // not enough columns
444446
}).AddRow(
445447
"abc123",
446448
))
447449

448-
mock.ExpectQuery(selectQueryTablesSamples).WithoutArgs().RowsWillBeClosed().
450+
mock.ExpectQuery(fmt.Sprintf(selectQueryTablesSamples, EXCLUDED_SCHEMAS, 250)).WithoutArgs().RowsWillBeClosed().
449451
WillReturnRows(
450452
sqlmock.NewRows([]string{
451453
"digest",
@@ -496,13 +498,14 @@ func TestQueryTablesSQLDriverErrors(t *testing.T) {
496498
collector, err := NewQueryDetails(QueryDetailsArguments{
497499
DB: db,
498500
CollectInterval: time.Second,
501+
StatementsLimit: 250,
499502
EntryHandler: lokiClient,
500503
Logger: log.NewLogfmtLogger(os.Stderr),
501504
})
502505
require.NoError(t, err)
503506
require.NotNil(t, collector)
504507

505-
mock.ExpectQuery(selectQueryTablesSamples).WithoutArgs().RowsWillBeClosed().
508+
mock.ExpectQuery(fmt.Sprintf(selectQueryTablesSamples, EXCLUDED_SCHEMAS, 250)).WithoutArgs().RowsWillBeClosed().
506509
WillReturnRows(
507510
sqlmock.NewRows([]string{
508511
"digest",
@@ -558,15 +561,16 @@ func TestQueryTablesSQLDriverErrors(t *testing.T) {
558561
collector, err := NewQueryDetails(QueryDetailsArguments{
559562
DB: db,
560563
CollectInterval: time.Second,
564+
StatementsLimit: 250,
561565
EntryHandler: lokiClient,
562566
Logger: log.NewLogfmtLogger(os.Stderr),
563567
})
564568
require.NoError(t, err)
565569
require.NotNil(t, collector)
566570

567-
mock.ExpectQuery(selectQueryTablesSamples).WithoutArgs().WillReturnError(fmt.Errorf("connection error"))
571+
mock.ExpectQuery(fmt.Sprintf(selectQueryTablesSamples, EXCLUDED_SCHEMAS, 250)).WithoutArgs().WillReturnError(fmt.Errorf("connection error"))
568572

569-
mock.ExpectQuery(selectQueryTablesSamples).WithoutArgs().RowsWillBeClosed().
573+
mock.ExpectQuery(fmt.Sprintf(selectQueryTablesSamples, EXCLUDED_SCHEMAS, 250)).WithoutArgs().RowsWillBeClosed().
570574
WillReturnRows(
571575
sqlmock.NewRows([]string{
572576
"digest",

‎internal/component/database_observability/mysql/component.go‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ type Arguments struct {
6565
CloudProvider *CloudProvider `alloy:"cloud_provider,block,optional"`
6666
SetupConsumersArguments SetupConsumersArguments `alloy:"setup_consumers,block,optional"`
6767
SetupActorsArguments SetupActorsArguments `alloy:"setup_actors,block,optional"`
68-
QueryTablesArguments QueryTablesArguments `alloy:"query_details,block,optional"`
69-
SchemaTablesArguments SchemaDetailsArguments `alloy:"schema_details,block,optional"`
68+
QueryDetailsArguments QueryDetailsArguments `alloy:"query_details,block,optional"`
69+
SchemaDetailsArguments SchemaDetailsArguments `alloy:"schema_details,block,optional"`
7070
ExplainPlansArguments ExplainPlansArguments `alloy:"explain_plans,block,optional"`
7171
LocksArguments LocksArguments `alloy:"locks,block,optional"`
7272
QuerySamplesArguments QuerySamplesArguments `alloy:"query_samples,block,optional"`
@@ -88,8 +88,9 @@ type AzureCloudProviderInfo struct {
8888
ServerName string `alloy:"server_name,attr,optional"`
8989
}
9090

91-
type QueryTablesArguments struct {
91+
type QueryDetailsArguments struct {
9292
CollectInterval time.Duration `alloy:"collect_interval,attr,optional"`
93+
StatementsLimit int `alloy:"statements_limit,attr,optional"`
9394
}
9495

9596
type SchemaDetailsArguments struct {
@@ -134,11 +135,12 @@ type HealthCheckArguments struct {
134135
var DefaultArguments = Arguments{
135136
AllowUpdatePerfSchemaSettings: false,
136137

137-
QueryTablesArguments: QueryTablesArguments{
138+
QueryDetailsArguments: QueryDetailsArguments{
138139
CollectInterval: 1 * time.Minute,
140+
StatementsLimit: 250,
139141
},
140142

141-
SchemaTablesArguments: SchemaDetailsArguments{
143+
SchemaDetailsArguments: SchemaDetailsArguments{
142144
CollectInterval: 1 * time.Minute,
143145
CacheEnabled: true,
144146
CacheSize: 256,
@@ -445,7 +447,8 @@ func (c *Component) startCollectors(serverID string, engineVersion string, parse
445447
if collectors[collector.QueryDetailsCollector] {
446448
qtCollector, err := collector.NewQueryDetails(collector.QueryDetailsArguments{
447449
DB: c.dbConnection,
448-
CollectInterval: c.args.QueryTablesArguments.CollectInterval,
450+
CollectInterval: c.args.QueryDetailsArguments.CollectInterval,
451+
StatementsLimit: c.args.QueryDetailsArguments.StatementsLimit,
449452
EntryHandler: entryHandler,
450453
Logger: c.opts.Logger,
451454
})
@@ -462,10 +465,10 @@ func (c *Component) startCollectors(serverID string, engineVersion string, parse
462465
if collectors[collector.SchemaDetailsCollector] {
463466
stCollector, err := collector.NewSchemaDetails(collector.SchemaDetailsArguments{
464467
DB: c.dbConnection,
465-
CollectInterval: c.args.SchemaTablesArguments.CollectInterval,
466-
CacheEnabled: c.args.SchemaTablesArguments.CacheEnabled,
467-
CacheSize: c.args.SchemaTablesArguments.CacheSize,
468-
CacheTTL: c.args.SchemaTablesArguments.CacheTTL,
468+
CollectInterval: c.args.SchemaDetailsArguments.CollectInterval,
469+
CacheEnabled: c.args.SchemaDetailsArguments.CacheEnabled,
470+
CacheSize: c.args.SchemaDetailsArguments.CacheSize,
471+
CacheTTL: c.args.SchemaDetailsArguments.CacheTTL,
469472
EntryHandler: entryHandler,
470473
Logger: c.opts.Logger,
471474
})

0 commit comments

Comments
 (0)