Skip to content

database_observability: fix schema_details collector to fetch column definitions with case sensitive table names #4872

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fridgepoet merged 4 commits into from
Nov 19, 2025
Merged

database_observability: fix schema_details collector to fetch column definitions with case sensitive table names #4872

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ce4c459
wip
fridgepoet Nov 18, 2025
30f1766
update changelog
fridgepoet Nov 18, 2025
2eff6e9
Merge branch 'main' into shirley/case-sensitivity-table-detection
fridgepoet Nov 19, 2025
d9d1341
not experimental
fridgepoet Nov 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ Main (unreleased)

- `loki.source.api` no longer drops request when relabel rules drops a specific stream. (@kalleep)

- (_Public Preview_) Additions to `database_observability.postgres` component:
Copy link
Contributor Author

@fridgepoet fridgepoet Nov 19, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Put this under Next > Bugfixes and with an Public Preview tag. Is that right?

- fixes collection of Postgres schema details for mixed case table names (@fridgepoet)

v1.12.0-rc.0
-----------------

Expand Down
4 changes: 3 additions & 1 deletion internal/component/database_observability/postgres/collector/connection_info_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@ import (
)

func TestConnectionInfo(t *testing.T) {
defer goleak.VerifyNone(t)
Copy link
Contributor Author

@fridgepoet fridgepoet Nov 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Three tests were failing locally if I ran them with -count=2 because of this. Just a drive-by improvement but can remove this from the PR if we prefer.

// The goroutine which deletes expired entries runs indefinitely,
// see https://github.com/hashicorp/golang-lru/blob/v2.0.7/expirable/expirable_lru.go#L79-L80
defer goleak.VerifyNone(t, goleak.IgnoreTopFunction("github.com/hashicorp/golang-lru/v2/expirable.NewLRU[...].func1"))

const baseExpectedMetrics = `
# HELP database_observability_connection_info Information about the connection
Expand Down
8 changes: 6 additions & 2 deletions internal/component/database_observability/postgres/collector/query_details_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ import (
)

func TestQueryDetails(t *testing.T) {
defer goleak.VerifyNone(t)
// The goroutine which deletes expired entries runs indefinitely,
// see https://github.com/hashicorp/golang-lru/blob/v2.0.7/expirable/expirable_lru.go#L79-L80
defer goleak.VerifyNone(t, goleak.IgnoreTopFunction("github.com/hashicorp/golang-lru/v2/expirable.NewLRU[...].func1"))

testcases := []struct {
name string
Expand Down Expand Up @@ -468,7 +470,9 @@ func TestQueryDetails(t *testing.T) {
}

func TestQueryDetails_SQLDriverErrors(t *testing.T) {
defer goleak.VerifyNone(t)
// The goroutine which deletes expired entries runs indefinitely,
// see https://github.com/hashicorp/golang-lru/blob/v2.0.7/expirable/expirable_lru.go#L79-L80
defer goleak.VerifyNone(t, goleak.IgnoreTopFunction("github.com/hashicorp/golang-lru/v2/expirable.NewLRU[...].func1"))

t.Run("recoverable sql error in result set", func(t *testing.T) {
t.Parallel()
Expand Down
11 changes: 7 additions & 4 deletions internal/component/database_observability/postgres/collector/schema_details.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,8 @@ const (
pg_attribute: stores column information
pg_attrdef: stores default values for columns
pg_constraint: stores primary key information
attr.attrelid = $1::regclass -- filter by the table name
JOIN pg_class to filter by table name
JOIN pg_namespace to filter by schema name
AND attr.attnum > 0 -- no system columns
AND NOT attr.attisdropped -- no dropped columns`
*/
Expand All @@ -87,10 +88,13 @@ const (
END as is_primary_key
FROM
pg_attribute attr
JOIN pg_catalog.pg_class pg_class ON attr.attrelid = pg_class.oid
JOIN pg_catalog.pg_namespace pg_namespace ON pg_class.relnamespace = pg_namespace.oid
LEFT JOIN pg_catalog.pg_attrdef def ON attr.attrelid = def.adrelid AND attr.attnum = def.adnum
LEFT JOIN pg_catalog.pg_constraint constraint_pk ON attr.attrelid = constraint_pk.conrelid AND attr.attnum = ANY(constraint_pk.conkey) AND constraint_pk.contype = 'p'
WHERE
attr.attrelid = $1::regclass
pg_namespace.nspname = $1
AND pg_class.relname = $2
AND attr.attnum > 0
AND NOT attr.attisdropped`

Expand Down Expand Up @@ -580,8 +584,7 @@ func (c *SchemaDetails) fetchTableDefinitions(ctx context.Context, table *tableI
}

func (c *SchemaDetails) fetchColumnsDefinitions(ctx context.Context, databaseName database, schemaName schema, tableName table, dbConnection *sql.DB) (*tableSpec, error) {
qualifiedTableName := fmt.Sprintf("%s.%s", schemaName, tableName)
colRS, err := dbConnection.QueryContext(ctx, selectColumnNames, qualifiedTableName)
colRS, err := dbConnection.QueryContext(ctx, selectColumnNames, schemaName, tableName)
if err != nil {
level.Error(c.logger).Log("msg", "failed to query table columns", "datname", databaseName, "schema", schemaName, "table", tableName, "err", err)
return nil, err
Expand Down
50 changes: 25 additions & 25 deletions internal/component/database_observability/postgres/collector/schema_details_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func Test_Postgres_SchemaDetails(t *testing.T) {
}).AddRow("authors"),
)

mock.ExpectQuery(selectColumnNames).WithArgs("public.authors").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "authors").RowsWillBeClosed().
WillReturnRows(
sqlmock.NewRows([]string{
"column_name",
Expand Down Expand Up @@ -189,7 +189,7 @@ func Test_Postgres_SchemaDetails(t *testing.T) {
}).AddRow("spatial_ref_sys"),
)

mock.ExpectQuery(selectColumnNames).WithArgs("public.authors").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "authors").RowsWillBeClosed().
WillReturnRows(
sqlmock.NewRows([]string{
"column_name",
Expand Down Expand Up @@ -223,7 +223,7 @@ func Test_Postgres_SchemaDetails(t *testing.T) {
}),
)

mock.ExpectQuery(selectColumnNames).WithArgs("public.categories").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "categories").RowsWillBeClosed().
WillReturnRows(
sqlmock.NewRows([]string{
"column_name",
Expand Down Expand Up @@ -257,7 +257,7 @@ func Test_Postgres_SchemaDetails(t *testing.T) {
}),
)

mock.ExpectQuery(selectColumnNames).WithArgs("postgis.spatial_ref_sys").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("postgis", "spatial_ref_sys").RowsWillBeClosed().
WillReturnRows(
sqlmock.NewRows([]string{
"column_name",
Expand Down Expand Up @@ -383,7 +383,7 @@ func Test_Postgres_SchemaDetails(t *testing.T) {
WillReturnRows(sqlmock.NewRows([]string{"schema_name"}).AddRow("public"))
db1Mock.ExpectQuery(selectTableNames).WithArgs("public").RowsWillBeClosed().
WillReturnRows(sqlmock.NewRows([]string{"table_name"}).AddRow("users"))
db1Mock.ExpectQuery(selectColumnNames).WithArgs("public.users").RowsWillBeClosed().
db1Mock.ExpectQuery(selectColumnNames).WithArgs("public", "users").RowsWillBeClosed().
WillReturnRows(
sqlmock.NewRows([]string{"column_name", "column_type", "not_nullable", "column_default", "identity_generation", "is_primary_key"}).
AddRow("id", "integer", true, nil, "", true),
Expand All @@ -397,7 +397,7 @@ func Test_Postgres_SchemaDetails(t *testing.T) {
WillReturnRows(sqlmock.NewRows([]string{"schema_name"}).AddRow("public"))
db2Mock.ExpectQuery(selectTableNames).WithArgs("public").RowsWillBeClosed().
WillReturnRows(sqlmock.NewRows([]string{"table_name"}).AddRow("metrics"))
db2Mock.ExpectQuery(selectColumnNames).WithArgs("public.metrics").RowsWillBeClosed().
db2Mock.ExpectQuery(selectColumnNames).WithArgs("public", "metrics").RowsWillBeClosed().
WillReturnRows(
sqlmock.NewRows([]string{"column_name", "column_type", "not_nullable", "column_default", "identity_generation", "is_primary_key"}).
AddRow("id", "bigint", true, nil, "", true),
Expand Down Expand Up @@ -485,7 +485,7 @@ func Test_Postgres_SchemaDetails(t *testing.T) {
}).AddRow("users"),
)

mock.ExpectQuery(selectColumnNames).WithArgs("public.users").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "users").RowsWillBeClosed().
WillReturnRows(
sqlmock.NewRows([]string{
"column_name",
Expand Down Expand Up @@ -654,7 +654,7 @@ func Test_Postgres_SchemaDetails(t *testing.T) {
}).AddRow("test_table"),
)

mock.ExpectQuery(selectColumnNames).WithArgs("public.test_table").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "test_table").RowsWillBeClosed().
WillReturnRows(
sqlmock.NewRows([]string{
"column_name",
Expand Down Expand Up @@ -765,7 +765,7 @@ func Test_Postgres_SchemaDetails_collector_detects_auto_increment_column(t *test
}).AddRow("users"),
)

mock.ExpectQuery(selectColumnNames).WithArgs("public.users").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "users").RowsWillBeClosed().
WillReturnRows(
sqlmock.NewRows([]string{
"column_name",
Expand Down Expand Up @@ -871,7 +871,7 @@ func Test_Postgres_SchemaDetails_collector_detects_auto_increment_column(t *test
}).AddRow("products"),
)

mock.ExpectQuery(selectColumnNames).WithArgs("public.products").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "products").RowsWillBeClosed().
WillReturnRows(
sqlmock.NewRows([]string{
"column_name",
Expand Down Expand Up @@ -978,7 +978,7 @@ func Test_Postgres_SchemaDetails_collector_detects_auto_increment_column(t *test
}).AddRow("books"),
)

mock.ExpectQuery(selectColumnNames).WithArgs("public.books").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "books").RowsWillBeClosed().
WillReturnRows(
sqlmock.NewRows([]string{
"column_name",
Expand Down Expand Up @@ -1085,7 +1085,7 @@ func Test_Postgres_SchemaDetails_caching(t *testing.T) {
)

// selectColumnNames, selectIndexes, selectForeignKeys called only first run
mock.ExpectQuery(selectColumnNames).WithArgs("public.test_table").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "test_table").RowsWillBeClosed().
WillReturnRows(
sqlmock.NewRows([]string{
"column_name",
Expand Down Expand Up @@ -1215,7 +1215,7 @@ func Test_Postgres_SchemaDetails_caching(t *testing.T) {
}).AddRow("test_table"),
)

mock.ExpectQuery(selectColumnNames).WithArgs("public.test_table").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "test_table").RowsWillBeClosed().
WillReturnRows(
sqlmock.NewRows([]string{
"column_name",
Expand Down Expand Up @@ -1322,7 +1322,7 @@ func Test_Postgres_SchemaDetails_ErrorCases(t *testing.T) {
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
require.NoError(t, err)
defer db.Close()
mock.ExpectQuery(selectColumnNames).WithArgs("public.test_table").
mock.ExpectQuery(selectColumnNames).WithArgs("public", "test_table").
WillReturnError(fmt.Errorf("column query error"))

collector := &SchemaDetails{logger: log.NewNopLogger()}
Expand All @@ -1347,7 +1347,7 @@ func Test_Postgres_SchemaDetails_ErrorCases(t *testing.T) {
logger: log.NewNopLogger(),
}

mock.ExpectQuery(selectColumnNames).WithArgs("public.test_table").
mock.ExpectQuery(selectColumnNames).WithArgs("public", "test_table").
WillReturnError(fmt.Errorf("column query error"))

_, err = collector.fetchColumnsDefinitions(context.Background(), "testdb", "public", "test_table", db)
Expand All @@ -1367,7 +1367,7 @@ func Test_Postgres_SchemaDetails_ErrorCases(t *testing.T) {
logger: log.NewNopLogger(),
}

mock.ExpectQuery(selectColumnNames).WithArgs("public.test_table").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "test_table").RowsWillBeClosed().
WillReturnRows(sqlmock.NewRows([]string{"column_name", "column_type", "not_nullable", "column_default", "identity_generation"}).
AddRow("id", "integer", true, nil, ""))

Expand All @@ -1383,7 +1383,7 @@ func Test_Postgres_SchemaDetails_ErrorCases(t *testing.T) {
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
require.NoError(t, err)
defer db.Close()
mock.ExpectQuery(selectColumnNames).WithArgs("public.test_table").
mock.ExpectQuery(selectColumnNames).WithArgs("public", "test_table").
WillReturnRows(
sqlmock.NewRows([]string{"column_name", "column_type", "not_nullable", "column_default", "identity_generation", "is_primary_key"}).
AddRow("id", "integer", true, nil, "", true).
Expand All @@ -1402,7 +1402,7 @@ func Test_Postgres_SchemaDetails_ErrorCases(t *testing.T) {
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
require.NoError(t, err)
defer db.Close()
mock.ExpectQuery(selectColumnNames).WithArgs("public.test_table").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "test_table").RowsWillBeClosed().
WillReturnRows(sqlmock.NewRows([]string{"column_name", "column_type", "not_nullable", "column_default", "identity_generation", "is_primary_key"}).
AddRow("id", "integer", true, nil, "", true))
mock.ExpectQuery(selectIndexes).WithArgs("public", "test_table").
Expand All @@ -1421,7 +1421,7 @@ func Test_Postgres_SchemaDetails_ErrorCases(t *testing.T) {
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
require.NoError(t, err)
defer db.Close()
mock.ExpectQuery(selectColumnNames).WithArgs("public.test_table").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "test_table").RowsWillBeClosed().
WillReturnRows(sqlmock.NewRows([]string{"column_name", "column_type", "not_nullable", "column_default", "identity_generation", "is_primary_key"}).
AddRow("id", "integer", true, nil, "", true))
mock.ExpectQuery(selectIndexes).WithArgs("public", "test_table").RowsWillBeClosed().
Expand All @@ -1441,7 +1441,7 @@ func Test_Postgres_SchemaDetails_ErrorCases(t *testing.T) {
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
require.NoError(t, err)
defer db.Close()
mock.ExpectQuery(selectColumnNames).WithArgs("public.test_table").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "test_table").RowsWillBeClosed().
WillReturnRows(sqlmock.NewRows([]string{"column_name", "column_type", "not_nullable", "column_default", "identity_generation", "is_primary_key"}).
AddRow("id", "integer", true, nil, "", true))
mock.ExpectQuery(selectIndexes).WithArgs("public", "test_table").
Expand All @@ -1464,7 +1464,7 @@ func Test_Postgres_SchemaDetails_ErrorCases(t *testing.T) {
require.NoError(t, err)
defer db.Close()

mock.ExpectQuery(selectColumnNames).WithArgs("public.test_table").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "test_table").RowsWillBeClosed().
WillReturnRows(sqlmock.NewRows([]string{"column_name", "column_type", "not_nullable", "column_default", "identity_generation", "is_primary_key"}).
AddRow("id", "integer", true, nil, "", true))
mock.ExpectQuery(selectIndexes).WithArgs("public", "test_table").RowsWillBeClosed().
Expand All @@ -1486,7 +1486,7 @@ func Test_Postgres_SchemaDetails_ErrorCases(t *testing.T) {
require.NoError(t, err)
defer db.Close()

mock.ExpectQuery(selectColumnNames).WithArgs("public.test_table").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "test_table").RowsWillBeClosed().
WillReturnRows(sqlmock.NewRows([]string{"column_name", "column_type", "not_nullable", "column_default", "identity_generation", "is_primary_key"}).
AddRow("id", "integer", true, nil, "", true))
mock.ExpectQuery(selectIndexes).WithArgs("public", "test_table").RowsWillBeClosed().
Expand All @@ -1508,7 +1508,7 @@ func Test_Postgres_SchemaDetails_ErrorCases(t *testing.T) {
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
require.NoError(t, err)
defer db.Close()
mock.ExpectQuery(selectColumnNames).WithArgs("public.test_table").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "test_table").RowsWillBeClosed().
WillReturnRows(sqlmock.NewRows([]string{"column_name", "column_type", "not_nullable", "column_default", "identity_generation", "is_primary_key"}).
AddRow("id", "integer", true, nil, "", true))
mock.ExpectQuery(selectIndexes).WithArgs("public", "test_table").RowsWillBeClosed().
Expand Down Expand Up @@ -1627,7 +1627,7 @@ func Test_SchemaDetails_populates_TableRegistry(t *testing.T) {
"table_name",
}).AddRow("users").AddRow("orders"),
)
mock.ExpectQuery(selectColumnNames).WithArgs("public.users").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "users").RowsWillBeClosed().
WillReturnRows(
sqlmock.NewRows([]string{
"column_name",
Expand Down Expand Up @@ -1658,7 +1658,7 @@ func Test_SchemaDetails_populates_TableRegistry(t *testing.T) {
"referenced_column_name",
}),
)
mock.ExpectQuery(selectColumnNames).WithArgs("public.orders").RowsWillBeClosed().
mock.ExpectQuery(selectColumnNames).WithArgs("public", "orders").RowsWillBeClosed().
WillReturnRows(
sqlmock.NewRows([]string{
"column_name",
Expand Down
Loading