From d7165e0cf057fa356a6d596b72f56a5e96ba97c3 Mon Sep 17 00:00:00 2001 From: yeya24 Date: Tue, 13 May 2025 09:59:08 -0700 Subject: [PATCH 1/2] move cleaner bucket index metrics before cleaning partition group Signed-off-by: yeya24 --- pkg/compactor/blocks_cleaner.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/compactor/blocks_cleaner.go b/pkg/compactor/blocks_cleaner.go index 8ef344e378f..273091e1a92 100644 --- a/pkg/compactor/blocks_cleaner.go +++ b/pkg/compactor/blocks_cleaner.go @@ -603,6 +603,10 @@ func (c *BlocksCleaner) cleanUser(ctx context.Context, userLogger log.Logger, us return err } level.Info(userLogger).Log("msg", "finish updating index", "duration", time.Since(begin), "duration_ms", time.Since(begin).Milliseconds()) + c.tenantBlocks.WithLabelValues(userID).Set(float64(len(idx.Blocks))) + c.tenantBlocksMarkedForDelete.WithLabelValues(userID).Set(float64(len(idx.BlockDeletionMarks))) + c.tenantBlocksMarkedForNoCompaction.WithLabelValues(userID).Set(float64(totalBlocksBlocksMarkedForNoCompaction)) + c.tenantPartialBlocks.WithLabelValues(userID).Set(float64(len(partials))) // Delete blocks marked for deletion. We iterate over a copy of deletion marks because // we'll need to manipulate the index (removing blocks which get deleted). @@ -662,18 +666,13 @@ func (c *BlocksCleaner) cleanUser(ctx context.Context, userLogger log.Logger, us } level.Info(userLogger).Log("msg", "finish writing new index", "duration", time.Since(begin), "duration_ms", time.Since(begin).Milliseconds()) } + c.tenantBucketIndexLastUpdate.WithLabelValues(userID).SetToCurrentTime() if c.cfg.ShardingStrategy == util.ShardingStrategyShuffle && c.cfg.CompactionStrategy == util.CompactionStrategyPartitioning { begin = time.Now() c.cleanPartitionedGroupInfo(ctx, userBucket, userLogger, userID) level.Info(userLogger).Log("msg", "finish cleaning partitioned group info files", "duration", time.Since(begin), "duration_ms", time.Since(begin).Milliseconds()) } - - c.tenantBlocks.WithLabelValues(userID).Set(float64(len(idx.Blocks))) - c.tenantBlocksMarkedForDelete.WithLabelValues(userID).Set(float64(len(idx.BlockDeletionMarks))) - c.tenantBlocksMarkedForNoCompaction.WithLabelValues(userID).Set(float64(totalBlocksBlocksMarkedForNoCompaction)) - c.tenantBucketIndexLastUpdate.WithLabelValues(userID).SetToCurrentTime() - c.tenantPartialBlocks.WithLabelValues(userID).Set(float64(len(partials))) return nil } From e8385a617d3439dcf52771c7b88d9f1c0d6c8ca8 Mon Sep 17 00:00:00 2001 From: yeya24 Date: Tue, 13 May 2025 10:29:17 -0700 Subject: [PATCH 2/2] fix metric Signed-off-by: yeya24 --- pkg/compactor/blocks_cleaner.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/compactor/blocks_cleaner.go b/pkg/compactor/blocks_cleaner.go index 273091e1a92..4c2e4887b08 100644 --- a/pkg/compactor/blocks_cleaner.go +++ b/pkg/compactor/blocks_cleaner.go @@ -603,10 +603,6 @@ func (c *BlocksCleaner) cleanUser(ctx context.Context, userLogger log.Logger, us return err } level.Info(userLogger).Log("msg", "finish updating index", "duration", time.Since(begin), "duration_ms", time.Since(begin).Milliseconds()) - c.tenantBlocks.WithLabelValues(userID).Set(float64(len(idx.Blocks))) - c.tenantBlocksMarkedForDelete.WithLabelValues(userID).Set(float64(len(idx.BlockDeletionMarks))) - c.tenantBlocksMarkedForNoCompaction.WithLabelValues(userID).Set(float64(totalBlocksBlocksMarkedForNoCompaction)) - c.tenantPartialBlocks.WithLabelValues(userID).Set(float64(len(partials))) // Delete blocks marked for deletion. We iterate over a copy of deletion marks because // we'll need to manipulate the index (removing blocks which get deleted). @@ -666,6 +662,10 @@ func (c *BlocksCleaner) cleanUser(ctx context.Context, userLogger log.Logger, us } level.Info(userLogger).Log("msg", "finish writing new index", "duration", time.Since(begin), "duration_ms", time.Since(begin).Milliseconds()) } + c.tenantBlocks.WithLabelValues(userID).Set(float64(len(idx.Blocks))) + c.tenantBlocksMarkedForDelete.WithLabelValues(userID).Set(float64(len(idx.BlockDeletionMarks))) + c.tenantBlocksMarkedForNoCompaction.WithLabelValues(userID).Set(float64(totalBlocksBlocksMarkedForNoCompaction)) + c.tenantPartialBlocks.WithLabelValues(userID).Set(float64(len(partials))) c.tenantBucketIndexLastUpdate.WithLabelValues(userID).SetToCurrentTime() if c.cfg.ShardingStrategy == util.ShardingStrategyShuffle && c.cfg.CompactionStrategy == util.CompactionStrategyPartitioning {