From 3f04ab7884b165fab2217afc5cbce10833b13036 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 3 Jun 2026 13:18:26 +0200 Subject: [PATCH] ref(taskbroker): Migrate ingest-profiles devenv to new config format Move the ingest-profiles taskbroker to the new kafka_clusters/kafka_topics format. Raw mode moves onto the profiles topic's per-topic raw block, and a retry topic is now mandatory for raw topics (raw messages aren't activations, so retries can't loop back into the raw topic). Retries are routed to the main taskworker topic so the default taskbroker picks them up instead of requiring another broker. Depends on taskbroker #668 (per-topic raw + mandatory-retry validation). ref STREAM-1093 Co-Authored-By: Claude Opus 4.8 --- devservices/config.yml | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/devservices/config.yml b/devservices/config.yml index 176478efee9d..b935865d5ef3 100644 --- a/devservices/config.yml +++ b/devservices/config.yml @@ -497,14 +497,31 @@ services: ingest-profiles: image: ghcr.io/getsentry/taskbroker:nightly environment: - TASKBROKER_KAFKA_CLUSTER: 'kafka-kafka-1:9093' - TASKBROKER_KAFKA_TOPIC: 'profiles' - TASKBROKER_KAFKA_CONSUMER_GROUP: 'ingest-profiles' + # Multi-topic/cluster config via env vars: nested keys use "__", and + # figment lowercases each segment, so topic/cluster names can only use + # underscores (not hyphens) when set this way. + TASKBROKER_KAFKA_CLUSTERS__DEFAULT__ADDRESS: 'kafka-kafka-1:9093' + # Consumed topic, in raw (passthrough) mode. Raw mode is configured per + # topic via the topic's `raw` block. + TASKBROKER_KAFKA_TOPICS__PROFILES__CLUSTER: 'default' + TASKBROKER_KAFKA_TOPICS__PROFILES__CONSUMER_GROUP: 'ingest-profiles' + TASKBROKER_KAFKA_TOPICS__PROFILES__RAW__NAMESPACE: 'ingest.profiling.passthrough' + TASKBROKER_KAFKA_TOPICS__PROFILES__RAW__APPLICATION: 'sentry' + TASKBROKER_KAFKA_TOPICS__PROFILES__RAW__TASKNAME: 'sentry.profiles.task.process_profile_from_kafka' + TASKBROKER_KAFKA_TOPICS__PROFILES__RAW__PROCESSING_DEADLINE_DURATION: '30' + # A raw topic's messages aren't activations, so retries (which are + # activation-encoded) cannot loop back into `profiles`; a separate retry + # topic is mandatory. Send them to the main `taskworker` topic so the + # default taskbroker picks them up instead of standing up another broker. + TASKBROKER_KAFKA_RETRY_TOPIC: 'taskworker' + TASKBROKER_KAFKA_TOPICS__TASKWORKER__CLUSTER: 'default' + TASKBROKER_KAFKA_TOPICS__TASKWORKER__CONSUMER_GROUP: 'ingest-profiles' + TASKBROKER_KAFKA_TOPICS__TASKWORKER__PRODUCE_ONLY: 'true' + TASKBROKER_KAFKA_DEADLETTER_TOPIC: 'taskworker_dlq' + TASKBROKER_KAFKA_TOPICS__TASKWORKER_DLQ__CLUSTER: 'default' + TASKBROKER_KAFKA_TOPICS__TASKWORKER_DLQ__CONSUMER_GROUP: 'ingest-profiles' + TASKBROKER_KAFKA_TOPICS__TASKWORKER_DLQ__PRODUCE_ONLY: 'true' TASKBROKER_CREATE_MISSING_TOPICS: 'true' - TASKBROKER_RAW_MODE: 'true' - TASKBROKER_RAW_NAMESPACE: 'ingest.profiling.passthrough' - TASKBROKER_RAW_APPLICATION: 'sentry' - TASKBROKER_RAW_TASKNAME: 'sentry.profiles.task.process_profile_from_kafka' # TODO(STREAM-1041): Remove debug logging once passthrough mode is stable TASKBROKER_LOG_FILTER: 'debug' ports: