diff --git a/devservices/config.yml b/devservices/config.yml index 176478efee9dd4..b935865d5ef39e 100644 --- a/devservices/config.yml +++ b/devservices/config.yml @@ -497,14 +497,31 @@ services: ingest-profiles: image: ghcr.io/getsentry/taskbroker:nightly environment: - TASKBROKER_KAFKA_CLUSTER: 'kafka-kafka-1:9093' - TASKBROKER_KAFKA_TOPIC: 'profiles' - TASKBROKER_KAFKA_CONSUMER_GROUP: 'ingest-profiles' + # Multi-topic/cluster config via env vars: nested keys use "__", and + # figment lowercases each segment, so topic/cluster names can only use + # underscores (not hyphens) when set this way. + TASKBROKER_KAFKA_CLUSTERS__DEFAULT__ADDRESS: 'kafka-kafka-1:9093' + # Consumed topic, in raw (passthrough) mode. Raw mode is configured per + # topic via the topic's `raw` block. + TASKBROKER_KAFKA_TOPICS__PROFILES__CLUSTER: 'default' + TASKBROKER_KAFKA_TOPICS__PROFILES__CONSUMER_GROUP: 'ingest-profiles' + TASKBROKER_KAFKA_TOPICS__PROFILES__RAW__NAMESPACE: 'ingest.profiling.passthrough' + TASKBROKER_KAFKA_TOPICS__PROFILES__RAW__APPLICATION: 'sentry' + TASKBROKER_KAFKA_TOPICS__PROFILES__RAW__TASKNAME: 'sentry.profiles.task.process_profile_from_kafka' + TASKBROKER_KAFKA_TOPICS__PROFILES__RAW__PROCESSING_DEADLINE_DURATION: '30' + # A raw topic's messages aren't activations, so retries (which are + # activation-encoded) cannot loop back into `profiles`; a separate retry + # topic is mandatory. Send them to the main `taskworker` topic so the + # default taskbroker picks them up instead of standing up another broker. + TASKBROKER_KAFKA_RETRY_TOPIC: 'taskworker' + TASKBROKER_KAFKA_TOPICS__TASKWORKER__CLUSTER: 'default' + TASKBROKER_KAFKA_TOPICS__TASKWORKER__CONSUMER_GROUP: 'ingest-profiles' + TASKBROKER_KAFKA_TOPICS__TASKWORKER__PRODUCE_ONLY: 'true' + TASKBROKER_KAFKA_DEADLETTER_TOPIC: 'taskworker_dlq' + TASKBROKER_KAFKA_TOPICS__TASKWORKER_DLQ__CLUSTER: 'default' + TASKBROKER_KAFKA_TOPICS__TASKWORKER_DLQ__CONSUMER_GROUP: 'ingest-profiles' + TASKBROKER_KAFKA_TOPICS__TASKWORKER_DLQ__PRODUCE_ONLY: 'true' TASKBROKER_CREATE_MISSING_TOPICS: 'true' - TASKBROKER_RAW_MODE: 'true' - TASKBROKER_RAW_NAMESPACE: 'ingest.profiling.passthrough' - TASKBROKER_RAW_APPLICATION: 'sentry' - TASKBROKER_RAW_TASKNAME: 'sentry.profiles.task.process_profile_from_kafka' # TODO(STREAM-1041): Remove debug logging once passthrough mode is stable TASKBROKER_LOG_FILTER: 'debug' ports: