diff --git a/instrumentation.ts b/instrumentation.ts index e9326579d3ee4..d52af7b7ea36b 100644 --- a/instrumentation.ts +++ b/instrumentation.ts @@ -1,5 +1,4 @@ import * as Sentry from '@sentry/nextjs'; - import {tracesSampler} from 'sentry-docs/tracesSampler'; export function register() { @@ -51,20 +50,6 @@ export function register() { } return metric; }, - - // temporary change for investigating edge middleware tx names - beforeSendTransaction(event) { - if ( - event.transaction?.includes('middleware GET') && - event.contexts?.trace?.data - ) { - event.contexts.trace.data = { - ...event.contexts.trace.data, - 'sentry.source': 'custom', - }; - } - return event; - }, }); } } diff --git a/middleware.ts b/middleware.ts index fc00a973fb14f..067ee41eaa760 100644 --- a/middleware.ts +++ b/middleware.ts @@ -1,14 +1,20 @@ import * as Sentry from '@sentry/nextjs'; import type {NextRequest} from 'next/server'; import {NextResponse, userAgent} from 'next/server'; -import {AI_AGENT_PATTERN, type TrafficType} from 'sentry-docs/lib/trafficClassification'; +import { + AI_AGENT_PATTERN, + matchPattern, + type TrafficType, +} from 'sentry-docs/lib/trafficClassification'; // DEVELOPER_DOCS is set via next.config.ts env field (inlined at build time for edge runtime). // NEXT_PUBLIC_DEVELOPER_DOCS is the canonical env var; DEVELOPER_DOCS is the build-time alias. const isDeveloperDocs = process.env.DEVELOPER_DOCS || process.env.NEXT_PUBLIC_DEVELOPER_DOCS; -const BASE_URL = isDeveloperDocs ? 'https://develop.sentry.dev' : 'https://docs.sentry.io'; +const BASE_URL = isDeveloperDocs + ? 'https://develop.sentry.dev' + : 'https://docs.sentry.io'; export const config = { // learn more: https://nextjs.org/docs/pages/building-your-application/routing/middleware#matcher @@ -24,6 +30,14 @@ export const config = { // This function can be marked `async` if using `await` inside export function middleware(request: NextRequest) { + // Classify once per request and record it as a counter. This metric — not + // trace sampling — is the source of truth for agent/bot/user traffic: the + // middleware root span is created by Next.js before any request data reaches + // Sentry, so classification is impossible at trace-sampling time (see + // src/tracesSampler.ts). + const classification = classifyTraffic(request); + recordClassification(request, classification); + // First, handle canonical URL redirects for deprecated paths const canonicalRedirect = handleRedirects(request); if (canonicalRedirect) { @@ -31,7 +45,32 @@ export function middleware(request: NextRequest) { } // Then, check for AI/LLM clients and redirect to markdown if appropriate - return handleAIClientRedirect(request); + return handleAIClientRedirect(request, classification); +} + +type TrafficClassification = ReturnType; + +/** + * Records the per-request traffic classification as a counter metric. + * Every request the middleware sees is counted — no sampling — making this + * the system of record for agent/bot/user traffic on the docs site. + */ +function recordClassification( + request: NextRequest, + classification: TrafficClassification +): void { + const agent = + classification.trafficType === 'ai_agent' + ? matchPattern(request.headers.get('user-agent') ?? '', AI_AGENT_PATTERN) + : undefined; + + Sentry.metrics.count('docs.request.classified', 1, { + attributes: { + traffic_type: classification.trafficType, + device_type: classification.deviceType, + ...(agent && {agent}), + }, + }); } // don't send Permanent Redirects (301) in dev mode - it gets cached for "localhost" by the browser @@ -123,8 +162,10 @@ function wantsMarkdown(request: NextRequest): boolean { * These headers are added to the REQUEST (not response) so tracesSampler can read them. * Uses NextResponse.next({ request: { headers } }) pattern to modify the request. */ -function createClassifiedRequestHeaders(request: NextRequest): Headers { - const classification = classifyTraffic(request); +function createClassifiedRequestHeaders( + request: NextRequest, + classification: TrafficClassification +): Headers { const headers = new Headers(request.headers); headers.set('x-traffic-type', classification.trafficType); headers.set('x-device-type', classification.deviceType); @@ -134,10 +175,13 @@ function createClassifiedRequestHeaders(request: NextRequest): Headers { /** * Creates a pass-through response with traffic classification headers on the request. */ -function nextWithClassification(request: NextRequest): NextResponse { +function nextWithClassification( + request: NextRequest, + classification: TrafficClassification +): NextResponse { return NextResponse.next({ request: { - headers: createClassifiedRequestHeaders(request), + headers: createClassifiedRequestHeaders(request, classification), }, }); } @@ -145,10 +189,14 @@ function nextWithClassification(request: NextRequest): NextResponse { /** * Creates a rewrite response with traffic classification headers on the request. */ -function rewriteWithClassification(request: NextRequest, destination: URL): NextResponse { +function rewriteWithClassification( + request: NextRequest, + destination: URL, + classification: TrafficClassification +): NextResponse { return NextResponse.rewrite(destination, { request: { - headers: createClassifiedRequestHeaders(request), + headers: createClassifiedRequestHeaders(request, classification), }, }); } @@ -169,7 +217,10 @@ function mdToCanonicalPath(mdPathname: string): string { /** * Handles redirection to markdown versions for AI/LLM clients */ -const handleAIClientRedirect = (request: NextRequest) => { +const handleAIClientRedirect = ( + request: NextRequest, + classification: TrafficClassification +) => { const userAgentString = request.headers.get('user-agent') || ''; const acceptHeader = request.headers.get('accept') || ''; const url = request.nextUrl; @@ -207,8 +258,11 @@ const handleAIClientRedirect = (request: NextRequest) => { // engine crawlers consolidate ranking to the human-readable URL instead of indexing the // raw markdown. AI agents don't act on this header, so LLM ingestion is unaffected. if (url.pathname.endsWith('.md')) { - const response = nextWithClassification(request); - response.headers.set('Link', `<${BASE_URL}${mdToCanonicalPath(url.pathname)}>; rel="canonical"`); + const response = nextWithClassification(request, classification); + response.headers.set( + 'Link', + `<${BASE_URL}${mdToCanonicalPath(url.pathname)}>; rel="canonical"` + ); return response; } @@ -221,7 +275,7 @@ const handleAIClientRedirect = (request: NextRequest) => { url.pathname ) ) { - return nextWithClassification(request); + return nextWithClassification(request, classification); } // Check for markdown request (Accept header, user-agent, or manual) @@ -248,11 +302,11 @@ const handleAIClientRedirect = (request: NextRequest) => { // Rewrite to serve markdown inline (same URL, different content) // The next.config.ts rewrite rule maps *.md to /md-exports/*.md - return rewriteWithClassification(request, newUrl); + return rewriteWithClassification(request, newUrl, classification); } // Default: pass through with traffic classification headers - return nextWithClassification(request); + return nextWithClassification(request, classification); }; const handleRedirects = (request: NextRequest) => { diff --git a/src/tracesSampler.ts b/src/tracesSampler.ts index f4a53ead9d8c5..fcb77a2ac71b1 100644 --- a/src/tracesSampler.ts +++ b/src/tracesSampler.ts @@ -1,5 +1,3 @@ -import * as Sentry from '@sentry/nextjs'; - import { AI_AGENT_PATTERN, BOT_PATTERN, @@ -41,107 +39,81 @@ function getHeaderValue( const VALID_TRAFFIC_TYPES = new Set(['ai_agent', 'bot', 'user', 'unknown']); /** - * Gets middleware traffic classification from headers. - * Validates that traffic type is one of the expected values. + * Gets the traffic classification the middleware stamped onto the forwarded + * request via the `x-traffic-type` header (see middleware.ts). */ -function getMiddlewareClassification(headers?: Record): { - deviceType: string | undefined; - trafficType: TrafficType | undefined; -} { +function getForwardedTrafficType( + headers?: Record +): TrafficType | undefined { const rawTrafficType = getHeaderValue(headers, 'x-traffic-type'); - return { - deviceType: getHeaderValue(headers, 'x-device-type'), - trafficType: VALID_TRAFFIC_TYPES.has(rawTrafficType as TrafficType) - ? (rawTrafficType as TrafficType) - : undefined, - }; + return VALID_TRAFFIC_TYPES.has(rawTrafficType as TrafficType) + ? (rawTrafficType as TrafficType) + : undefined; +} + +/** + * Middleware root spans are created by Next.js itself ('Middleware.execute') + * before any request data reaches Sentry, so they can never be classified + * here — and they carry no useful detail (the name is collapsed to + * `middleware GET`). Per-request traffic classification is recorded as the + * `docs.request.classified` metric in middleware.ts instead. + */ +function isMiddlewareRootSpan(samplingContext: SamplingContext): boolean { + return ( + samplingContext.attributes?.['next.span_type'] === 'Middleware.execute' || + samplingContext.attributes?.['sentry.op'] === 'http.server.middleware' || + samplingContext.name === 'middleware' || + Boolean(samplingContext.name?.startsWith('middleware ')) + ); } /** * Determines trace sample rate based on traffic classification. - * Uses middleware classification headers when available, falls back to user-agent pattern matching. * * Sample rates (from shared config): + * - Middleware root spans: 0% (unclassifiable by architecture and information-free; + * traffic counting happens in middleware.ts via the docs.request.classified metric) * - AI agents: 100% (full visibility into agentic docs consumption) * - Bots/crawlers: 0% (filter out noise) * - Real users: 30% * - Unknown: 30% (tracked separately for visibility) * - * AI agents are checked first; if something matches both AI and bot patterns, we sample it. + * Classification prefers the `x-traffic-type` header stamped by the middleware + * onto forwarded requests, falling back to user-agent pattern matching for + * requests that didn't pass through the middleware. AI agents are checked + * before bots; if something matches both patterns, we sample it. */ export function tracesSampler(samplingContext: SamplingContext): number { - const headers = samplingContext.normalizedRequest?.headers; - - // Check for middleware classification headers first (most reliable) - const middlewareClassification = getMiddlewareClassification(headers); - - // If middleware provided valid classification, use it - if (middlewareClassification.trafficType) { - const {trafficType, deviceType} = middlewareClassification; - const sampleRate = SAMPLE_RATES[trafficType]; + if (isMiddlewareRootSpan(samplingContext)) { + return 0; + } - Sentry.metrics.count('docs.trace.sampled', 1, { - attributes: { - sample_rate: sampleRate, - traffic_type: trafficType, - device_type: deviceType || 'unknown', - }, - }); + const headers = samplingContext.normalizedRequest?.headers; - return sampleRate; + // Trust the classification the middleware stamped onto the forwarded request + const forwardedType = getForwardedTrafficType(headers); + if (forwardedType) { + return SAMPLE_RATES[forwardedType]; } // Fallback to user-agent pattern matching if no middleware classification - // This handles cases where middleware didn't run (API routes, etc.) + // (e.g. requests that bypass the middleware matcher) const userAgent = getHeaderValue(headers, 'user-agent') ?? (samplingContext.attributes?.['http.user_agent'] as string | undefined) ?? (samplingContext.attributes?.['user_agent.original'] as string | undefined); - // No user-agent = unknown traffic, track it explicitly if (!userAgent) { - Sentry.metrics.count('docs.trace.sampled', 1, { - attributes: { - sample_rate: SAMPLE_RATES.unknown, - traffic_type: 'unknown', - device_type: 'unknown', - }, - }); return SAMPLE_RATES.unknown; } - // Check for AI agents first - const aiAgent = matchPattern(userAgent, AI_AGENT_PATTERN); - if (aiAgent) { - Sentry.metrics.count('docs.trace.sampled', 1, { - attributes: { - sample_rate: SAMPLE_RATES.ai_agent, - traffic_type: 'ai_agent', - agent_match: aiAgent, - }, - }); + if (matchPattern(userAgent, AI_AGENT_PATTERN)) { return SAMPLE_RATES.ai_agent; } - // Check for bots/crawlers - const bot = matchPattern(userAgent, BOT_PATTERN); - if (bot) { - Sentry.metrics.count('docs.trace.sampled', 1, { - attributes: { - sample_rate: SAMPLE_RATES.bot, - traffic_type: 'bot', - bot_match: bot, - }, - }); + if (matchPattern(userAgent, BOT_PATTERN)) { return SAMPLE_RATES.bot; } - // Sample real users at default rate - Sentry.metrics.count('docs.trace.sampled', 1, { - attributes: { - sample_rate: SAMPLE_RATES.user, - traffic_type: 'user', - }, - }); return SAMPLE_RATES.user; }