feat(profiling): Support sample v2 profiles for Android

fixtures/profiles/valid_android_profile.json

@@ -17,7 +17,9 @@
   "environment": "debug",
   "platform": "android",
   "profile_id": "9bd24eb4-fd81-4a00-8dde-fd035d363570",
-  "profile": {},
+  "profile": {
+    "methods": []
+  },
   "trace_id": "eb5d1263c7924c6aac12b5bcf2132cf6",
   "transaction_id": "7687200252294dc2af6d472cd17a828d",
   "transaction_name": "SecondActivity",

pyproject.toml

@@ -118,7 +118,7 @@ dependencies = [
   # note: we cannot resolve urllib3[brotli] but brotli is installed
   #       and will be used
   "urllib3>=2.7.0",
-  "vroomrs>=0.1.21",
+  "vroomrs>=0.1.22",
   "xmlsec>=1.3.17",
   "zstandard>=0.18.0",
   # [begin] getsentry

src/sentry/profiles/java.py

@@ -2,6 +2,8 @@
 
 from symbolic.proguard import ProguardMapper
 
+from sentry.profiles.utils import Profile, is_android_trace_format, is_jvm_frame
+
 JAVA_BASE_TYPES = {
     "Z": "boolean",
     "B": "byte",
@@ -114,22 +116,47 @@ def deobfuscate_signature(
     return parameter_java_types, return_java_type
 
 
-def convert_android_methods_to_jvm_frames(methods: list[dict[str, Any]]) -> list[dict[str, Any]]:
+def convert_android_methods_to_jvm_frames(profile: Profile) -> list[dict[str, Any]]:
     frames = []
-    for i, m in enumerate(methods):
-        f = {
-            "function": m["name"],
-            "index": i,
-            "module": m["class_name"],
-        }
-        if "signature" in m:
-            f["signature"] = m["signature"]
-        if "source_line" in m:
-            f["lineno"] = m["source_line"]
-        if "source_file" in m:
-            f["filename"] = m["source_file"]
-        frames.append(f)
-    return frames
+
+    if is_android_trace_format(profile):
+        methods = profile["profile"]["methods"]
+        for i, m in enumerate(methods):
+            f = {
+                "function": m["name"],
+                "index": i,
+                "module": m["class_name"],
+            }
+            if "signature" in m:
+                f["signature"] = m["signature"]
+            if "source_line" in m:
+                f["lineno"] = m["source_line"]
+            if "source_file" in m:
+                f["filename"] = m["source_file"]
+            frames.append(f)
+        return frames
+    else:
+        # sample v2: JVM frames live in profile["profile"]["frames"] and are
+        # identified via is_jvm_frame. `index` records each JVM frame's position
+        # in the original frames list so the results can be merged back.
+        for i, f in enumerate(profile["profile"]["frames"]):
+            if not is_jvm_frame(f, profile):
+                continue
+            # function and module are optional on sample format frames; keep
+            # frames missing one of them so the other still gets deobfuscated
+            jvm_frame = {
+                "function": f.get("function", ""),
+                "index": i,
+                "module": f.get("module", ""),
+            }
+            if "signature" in f:
+                jvm_frame["signature"] = f["signature"]
+            if "lineno" in f:
+                jvm_frame["lineno"] = f["lineno"]
+            if "filename" in f:
+                jvm_frame["filename"] = f["filename"]
+            frames.append(jvm_frame)
+        return frames
 
 
 def _merge_jvm_frame_and_android_method(f: dict[str, Any], m: dict[str, Any]) -> None:
@@ -146,20 +173,69 @@ def _merge_jvm_frame_and_android_method(f: dict[str, Any], m: dict[str, Any]) ->
         m["in_app"] = f["in_app"]
 
 
-def merge_jvm_frames_with_android_methods(
-    frames: list[dict[str, Any]], methods: list[dict[str, Any]]
-) -> None:
-    for f in frames:
-        m = methods[f["index"]]
-        # Update the method if it's the first time we see it.
-        if m.get("data", {}).get("deobfuscation_status", "") != "deobfuscated":
-            _merge_jvm_frame_and_android_method(f, m)
-        # Otherwise, it's an additional method returned, we add it to the inline frames.
-        else:
-            # We copy the frame triggering the inline ones so we only have to
-            # look at this field later one to construct a stack trace.
-            if "inline_frames" not in m:
-                m["inline_frames"] = [m.copy()]
-            im: dict[str, Any] = {}
-            _merge_jvm_frame_and_android_method(f, im)
-            m["inline_frames"].append(im)
+def _apply_jvm_frame_to_sample_v2_frame(f: dict[str, Any], frame: dict[str, Any]) -> None:
+    frame["module"] = f["module"]
+    frame["function"] = f["function"]
+    frame["data"] = {"deobfuscation_status": "deobfuscated"}
+    if "signature" in f:
+        frame["signature"] = f["signature"]
+    if "filename" in f:
+        frame["filename"] = f["filename"]
+    if "lineno" in f and f["lineno"] != 0:
+        frame["lineno"] = f["lineno"]
+    if "in_app" in f:
+        frame["in_app"] = f["in_app"]
+
+
+def merge_jvm_frames_with_android_methods(frames: list[dict[str, Any]], profile: Profile) -> None:
+    if is_android_trace_format(profile):
+        methods = profile["profile"]["methods"]
+        for f in frames:
+            m = methods[f["index"]]
+            # Update the method if it's the first time we see it.
+            if m.get("data", {}).get("deobfuscation_status", "") != "deobfuscated":
+                _merge_jvm_frame_and_android_method(f, m)
+            # Otherwise, it's an additional method returned, we add it to the inline frames.
+            else:
+                # We copy the frame triggering the inline ones so we only have to
+                # look at this field later one to construct a stack trace.
+                if "inline_frames" not in m:
+                    m["inline_frames"] = [m.copy()]
+                im: dict[str, Any] = {}
+                _merge_jvm_frame_and_android_method(f, im)
+                m["inline_frames"].append(im)
+    else:
+        _merge_jvm_frames_with_sample_v2(frames, profile)
+
+
+def _merge_jvm_frames_with_sample_v2(jvm_frames: list[dict[str, Any]], profile: Profile) -> None:
+    # Symbolicator may return several frames for a single input frame (inlines).
+    # Sample v2 has no per-frame `inline_frames`; inlining is expressed by
+    # expanding the frame list and remapping the stacks that reference it.
+    deobf_by_index: dict[int, list[dict[str, Any]]] = {}
+    for f in jvm_frames:
+        deobf_by_index.setdefault(f["index"], []).append(f)
+
+    original_frames = profile["profile"]["frames"]
+    new_frames: list[dict[str, Any]] = []
+    # original frame index -> list of indices in the rebuilt frame list
+    index_map: dict[int, list[int]] = {}
+    for old_index, frame in enumerate(original_frames):
+        deobf = deobf_by_index.get(old_index)
+        if not deobf:
+            index_map[old_index] = [len(new_frames)]
+            new_frames.append(frame)
+            continue
+        new_indices = []
+        for jvm_frame in deobf:
+            merged = dict(frame)
+            _apply_jvm_frame_to_sample_v2_frame(jvm_frame, merged)
+            new_indices.append(len(new_frames))
+            new_frames.append(merged)
+        index_map[old_index] = new_indices
+
+    profile["profile"]["stacks"] = [
+        [new_index for old_index in stack for new_index in index_map.get(old_index, [old_index])]
+        for stack in profile["profile"]["stacks"]
+    ]
+    profile["profile"]["frames"] = new_frames

src/sentry/profiles/task.py

@@ -59,8 +59,10 @@
     merge_jvm_frames_with_android_methods,
 )
 from sentry.profiles.utils import (
+    PROFILE_FORMAT_V2_ANDROID_TRACE,
     Profile,
     apply_stack_trace_rules_to_profile,
+    is_android_trace_format,
 )
 from sentry.search.utils import DEVICE_CLASS
 from sentry.signals import first_profile_received
@@ -286,16 +288,20 @@ def process_profile_task(
     sentry_sdk.set_tag("platform", profile["platform"])
     sentry_sdk.set_attribute("platform", profile["platform"])
 
-    if "version" in profile:
-        version = profile["version"]
+    version = profile.get("version")
+
+    if is_android_trace_format(profile):
+        # Android trace format is sent both as legacy transaction profiles and as
+        # continuous-profiling chunks, the latter identified by a profiler_id.
+        fmt = "android_chunk" if "profiler_id" in profile else "legacy"
+        sentry_sdk.set_tag("format", fmt)
+        sentry_sdk.set_attribute("format", fmt)
+    elif version is not None:
         sentry_sdk.set_tag("format", f"sample_v{version}")
         sentry_sdk.set_attribute("format", f"sample_v{version}")
         set_span_attribute("profile.samples", len(profile["profile"]["samples"]))
         set_span_attribute("profile.stacks", len(profile["profile"]["stacks"]))
         set_span_attribute("profile.frames", len(profile["profile"]["frames"]))
-    elif "profiler_id" in profile and profile["platform"] == "android":
-        sentry_sdk.set_tag("format", "android_chunk")
-        sentry_sdk.set_attribute("format", "android_chunk")
     else:
         sentry_sdk.set_tag("format", "legacy")
         sentry_sdk.set_attribute("format", "legacy")
@@ -319,7 +325,7 @@ def process_profile_task(
     # only for those platforms that didn't go through symbolication
     _set_frames_platform(profile)
 
-    if "version" in profile:
+    if version is not None and not is_android_trace_format(profile):
         set_span_attribute("profile.samples.processed", len(profile["profile"]["samples"]))
         set_span_attribute("profile.stacks.processed", len(profile["profile"]["stacks"]))
         set_span_attribute("profile.frames.processed", len(profile["profile"]["frames"]))
@@ -627,7 +633,12 @@ def _normalize(profile: Profile, organization: Organization) -> None:
     platform = profile["platform"]
     version = profile.get("version")
 
-    if platform not in {"cocoa", "android"} or version == "2":
+    # Skip unsupported platforms and sample v2 profiles, which don't carry device
+    # classification. The version can't be trusted on android though, so only skip
+    # genuine sample v2 profiles there and not the (faulty-version) legacy format.
+    if platform not in {"cocoa", "android"} or (
+        version == "2" and not is_android_trace_format(profile)
+    ):
         return
 
     classification = profile.get("transaction_tags", {}).get("device.class", None)
@@ -1084,11 +1095,7 @@ def on_symbolicator_request() -> None:
                     }
                 ],
                 stacktraces=[
-                    {
-                        "frames": convert_android_methods_to_jvm_frames(
-                            profile["profile"]["methods"]
-                        )
-                    },
+                    {"frames": convert_android_methods_to_jvm_frames(profile)},
                 ],
                 # Methods in a profile aren't inherently ordered, but the order of returned
                 # inlinees should be caller first.
@@ -1111,7 +1118,7 @@ def on_symbolicator_request() -> None:
                 if "stacktraces" in response:
                     merge_jvm_frames_with_android_methods(
                         frames=response["stacktraces"][0]["frames"],
-                        methods=profile["profile"]["methods"],
+                        profile=profile,
                     )
                     return True
             else:
@@ -1137,12 +1144,16 @@ def get_debug_file_id(profile: Profile) -> str | None:
 @metrics.wraps("process_profile.deobfuscate")
 def _deobfuscate(profile: Profile, project: Project) -> None:
     debug_file_id = get_debug_file_id(profile)
+
+    # if no proguard mapping was provided, we still need to decode the
+    # signatures on the legacy android trace format; sample v2 frames don't
+    # carry signatures, so there's nothing to do there.
     if debug_file_id is None:
-        # we still need to decode signatures
-        for m in profile["profile"]["methods"]:
-            if m.get("signature"):
-                types = deobfuscate_signature(m["signature"])
-                m["signature"] = format_signature(types)
+        if is_android_trace_format(profile):
+            for m in profile["profile"]["methods"]:
+                if m.get("signature"):
+                    types = deobfuscate_signature(m["signature"])
+                    m["signature"] = format_signature(types)
         return
 
     try:
@@ -1168,16 +1179,6 @@ def get_event_id(profile: Profile) -> str:
     return profile["event_id"]
 
 
-def get_data_category(profile: Profile) -> DataCategory:
-    if profile.get("version") == "2":
-        return (
-            DataCategory.PROFILE_CHUNK_UI
-            if profile["platform"] in UI_PROFILE_PLATFORMS
-            else DataCategory.PROFILE_CHUNK
-        )
-    return DataCategory.PROFILE_INDEXED
-
-
 @metrics.wraps("process_profile.track_outcome")
 def _track_outcome(
     profile: Profile,
@@ -1265,16 +1266,13 @@ def _get_duration_category(profile: Profile) -> DataCategory:
 
 
 def _calculate_profile_duration_ms(profile: Profile) -> int:
+    if is_android_trace_format(profile):
+        return _calculate_duration_for_android_format(profile)
     version = profile.get("version")
-    if version:
-        if version == "1":
-            return _calculate_duration_for_sample_format_v1(profile)
-        elif version == "2":
-            return _calculate_duration_for_sample_format_v2(profile)
-    else:
-        platform = profile["platform"]
-        if platform == "android":
-            return _calculate_duration_for_android_format(profile)
+    if version == "1":
+        return _calculate_duration_for_sample_format_v1(profile)
+    elif version == "2":
+        return _calculate_duration_for_sample_format_v2(profile)
     return 0
 
 
@@ -1329,7 +1327,9 @@ def _calculate_duration_for_android_format(profile: Profile) -> int:
 def _set_frames_platform(profile: Profile) -> None:
     platform = profile["platform"]
     frames = (
-        profile["profile"]["methods"] if platform == "android" else profile["profile"]["frames"]
+        profile["profile"]["methods"]
+        if is_android_trace_format(profile)
+        else profile["profile"]["frames"]
     )
     for f in frames:
         if "platform" not in f:
@@ -1345,18 +1345,18 @@ class UnknownClientSDKException(Exception):
 
 
 def determine_profile_type(profile: Profile) -> EventType:
-    if "version" in profile:
-        version = profile["version"]
-        if version == "1":
-            return EventType.PROFILE
-        elif version == "2":
-            return EventType.PROFILE_CHUNK
-    elif profile["platform"] == "android":
+    if is_android_trace_format(profile):
         if "profiler_id" in profile:
             return EventType.PROFILE_CHUNK
         else:
             # This is the legacy android format
             return EventType.PROFILE
+
+    version = profile.get("version")
+    if version == "1":
+        return EventType.PROFILE
+    elif version == "2":
+        return EventType.PROFILE_CHUNK
     raise UnknownProfileTypeException
 
 
@@ -1583,7 +1583,19 @@ def _process_vroomrs_chunk_profile(profile: Profile, project: Project) -> bool:
                     tags={"type": "chunk", "platform": profile["platform"]},
                 )
             with start_span(op="json.unmarshal", name="json.unmarshal"):
-                chunk = vroomrs.profile_chunk_from_json_str(json_profile, profile["platform"])
+                # Detect the android trace format before trusting `version`,
+                # analogous to how `symbolicate()` special-cases android: a
+                # faulty version can't be relied on, so a trace profile is
+                # always deserialized as "2.android-trace".
+                version = profile.get("version")
+                if is_android_trace_format(profile):
+                    chunk = vroomrs.profile_chunk_from_json_str_and_version(
+                        json_profile, PROFILE_FORMAT_V2_ANDROID_TRACE
+                    )
+                elif version is not None:
+                    chunk = vroomrs.profile_chunk_from_json_str_and_version(json_profile, version)
+                else:
+                    chunk = vroomrs.profile_chunk_from_json_str(json_profile, profile["platform"])
             chunk.normalize()
             with start_span(op="gcs.write", name="compress and write"):
                 storage = get_profiles_storage()