Skip to content

gh-150389: Make perf profiler tests resilient #150437

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pablogsal wants to merge 1 commit into
base: main
Choose a base branch
from
+104 −62
Open

gh-150389: Make perf profiler tests resilient #150437

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 104 additions & 62 deletions Lib/test/test_perf_profiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,21 @@ def supports_trampoline_profiling():
raise unittest.SkipTest("perf trampoline profiling not supported")


def _perf_env(**env_vars):
env = os.environ.copy()
# Keep perf's output stable regardless of the builder's perf config.
env.update(
{
"DEBUGINFOD_URLS": "",
"PERF_CONFIG": os.devnull,
}
)
if env_vars:
env.update(env_vars)
env["PYTHON_JIT"] = "0"
return env


class TestPerfTrampoline(unittest.TestCase):
def setUp(self):
super().setUp()
Expand Down Expand Up @@ -63,13 +78,12 @@ def baz():
"""
with temp_dir() as script_dir:
script = make_script(script_dir, "perftest", code)
env = {**os.environ, "PYTHON_JIT": "0"}
with subprocess.Popen(
[sys.executable, "-Xperf", script],
text=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
env=env,
env=_perf_env(),
) as process:
stdout, stderr = process.communicate()

Expand Down Expand Up @@ -132,13 +146,12 @@ def baz():
"""
with temp_dir() as script_dir:
script = make_script(script_dir, "perftest", code)
env = {**os.environ, "PYTHON_JIT": "0"}
with subprocess.Popen(
[sys.executable, "-Xperf", script],
text=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
env=env,
env=_perf_env(),
) as process:
stdout, stderr = process.communicate()

Expand Down Expand Up @@ -198,13 +211,12 @@ def test_trampoline_works_after_fork_with_many_code_objects(self):
"""
with temp_dir() as script_dir:
script = make_script(script_dir, "perftest", code)
env = {**os.environ, "PYTHON_JIT": "0"}
with subprocess.Popen(
[sys.executable, "-Xperf", script],
text=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
env=env,
env=_perf_env(),
) as process:
stdout, stderr = process.communicate()

Expand Down Expand Up @@ -242,13 +254,12 @@ def baz():
code = set_eval_hook + code
with temp_dir() as script_dir:
script = make_script(script_dir, "perftest", code)
env = {**os.environ, "PYTHON_JIT": "0"}
with subprocess.Popen(
[sys.executable, script],
text=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
env=env,
env=_perf_env(),
) as process:
stdout, stderr = process.communicate()

Expand Down Expand Up @@ -345,9 +356,12 @@ def perf_command_works():
"-c",
'print("hello")',
)
env = {**os.environ, "PYTHON_JIT": "0"}
stdout = subprocess.check_output(
cmd, cwd=script_dir, text=True, stderr=subprocess.STDOUT, env=env
cmd,
cwd=script_dir,
text=True,
stderr=subprocess.STDOUT,
env=_perf_env(),
)
except (subprocess.SubprocessError, OSError):
return False
Expand All @@ -359,43 +373,49 @@ def perf_command_works():


def run_perf(cwd, *args, use_jit=False, **env_vars):
env = os.environ.copy()
if env_vars:
env.update(env_vars)
env["PYTHON_JIT"] = "0"
env = _perf_env(**env_vars)
output_file = cwd + "/perf_output.perf"
if not use_jit:
base_cmd = (
"perf",
"record",
"--no-buildid",
"--no-buildid-cache",
"-g",
"--call-graph=fp",
"-o", output_file,
"--"
)
base_cmd = [
"perf",
"record",
"--no-buildid",
"--no-buildid-cache",
"-g",
"--call-graph=dwarf,65528" if use_jit else "--call-graph=fp",
]
if use_jit:
perf_commands = []
# Some builders have low perf_event_mlock_kb limits.
mmap_sizes = ("4M", "2M", "1M", "512K", "256K", "128K", None)
for mmap_size in mmap_sizes:
command = base_cmd.copy()
if mmap_size is not None:
command += ["-F99", "-k1", "-m", mmap_size]
else:
command += ["-F99", "-k1"]
command += ["-o", output_file, "--"]
perf_commands.append(command)
else:
base_cmd = (
"perf",
"record",
"--no-buildid",
"--no-buildid-cache",
"-g",
"--call-graph=dwarf,65528",
"-F99",
"-k1",
"-o",
output_file,
"--",
perf_commands = [base_cmd + ["-o", output_file, "--"]]

mmap_pages_error = "try again with a smaller value of -m/--mmap_pages"
for index, base_cmd in enumerate(perf_commands):
proc = subprocess.run(
base_cmd + list(args),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
text=True,
)
proc = subprocess.run(
base_cmd + args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
text=True,
)
if (
proc.returncode
and use_jit
and index != len(perf_commands) - 1
and mmap_pages_error in proc.stderr
):
continue
break

if proc.returncode:
print(proc.stderr, file=sys.stderr)
raise ValueError(f"Perf failed with return code {proc.returncode}")
Expand Down Expand Up @@ -425,54 +445,77 @@ def run_perf(cwd, *args, use_jit=False, **env_vars):


class TestPerfProfilerMixin:
def run_perf(self, script_dir, perf_mode, script):
PERF_CAPTURE_ATTEMPTS = 3

def run_perf(self, script_dir, script, activate_trampoline=True):
raise NotImplementedError()

def run_perf_with_retries(
self, script_dir, script, expected_symbols=(), activate_trampoline=True
):
stdout = stderr = ""
for _ in range(self.PERF_CAPTURE_ATTEMPTS):
stdout, stderr = self.run_perf(
script_dir, script, activate_trampoline=activate_trampoline
)
if activate_trampoline and any(
symbol not in stdout for symbol in expected_symbols
):
continue
break
return stdout, stderr

def test_python_calls_appear_in_the_stack_if_perf_activated(self):
with temp_dir() as script_dir:
code = """if 1:
from itertools import repeat

def foo(n):
x = 0
for i in range(n):
x += i
for _ in repeat(None, n):
pass

def bar(n):
foo(n)

def baz(n):
bar(n)

baz(10000000)
baz(40000000)
"""
script = make_script(script_dir, "perftest", code)
stdout, stderr = self.run_perf(script_dir, script)
self.assertEqual(stderr, "")
expected_symbols = [
f"py::foo:{script}",
f"py::bar:{script}",
f"py::baz:{script}",
]
stdout, _ = self.run_perf_with_retries(
script_dir, script, expected_symbols
)

self.assertIn(f"py::foo:{script}", stdout)
self.assertIn(f"py::bar:{script}", stdout)
self.assertIn(f"py::baz:{script}", stdout)
for expected_symbol in expected_symbols:
self.assertIn(expected_symbol, stdout)

def test_python_calls_do_not_appear_in_the_stack_if_perf_deactivated(self):
with temp_dir() as script_dir:
code = """if 1:
from itertools import repeat

def foo(n):
x = 0
for i in range(n):
x += i
for _ in repeat(None, n):
pass

def bar(n):
foo(n)

def baz(n):
bar(n)

baz(10000000)
baz(40000000)
"""
script = make_script(script_dir, "perftest", code)
stdout, stderr = self.run_perf(
stdout, _ = self.run_perf_with_retries(
script_dir, script, activate_trampoline=False
)
self.assertEqual(stderr, "")

self.assertNotIn(f"py::foo:{script}", stdout)
self.assertNotIn(f"py::bar:{script}", stdout)
Expand Down Expand Up @@ -542,13 +585,12 @@ def compile_trampolines_for_all_functions():

with temp_dir() as script_dir:
script = make_script(script_dir, "perftest", code)
env = {**os.environ, "PYTHON_JIT": "0"}
with subprocess.Popen(
[sys.executable, "-Xperf", script],
universal_newlines=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
env=env,
env=_perf_env(),
) as process:
stdout, stderr = process.communicate()

Expand Down
Loading