Skip to content

feat(serve): list/filter jobs, deploy from recommendation row, compare benchmarks, DataFrame views #6148

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ZealSV wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat(serve): list/filter jobs, deploy from recommendation row, compare benchmarks, DataFrame views #6148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions sagemaker-serve/src/sagemaker/serve/ai_inference_recommender/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,17 @@
BenchmarkJob,
RecommendationJob,
)
from sagemaker.serve.ai_inference_recommender.listing import (
list_benchmarks,
list_recommendations,
)
from sagemaker.serve.ai_inference_recommender.result import (
BenchmarkComparison,
BenchmarkMetric,
BenchmarkMetrics,
BenchmarkResult,
BenchmarkSearchResult,
compare_benchmarks,
)
from sagemaker.serve.ai_inference_recommender.secrets import Secret
from sagemaker.serve.ai_inference_recommender.workload import Workload
Expand All @@ -39,6 +45,7 @@


__all__ = [
"BenchmarkComparison",
"BenchmarkJob",
"BenchmarkMetric",
"BenchmarkMetrics",
Expand All @@ -51,5 +58,8 @@
"Secret",
"Workload",
"WorkloadValidationError",
"compare_benchmarks",
"list_benchmarks",
"list_recommendations",
"start_benchmark",
]
186 changes: 138 additions & 48 deletions sagemaker-serve/src/sagemaker/serve/ai_inference_recommender/_recommendation_view.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@
from typing import Any, Dict, List, Optional

from sagemaker.serve.ai_inference_recommender.result import (
_coerce_numeric,
_fmt_number,
_format_table,
_indent,
_require_pandas,
)


Expand Down Expand Up @@ -63,9 +65,7 @@ def stats(self) -> Dict[str, float]:
return dict(self._stats)

def __repr__(self) -> str:
parts = ", ".join(
f"{stat}={_fmt_number(v)}" for stat, v in self._stats.items()
)
parts = ", ".join(f"{stat}={_fmt_number(v)}" for stat, v in self._stats.items())
unit = f" {self.unit}" if self.unit else ""
return f"<{parts}{unit}>"

Expand All @@ -82,9 +82,7 @@ class _ExpectedPerformanceView:
__slots__ = ("_by_metric",)

def __init__(self, raw_rows: Optional[List[Any]]):
by_metric: Dict[str, Dict[str, Any]] = defaultdict(
lambda: {"unit": None, "stats": {}}
)
by_metric: Dict[str, Dict[str, Any]] = defaultdict(lambda: {"unit": None, "stats": {}})
for row in raw_rows or []:
metric = getattr(row, "metric", None)
if not metric:
Expand Down Expand Up @@ -139,9 +137,9 @@ def __len__(self) -> int:
return len(self._by_metric)

def __repr__(self) -> str:
return "{" + ", ".join(
f"{name}: {metric!r}" for name, metric in self._by_metric.items()
) + "}"
return (
"{" + ", ".join(f"{name}: {metric!r}" for name, metric in self._by_metric.items()) + "}"
)


def _to_float(value):
Expand Down Expand Up @@ -195,7 +193,6 @@ def __getattr__(self, name):
def __str__(self) -> str:
md = getattr(self._raw, "model_details", None)
dc = getattr(self._raw, "deployment_configuration", None)
ep = getattr(self._raw, "expected_performance", None) or []

config_lines = [
f"instance_type: {_safe_str(dc, 'instance_type')}",
Expand All @@ -217,14 +214,16 @@ def __str__(self) -> str:
env_lines = [f" {k} = {v}" for k, v in items]
env_block = "\nenv vars ({0}):\n{1}".format(len(items), "\n".join(env_lines))

perf_rows = []
for m in ep:
perf_rows.append([
_safe_str(m, "metric"),
_safe_str(m, "stat"),
_fmt_number(_safe_float(m, "value")),
_safe_str(m, "unit"),
])
# Same records to_dataframe() uses, formatted for display here.
perf_rows = [
[
rec["metric"] if rec["metric"] not in (None, "") else "-",
rec["stat"] if rec["stat"] not in (None, "") else "-",
_fmt_number(rec["value"]),
rec["unit"] if rec["unit"] not in (None, "") else "-",
]
for rec in self._perf_records()
]
perf_table = _format_table(
headers=["metric", "stat", "value", "unit"],
rows=perf_rows,
Expand All @@ -249,6 +248,30 @@ def _repr_pretty_(self, p, cycle):
# Render the full table in notebooks (Jupyter uses this hook).
p.text("..." if cycle else str(self))

def _perf_records(self) -> List[Dict[str, Any]]:
"""(metric, stat, value, unit) records for this row's expected
performance — the rows of the printed table, one per (metric, stat)."""
ep = getattr(self._raw, "expected_performance", None) or []
return [
{
"metric": getattr(m, "metric", None),
"stat": getattr(m, "stat", None),
"value": _safe_float(m, "value"),
"unit": getattr(m, "unit", None),
}
for m in ep
]

def to_dataframe(self):
"""Return this recommendation's expected performance as a pandas
``DataFrame`` — the same ``metric``/``stat``/``value``/``unit`` rows the
printed ``expected performance`` table shows, one row per (metric, stat).

Requires pandas.
"""
pd = _require_pandas()
return pd.DataFrame(self._perf_records(), columns=["metric", "stat", "value", "unit"])


def _safe_str(obj, attr) -> str:
if obj is None:
Expand Down Expand Up @@ -291,48 +314,115 @@ def _repr_pretty_(self, p, cycle):
# Render the full table in notebooks (Jupyter uses this hook).
p.text("..." if cycle else str(self))

# Column labels for the comparative table / DataFrame, in display order.
_TABLE_COLUMNS = (
"idx",
"spec_name",
"instance_type",
"instances",
"copies/inst",
"container",
"req/s",
"tok/s",
"lat_p50",
"lat_p90",
"lat_p99",
"ttft_p50",
"itl_p50",
)
# Metric columns holding numbers; the rest are strings or the idx.
_NUMERIC_COLUMNS = (
"req/s",
"tok/s",
"lat_p50",
"lat_p90",
"lat_p99",
"ttft_p50",
"itl_p50",
)

def _row_records(self) -> List[Dict[str, Any]]:
"""One record per row, keyed by ``_TABLE_COLUMNS``, with native values.

Shared by ``__str__`` and ``to_dataframe()`` so they cannot drift.
"""
records = []
for view in self:
dc = getattr(view.raw, "deployment_configuration", None)
ep = view.expected_performance
records.append(
{
"idx": view._index,
"spec_name": view.recommendation_spec_name,
"instance_type": getattr(dc, "instance_type", None) if dc else None,
"instances": getattr(dc, "instance_count", None) if dc else None,
"copies/inst": (getattr(dc, "copy_count_per_instance", None) if dc else None),
# None when absent (not "-"), so the DataFrame keeps it as
# missing; __str__ renders the dash.
"container": (
_short_container_tag(dc.image_uri)
if dc is not None and getattr(dc, "image_uri", None)
else None
),
"req/s": _get_metric_stat(ep, "request_throughput", "avg"),
"tok/s": _get_metric_stat(ep, "output_token_throughput", "avg"),
"lat_p50": _get_metric_stat(ep, "request_latency", "p50"),
"lat_p90": _get_metric_stat(ep, "request_latency", "p90"),
"lat_p99": _get_metric_stat(ep, "request_latency", "p99"),
"ttft_p50": _get_metric_stat(ep, "time_to_first_token", "p50"),
"itl_p50": _get_metric_stat(ep, "inter_token_latency", "p50"),
}
)
return records

def __str__(self) -> str:
if not self:
return "Recommendations[0] (no rows)"

rows = []
for view in self:
dc = getattr(view.raw, "deployment_configuration", None)
ep = view.expected_performance
rows.append([
f"[{view._index}]",
view.recommendation_spec_name or "-",
_safe_str(dc, "instance_type"),
_safe_str(dc, "instance_count"),
_safe_str(dc, "copy_count_per_instance"),
_short_container_tag(_safe_str(dc, "image_uri")),
_fmt_number(_get_metric_stat(ep, "request_throughput", "avg")),
_fmt_number(_get_metric_stat(ep, "output_token_throughput", "avg")),
_fmt_number(_get_metric_stat(ep, "request_latency", "p50")),
_fmt_number(_get_metric_stat(ep, "request_latency", "p90")),
_fmt_number(_get_metric_stat(ep, "request_latency", "p99")),
_fmt_number(_get_metric_stat(ep, "time_to_first_token", "p50")),
_fmt_number(_get_metric_stat(ep, "inter_token_latency", "p50")),
])

table = _format_table(
headers=[
"idx", "spec_name", "instance_type",
"instances", "copies/inst",
"container",
"req/s", "tok/s",
"lat_p50", "lat_p90", "lat_p99",
"ttft_p50", "itl_p50",
],
rows=rows,
)
for rec in self._row_records():
row = []
for col in self._TABLE_COLUMNS:
value = rec[col]
if col == "idx":
row.append(f"[{value}]")
elif col in self._NUMERIC_COLUMNS:
row.append(_fmt_number(value))
else:
row.append("-" if value in (None, "") else str(value))
rows.append(row)

table = _format_table(headers=list(self._TABLE_COLUMNS), rows=rows)

return (
f"Recommendations[{len(self)}] (.best = top row; index by [N] for full detail)\n"
f"{table}\n"
f"lat/ttft/itl in ms; req/s = requests/sec; tok/s = output tokens/sec"
)

def to_dataframe(self):
"""Return the recommendations as a pandas ``DataFrame`` — one row per
recommendation, columns matching the printed comparative table
(``instance_type``, ``instances``, ``req/s``, ``lat_p50``, ...), indexed
by the recommendation index ``idx``.

Latency columns are milliseconds; ``req/s`` is requests/sec and ``tok/s``
is output tokens/sec, same as the printed table's footnote. Numeric
columns hold real numbers (``NaN`` where a metric is absent), not
Comment thread
ZealSV marked this conversation as resolved.
preformatted strings.

Requires pandas.
"""
pd = _require_pandas()
columns = [c for c in self._TABLE_COLUMNS if c != "idx"]
records = self._row_records()
frame = pd.DataFrame(
[{c: rec[c] for c in columns} for rec in records],
columns=columns,
index=pd.Index([rec["idx"] for rec in records], name="idx"),
)
return _coerce_numeric(pd, frame, self._NUMERIC_COLUMNS)


def _get_metric_stat(ep_view, metric_name: str, stat: str):
try:
Expand Down
Loading
Loading