diff --git a/.agents/skills/deployment/references/trtllm.md b/.agents/skills/deployment/references/trtllm.md index 5725bed3bf7..865c11e7eee 100644 --- a/.agents/skills/deployment/references/trtllm.md +++ b/.agents/skills/deployment/references/trtllm.md @@ -42,46 +42,24 @@ llm = LLM(model="", tensor_parallel_size=4) ## AutoDeploy (for AutoQuant / mixed-precision) -AutoDeploy automates graph transformations for optimized inference. Required for AutoQuant checkpoints. +AutoDeploy automates graph transformations for optimized inference and is useful for +AutoQuant / mixed-precision checkpoints. The standalone `examples/llm_autodeploy` example +was removed in 0.46; use TensorRT-LLM's +[AutoDeploy](https://github.com/NVIDIA/TensorRT-LLM/tree/main/examples/auto_deploy) +directly together with a ModelOpt-quantized checkpoint. -### End-to-end script +### Workflow -```bash -# Quantize and deploy in one step -./examples/llm_autodeploy/scripts/run_auto_quant_and_deploy.sh \ - --hf_ckpt \ - --save_quantized_ckpt \ - --quant fp8,nvfp4 \ - --effective_bits 4.5 -``` - -Parameters: - -- `--hf_ckpt`: Path to unquantized HuggingFace checkpoint -- `--save_quantized_ckpt`: Output path for quantized checkpoint -- `--quant`: Quantization formats (e.g., `fp8,nvfp4`) -- `--effective_bits`: Target precision (higher = more accuracy for sensitive layers) -- `--world_size`: Number of GPUs for tensor parallelism -- `--calib_batch_size`: Calibration batch size (reduce if OOM, default 8) - -### AutoDeploy API server - -```python -# examples/llm_autodeploy/api_server.py provides a FastAPI server -# with OpenAI-compatible endpoints using AutoDeploy -``` - -### Test AutoDeploy - -```bash -python examples/llm_autodeploy/api_client.py --prompt "What is AI?" "What is golf?" -``` +1. Quantize the checkpoint with ModelOpt PTQ (including AutoQuant / mixed precision) via + `examples/llm_ptq` (`hf_ptq.py` / `scripts/huggingface_example.sh`), which produces a + unified HuggingFace checkpoint with `hf_quant_config.json`. +2. Deploy that checkpoint with TensorRT-LLM's AutoDeploy backend (see the upstream + `examples/auto_deploy` docs for the current API and `trtllm-serve` flags). ### Notes -- NVFP4 in AutoDeploy requires Blackwell GPUs -- For Hopper: remove `nvfp4` from `--quant` and set `--effective_bits` above 8.0 -- AutoDeploy supports CUDA graphs, torch compile backends, and KV cache optimization +- NVFP4 in AutoDeploy requires Blackwell GPUs; on Hopper use FP8 instead. +- AutoDeploy supports CUDA graphs, torch compile backends, and KV cache optimization. ## Legacy TRT-LLM Checkpoint (deprecated) diff --git a/.agents/skills/deployment/scripts/deploy.sh b/.agents/skills/deployment/scripts/deploy.sh index 51a166de00c..1244b49e2c3 100755 --- a/.agents/skills/deployment/scripts/deploy.sh +++ b/.agents/skills/deployment/scripts/deploy.sh @@ -337,12 +337,10 @@ start_trtllm() { cat < \\ - --quant fp8,nvfp4 \\ - --effective_bits 4.5 +# Option 1: AutoDeploy (recommended for AutoQuant / mixed-precision) +# Quantize with ModelOpt PTQ (examples/llm_ptq) to produce a unified HF checkpoint, +# then deploy it with TensorRT-LLM's AutoDeploy backend: +# https://github.com/NVIDIA/TensorRT-LLM/tree/main/examples/auto_deploy # Option 2: Python API python3 -c " diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 96003c7c568..46d2e4f4e32 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -46,7 +46,6 @@ modelopt_recipes @NVIDIA/modelopt-recipes-codeowners /examples/deepseek @NVIDIA/modelopt-deploy-codeowners /examples/diffusers @NVIDIA/modelopt-examples-diffusers-codeowners /examples/gpt-oss @NVIDIA/modelopt-examples-gpt-oss-codeowners -/examples/llm_autodeploy @NVIDIA/modelopt-deploy-codeowners /examples/llm_distill @NVIDIA/modelopt-torch-distill-codeowners /examples/llm_eval @NVIDIA/modelopt-examples-llm_ptq-codeowners /examples/llm_ptq @NVIDIA/modelopt-examples-llm_ptq-codeowners diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml index b80f6820ba9..83d5ee58cba 100644 --- a/.github/workflows/example_tests.yml +++ b/.github/workflows/example_tests.yml @@ -48,7 +48,7 @@ jobs: pip_install_extras: "[hf,dev-test]" runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} - ##### TensorRT-LLM Example Tests (pr/non-pr split: non-pr runs extra autodeploy+eval examples) ##### + ##### TensorRT-LLM Example Tests (pr/non-pr split: non-pr runs extra eval examples) ##### trtllm-pr: needs: [pr-gate] if: startsWith(github.ref, 'refs/heads/pull-request/') && needs.pr-gate.outputs.any_changed == 'true' @@ -69,7 +69,7 @@ jobs: strategy: fail-fast: false matrix: - example: [llm_autodeploy, llm_eval, llm_ptq] + example: [llm_eval, llm_ptq] uses: ./.github/workflows/_example_tests_runner.yml secrets: inherit with: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6ba4c967a2a..f33babadce5 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,7 @@ Changelog **Backward Breaking Changes** - Remove the ``examples/diffusers/eval`` image-quality evaluation example (ImageReward / CLIP-IQA / CLIP metrics) and its references in ``examples/diffusers/README.md``. The example was deprecated in 0.45 and is no longer maintained. +- Remove the deprecated ``examples/llm_autodeploy`` example (deprecated in 0.45). Use TensorRT-LLM's `AutoDeploy `_ directly together with ModelOpt PTQ in ``examples/llm_ptq``. **Deprecations** diff --git a/examples/llm_autodeploy/README.md b/examples/llm_autodeploy/README.md deleted file mode 100644 index f477f80d778..00000000000 --- a/examples/llm_autodeploy/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# Deploy AutoQuant Models with AutoDeploy - -> [!WARNING] -> **Deprecated (ModelOpt 0.45).** This example is deprecated and will be removed in -> a future release (0.46). For mixed-precision deployment, use TensorRT-LLM's -> [AutoDeploy](https://github.com/NVIDIA/TensorRT-LLM/tree/main/examples/auto_deploy) -> directly, combined with ModelOpt PTQ/AutoQuant from -> [`examples/llm_ptq`](../llm_ptq/README.md). - -This guide demonstrates how to deploy mixed-precision models using ModelOpt's AutoQuant and TRT-LLM's AutoDeploy. - -[ModelOpt's AutoQuant](https://nvidia.github.io/Model-Optimizer/reference/generated/modelopt.torch.quantization.model_quant.html#modelopt.torch.quantization.model_quant.auto_quantize) is a post-training quantization (PTQ) algorithm that optimizes model quantization by selecting the best quantization format for each layer while adhering to user-defined compression constraints. This approach allows users to balance model accuracy and performance effectively. - -[TRT-LLM's AutoDeploy](https://github.com/NVIDIA/TensorRT-LLM/tree/main/examples/auto_deploy) is designed to simplify and accelerate the deployment of PyTorch models, including off-the-shelf models like those from Hugging Face, to optimized inference environments with TRT-LLM. It automates graph transformations to integrate inference optimizations such as tensor parallelism, KV-caching and quantization. AutoDeploy supports optimized in-framework deployment, minimizing the amount of manual modification needed. - -## Prerequisites - -AutoDeploy is available in TensorRT-LLM docker images. Please refer to our [Installation Guide](../../README.md#installation) for more details. - -### 1. Quantize and Deploy Model - -Run the following command to quantize your model and launch an OpenAI-compatible endpoint: - -```bash -./scripts/run_auto_quant_and_deploy.sh \ - --hf_ckpt \ - --save_quantized_ckpt \ - --quant fp8,nvfp4 \ - --effective_bits 4.5 -``` - -Parameters: - -- `--hf_ckpt`: Path to the unquantized Hugging Face checkpoint -- `--save_quantized_ckpt`: Output path for the quantized checkpoint -- `--quant`: Quantization formats to use (e.g., `fp8,nvfp4`) -- `--effective_bits`: Target overall precision (higher values preserve accuracy for sensitive layers) -- `--calib_batch_size`: (Optional, default=8) Calibration batch size. Reduce if encountering OOM issues - -> **Note**: -> -> - NVFP4 is only available on Blackwell GPUs. For Hopper GPUs: -> - Remove `nvfp4` from the `--quant` parameter -> - Increase `--effective_bits` above 8.0 for FP8-only AutoQuant -> - For tensor parallelism, add `--world_size ` -> - Additional generation and sampling configurations can be found in `api_server.py` - -### 2. Test the Deployment - -Send test prompts to the server: - -```bash -python api_client.py --prompt "What is AI?" "What is golf?" -``` - -This will return generated responses for both prompts from your deployed model. diff --git a/examples/llm_autodeploy/api_client.py b/examples/llm_autodeploy/api_client.py deleted file mode 100644 index 6ef216d0525..00000000000 --- a/examples/llm_autodeploy/api_client.py +++ /dev/null @@ -1,61 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse - -import requests - - -def get_server_args(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--host", type=str, default="127.0.0.1", help="Host to run the server on") - parser.add_argument("--port", type=int, default=8000, help="port number") - parser.add_argument( - "--prompt", - nargs="+", # Allows multiple inputs - required=True, - help="List of prompts to send (e.g., --prompt 'What is AI?' 'What is golf?')", - ) - parser.add_argument( - "--stop", - nargs="+", # Allows multiple inputs - default=None, - help="List of stop words.", - ) - - return parser.parse_args() - - -def send_request(host, port, prompts, stop): - url = f"http://{host}:{port}/v1/completions" - data = {"prompt": prompts, "stop": stop, "model": "autodeploy_demo"} - - try: - response = requests.post(url, json=data, headers={"Content-Type": "application/json"}) - response.raise_for_status() # Check for HTTP errors - if response.status_code == 200: - response_dict = response.json() - for prompt, output in zip(prompts, response_dict["choices"]): - print(f"{prompt}{output['text']}") - else: - print(f"Error: {response.status_code}, {response.text}") - - except requests.exceptions.RequestException as e: - print(f"Error: {e}") - - -if __name__ == "__main__": - args = get_server_args() - send_request(args.host, args.port, args.prompt, args.stop) diff --git a/examples/llm_autodeploy/api_server.py b/examples/llm_autodeploy/api_server.py deleted file mode 100644 index 0498ed87391..00000000000 --- a/examples/llm_autodeploy/api_server.py +++ /dev/null @@ -1,207 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import sys -import time -import uuid - -import uvicorn -from fastapi import FastAPI, HTTPException -from tensorrt_llm._torch.auto_deploy import LLM -from tensorrt_llm.llmapi.llm import RequestOutput -from tensorrt_llm.sampling_params import SamplingParams -from tensorrt_llm.serve.openai_protocol import ( - CompletionRequest, - CompletionResponse, - CompletionResponseChoice, - UsageInfo, -) - -import modelopt.torch.opt as mto - -# global vars -app = FastAPI() -model_runner = None -args = None -sampling_params = None -model = "autodeploy_demo" - - -def build_runner_from_config(args) -> LLM: - """Builds a model runner from our config.""" - mto.enable_huggingface_checkpointing() - model_kwargs = {"max_position_embeddings": args.max_seq_len, "use_cache": False} - - llm = LLM( - model=args.ckpt_path, - compile_backend=args.compile_backend, - device=args.device, - world_size=args.world_size, - max_batch_size=args.max_batch_size, - max_seq_len=args.max_seq_len, - max_num_tokens=args.max_num_tokens, - model_kwargs=model_kwargs, - attn_backend="flashinfer", - ) - - return llm - - -def apply_stop_tokens(text: str, stop_words: list[str] | None) -> str: - """Truncate text at the first occurrence of any stop token.""" - if not stop_words: - return text # No stop tokens provided, return as is - - for stop in stop_words: - stop_idx = text.find(stop) - if stop_idx != -1: - return text[:stop_idx] # Truncate at the first stop token - - return text # No stop token found, return original text - - -@app.post("/v1/completions", response_model=CompletionResponse) -async def create_completion(request: CompletionRequest): - """Endpoint to handle completion requests.""" - global model_runner, model, sampling_params - - if model_runner is None: - raise HTTPException(status_code=500, detail="Runner is not initialized") - - # Run inference using the model_runner - if isinstance(request.prompt, str): - prompts = [request.prompt] # Single string becomes a list with one element - elif isinstance(request.prompt, list): - if all(isinstance(p, str) for p in request.prompt): # List of strings - prompts = request.prompt - else: - raise HTTPException(status_code=400, detail="Invalid prompt type") - sampling_params.temperature = request.temperature - outs = model_runner.generate(prompts, sampling_params) - - # formatting outputs - outputs = [] - if isinstance(outs, RequestOutput): - outs = [outs] - for i, out in enumerate(outs): - outputs.append({"prompt": out.prompt, "text": out.outputs[0].text}) - - # Generate unique ID - unique_id = str(uuid.uuid4()) - - # Generate timestamp - created_timestamp = int(time.time()) - - # Construct response - response = CompletionResponse( - id=unique_id, - object="text_completion", - created=created_timestamp, - model=model, - choices=[ - CompletionResponseChoice( - index=i, - text=apply_stop_tokens(output["text"], request.stop), - stop_reason="stop" - if any(st in output["text"] for st in request.stop or []) - else "length", - ) - for i, output in enumerate(outputs) - ], - usage=UsageInfo(), - ) - return response - - -def get_server_args(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--host", type=str, default="127.0.0.1", help="Host to run the server on") - parser.add_argument("--port", type=int, default=8000, help="port number") - parser.add_argument( - "--ckpt_path", - help="Specify where the HF checkpoint path is.", - required=True, - ) - parser.add_argument( - "--device", - type=str, - default="cuda", - help=("Target device to host the model."), - ) - parser.add_argument( - "--backend", - type=str, - default="torch-opt", - help=("backend to compile to model."), - ) - parser.add_argument( - "--world_size", - type=int, - default=0, - help=("target world size for hosting the model."), - ) - parser.add_argument( - "--max_batch_size", - type=int, - default=8, - help=("max dimension for statically allocated kv cache"), - ) - parser.add_argument( - "--max_seq_len", - type=int, - default=2048, - help=("max sequence length for inference/cache"), - ) - parser.add_argument( - "--max_num_tokens", - type=int, - default=128, - help=("max tokens to generate."), - ) - parser.add_argument( - "--top_k", - type=int, - default=200, - help=("top_k for output sampling."), - ) - parser.add_argument( - "--compile_backend", - type=str, - default="torch-opt", - help=("backend to compile the torch graph."), - ) - return parser.parse_args() - - -def run_server(): - try: - global model_runner, args, sampling_params - args = get_server_args() - model_runner = build_runner_from_config(args) - sampling_params = SamplingParams( - max_tokens=args.max_num_tokens, - top_k=args.top_k, - temperature=1.0, # default value, we will take temperature from requests - ) - - uvicorn.run(app, host=args.host, port=args.port) - except Exception as e: - print(f"Error: {e}") - sys.exit(1) - - -if __name__ == "__main__": - run_server() diff --git a/examples/llm_autodeploy/run_auto_quantize.py b/examples/llm_autodeploy/run_auto_quantize.py deleted file mode 100644 index db35e4841fb..00000000000 --- a/examples/llm_autodeploy/run_auto_quantize.py +++ /dev/null @@ -1,234 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -from collections import defaultdict -from typing import Any - -import torch -from transformers import AutoModelForCausalLM, AutoTokenizer - -import modelopt.torch.opt as mto -import modelopt.torch.quantization as mtq -from modelopt.torch.utils import create_forward_loop -from modelopt.torch.utils.dataset_utils import get_dataset_dataloader - -SUPPORT_QUANT_FORMAT: dict[str, dict[str, Any]] = { - "fp8": mtq.FP8_DEFAULT_CFG, - "nvfp4": mtq.NVFP4_DEFAULT_CFG, -} - - -def update_weight_quantizer_amax_for_fusion(model: torch.nn.Module): - """Group modules that take the same input and set amax to enable gemm fusion.""" - input_to_linear = defaultdict(list) - - def _input_hook(module, input, output): - input_to_linear[input[0]].append(module) - - handles = [] - - for name, module in model.named_modules(): - if "QuantLinear" in type(module).__name__: - module.name = name - handle = module.register_forward_hook(_input_hook) - handles.append(handle) - - with torch.no_grad(): - fake_input = torch.ones([1, 2], dtype=torch.long).to(model.device) - # Run forward pass so that all modules sharing the same input are collected using forward hook. - model(fake_input) - for handle in handles: - handle.remove() - - for modules in input_to_linear.values(): - # make sure they have the same input amax - if modules[0].input_quantizer.is_enabled: - amax = modules[0].input_quantizer.amax - for m in modules: - assert m.input_quantizer.is_enabled - assert m.input_quantizer.amax == amax - - # set amax of weight_quantizer - if modules[0].weight_quantizer.is_enabled: - max_weight_amax = max([m.weight_quantizer.amax for m in modules]) - for m in modules: - m.weight_quantizer.amax = max_weight_amax - - -def auto_quantize( - model, qformat, auto_quantize_bits, calib_dataloader, calibrate_loop, batch_size=1 -): - qformat_list = qformat.split(",") - # Check if all provided quantization formats are supported - assert all(qformat in SUPPORT_QUANT_FORMAT for qformat in qformat_list), ( - "One or more quantization formats provided are not supported for unified checkpoint export" - ) - - def loss_func(output, data): - # For transformers AutoModelForCausalLM models, the outputs are wrapped in `CausalLMOutputWithPast` - # which contains the loss attribute. - return output.loss - - model, _ = mtq.auto_quantize( - model, - constraints={"effective_bits": auto_quantize_bits}, - data_loader=calib_dataloader, - forward_step=lambda model, batch: model(**batch), - loss_func=loss_func, - quantization_formats=[SUPPORT_QUANT_FORMAT[quant_format] for quant_format in qformat_list], - num_calib_steps=len(calib_dataloader), - num_score_steps=min( - len(calib_dataloader), 128 // batch_size - ), # Limit the number of score steps to avoid long calibration time - verbose=True, - ) - - # We need to explicitly calibrate for kv cache quantization - enable_kv_cache_quantization = "int8" not in qformat - if enable_kv_cache_quantization: - mtq.set_quantizer_by_cfg( - model, - quant_cfg=[ - { - "quantizer_name": "*output_quantizer", - "cfg": {"num_bits": (4, 3), "axis": None}, - "enable": True, - } - ], - ) - # Lets calibrate only the output quantizer this time. Let's disable all other quantizers. - with mtq.set_quantizer_by_cfg_context( - model, - [ - {"quantizer_name": "*", "enable": False}, - {"quantizer_name": "*output_quantizer", "enable": True}, - ], - ): - mtq.calibrate(model, algorithm="max", forward_loop=calibrate_loop) - return model - - -def modelopt_ptq( - model_path: str, - output_dir: str, - qformat: str | None = None, - num_samples: int = 512, - auto_quantize_bits: float | None = None, - calib_dataset: str = "cnn_dailymail", - calib_batch_size: int = 8, - trust_remote_code: bool = False, -) -> torch.nn.Module: - """Quantize the model with modelopt.""" - model = AutoModelForCausalLM.from_pretrained( - model_path, trust_remote_code=trust_remote_code, dtype="auto", device_map="auto" - ) - model.eval() - - tokenizer = AutoTokenizer.from_pretrained( - model_path, - model_max_length=2048, - padding_side="left", - trust_remote_code=trust_remote_code, - ) - # sanitize tokenizer - if tokenizer.pad_token != "": - tokenizer.pad_token = tokenizer.eos_token - if tokenizer.pad_token is None: - tokenizer.pad_token = tokenizer.eos_token - - # create the forward loop for calibration - calib_dataloader = get_dataset_dataloader( - dataset_name=calib_dataset, - tokenizer=tokenizer, - batch_size=calib_batch_size, - num_samples=num_samples, - device=model.device, - include_labels=auto_quantize_bits is not None, - ) - calibrate_loop = create_forward_loop(dataloader=calib_dataloader) - - # quantize the model - model = auto_quantize( - model, - qformat, - auto_quantize_bits, - calib_dataloader, - calibrate_loop, - calib_batch_size, - ) - - # Post processing for weight scaling factors - # 1. detect modules which will be fused and shared the same input - # 2. update the weight amax, as the modules will be fused during deployment - # This is required for nvfp4, fp8 for gemm fusion - update_weight_quantizer_amax_for_fusion(model) - - # enable huggingface checkpointing for ModelOpt - mto.enable_huggingface_checkpointing() - - print(f"Saving the quantized model to {output_dir}.") - tokenizer.save_pretrained(output_dir) - model.save_pretrained(output_dir) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--hf_ckpt", - help="Specify where the unqunatized HF checkpoint path is.", - required=True, - ) - parser.add_argument( - "--quant", - help=(f"Quantization format. Available options: {list(SUPPORT_QUANT_FORMAT.keys())}."), - default="fp8", - ) - parser.add_argument( - "--output_dir", - help=("Output directory of the quantized checkpoint with tokenizer."), - ) - parser.add_argument( - "--num_samples", help="Number of samples for calibration.", type=int, default=512 - ) - parser.add_argument( - "--calib_batch_size", help="Batch size for calibration.", type=int, default=8 - ) - parser.add_argument( - "--effective_bits", - default=8.0, - type=float, - help=( - "Effective bits constraint for auto_quantize. If not set, " - "regular quantization without auto_quantize search will be applied." - ), - ) - parser.add_argument( - "--trust_remote_code", - action="store_true", - help="Set trust_remote_code for Huggingface models and tokenizers", - ) - - args = parser.parse_args() - - modelopt_ptq( - args.hf_ckpt, - args.output_dir, - args.quant, - args.num_samples, - auto_quantize_bits=args.effective_bits, - calib_batch_size=args.calib_batch_size, - trust_remote_code=args.trust_remote_code, - ) diff --git a/examples/llm_autodeploy/scripts/run_auto_quant_and_deploy.sh b/examples/llm_autodeploy/scripts/run_auto_quant_and_deploy.sh deleted file mode 100755 index d97e3d070b9..00000000000 --- a/examples/llm_autodeploy/scripts/run_auto_quant_and_deploy.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Exit immediately if a command exits with a non-zero status -set -e -set -x - -# Function to print error message and exit -error_exit() { - echo "Error: $1" >&2 - exit 1 -} - -# Check if required arguments are provided -if [[ $# -lt 2 ]]; then - echo "Usage: $0 --hf_ckpt --save_quantized_ckpt [--quant ] [--effective_bits ] [--host ] [--port ]" - exit 1 -fi - -# Default values -HOST="127.0.0.1" -PORT="8000" -WORLD_SIZE="1" -CALIB_BATCH_SIZE=8 - -# Parse arguments -while [[ $# -gt 0 ]]; do - case "$1" in - --hf_ckpt) - HF_CKPT="$2" - shift 2 - ;; - --save_quantized_ckpt) - save_quantized_ckpt="$2" - shift 2 - ;; - --quant) - QUANT="$2" - shift 2 - ;; - --effective_bits) - EFFECTIVE_BITS="$2" - shift 2 - ;; - --host) - HOST="$2" - shift 2 - ;; - --port) - PORT="$2" - shift 2 - ;; - --world_size) - WORLD_SIZE="$2" - shift 2 - ;; - --calib_batch_size) - CALIB_BATCH_SIZE="$2" - shift 2 - ;; - *) - error_exit "Unknown argument: $1" - ;; - esac -done - -# Ensure QUANTIZER_CKPT is provided -if [[ -z "$save_quantized_ckpt" ]]; then - error_exit "--save_quantized_ckpt is required to specify the path to save quantized checkpoint." -fi - -if [[ ! -d "$HF_CKPT" ]]; then - error_exit "Checkpoint path '$HF_CKPT' does not exist!" -fi - -# Step 1: Quantize the model only if QUANTIZER_CKPT does not exist -if [[ -d "$save_quantized_ckpt" ]]; then - echo "Quantized model already exists at '$save_quantized_ckpt'. Skipping quantization step." -else - if [[ -z "$HF_CKPT" || -z "$QUANT" || -z "$EFFECTIVE_BITS" ]]; then - error_exit "--hf_ckpt, --quant, --effective_bits must be specified to generate quantized checkpoint." - fi - - echo "Running Model Quantization..." - QUANTIZE_CMD="python run_auto_quantize.py --hf_ckpt $HF_CKPT --output_dir $save_quantized_ckpt --quant $QUANT --effective_bits $EFFECTIVE_BITS --calib_batch_size $CALIB_BATCH_SIZE" - eval "$QUANTIZE_CMD" - echo "Model Quantization completed successfully!" -fi - -# Step 2: Launch the inference server -echo "Starting Inference Server..." -SERVER_CMD="python api_server.py --ckpt_path $save_quantized_ckpt --host $HOST --port $PORT --world_size $WORLD_SIZE" -eval "$SERVER_CMD" -echo "Inference Server is now running!" diff --git a/tests/examples/llm_autodeploy/test_llama.py b/tests/examples/llm_autodeploy/test_llama.py deleted file mode 100644 index 31d9d68332b..00000000000 --- a/tests/examples/llm_autodeploy/test_llama.py +++ /dev/null @@ -1,78 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import subprocess -import time - -from _test_utils.examples.run_command import ( - extend_cmd_parts, - run_command_in_background, - run_example_command, -) -from _test_utils.torch.distributed.utils import get_free_port -from _test_utils.torch.misc import minimum_sm - - -def run_llm_autodeploy_command( - model: str, quant: str, effective_bits: float, output_dir: str, **kwargs -): - # Create temporary directory for saving the quantized checkpoint - port = get_free_port() - quantized_ckpt_dir = os.path.join(output_dir, "quantized_model") - kwargs.update( - { - "hf_ckpt": model, - "quant": quant, - "effective_bits": effective_bits, - "save_quantized_ckpt": quantized_ckpt_dir, - "port": port, - } - ) - - server_handler = None - try: - # Quantize and deploy the model to the background - cmd_parts = extend_cmd_parts(["scripts/run_auto_quant_and_deploy.sh"], **kwargs) - # Pass None to stdout and stderr to see the output in the console - server_handler = run_command_in_background( - cmd_parts, "llm_autodeploy", stdout=None, stderr=None - ) - - # Wait for the server to start. We might need to build - time.sleep(100) - - # Test the deployment - run_example_command( - ["python", "api_client.py", "--prompt", "What is AI?", "--port", str(port)], - "llm_autodeploy", - ) - finally: - if server_handler: - server_handler.terminate() - - -@minimum_sm(89) -def test_llama_fp8(tiny_llama_path, tmp_path): - try: - run_llm_autodeploy_command( - model=tiny_llama_path, - quant="fp8", - effective_bits=8.5, - output_dir=tmp_path, - ) - finally: - # Force kill llm-serve if it's still running - subprocess.run(["pkill", "-f", "llm-serve"], check=False)