From add21cf35293c3ba2a648c3367c18b6d8f3f1c6e Mon Sep 17 00:00:00 2001 From: "huangjintao.hjt" Date: Tue, 24 Sep 2024 16:19:30 +0800 Subject: [PATCH 1/4] update --- swift/llm/utils/template.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/swift/llm/utils/template.py b/swift/llm/utils/template.py index f3124d47e7..8411f7d131 100644 --- a/swift/llm/utils/template.py +++ b/swift/llm/utils/template.py @@ -18,6 +18,7 @@ from transformers import PreTrainedTokenizerBase, StoppingCriteria from transformers.dynamic_module_utils import get_class_from_dynamic_module from transformers.integrations import is_deepspeed_zero3_enabled +from transformers.utils import strtobool from swift.llm.agent.utils import calculate_loss_scale, get_tools_prompt from swift.torchacc_utils import pad_and_split_batch @@ -179,6 +180,10 @@ def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor, **kwargs) -> b return False +def is_deepspeed_enabled(): + return strtobool(os.environ.get('ACCELERATE_USE_DEEPSPEED', 'False')) + + class Template: """A template class for all supported models. @@ -2150,7 +2155,7 @@ def _post_encode(self, model, data: Any) -> Dict[str, Any]: vit_embeds = model.extract_feature(pixel_values).to(device=device) selected = (input_ids == self.tokenizer.encode('', add_special_tokens=False)[0]) inputs_embeds[selected] = vit_embeds.reshape(-1, vit_embeds.shape[-1]) - elif is_deepspeed_zero3_enabled(): + elif is_deepspeed_enabled(): dummy_pixel_values = torch.zeros((1, 3, 32, 32), device=device, dtype=inputs_embeds.dtype) vit_embeds = model.extract_feature(dummy_pixel_values).to(device=device) inputs_embeds += vit_embeds.mean() * 0. From a10f1ace58bac854c5ad4e52e9689d3f7d6eacde Mon Sep 17 00:00:00 2001 From: "huangjintao.hjt" Date: Tue, 24 Sep 2024 17:15:59 +0800 Subject: [PATCH 2/4] fix qwen2-vl zero2/zero3 --- swift/llm/utils/model.py | 1 + swift/llm/utils/template.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/swift/llm/utils/model.py b/swift/llm/utils/model.py index bc676110cd..3c71a9a726 100644 --- a/swift/llm/utils/model.py +++ b/swift/llm/utils/model.py @@ -3649,6 +3649,7 @@ def _read_from_stream(container: 'av.container.Container', start_offset: float, model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, torch_dtype, model_kwargs, load_model, **kwargs) tokenizer.processor = processor if model is not None: + model.model.embed_tokens.register_forward_hook(_clone_hook) model.model.embed_tokens.register_forward_hook(_output_device_map_hook) return model, tokenizer diff --git a/swift/llm/utils/template.py b/swift/llm/utils/template.py index 8411f7d131..c04e13b94b 100644 --- a/swift/llm/utils/template.py +++ b/swift/llm/utils/template.py @@ -1483,6 +1483,9 @@ def _encode(self, example: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, An labels = inputs['labels'] images = example.get('images') or [] videos = example.get('videos') or [] + plain_text = False + if is_deepspeed_enabled() and not images and not videos: + plain_text = True for media_type in ['images', 'videos']: if locals()[media_type]: if media_type == 'images': @@ -1509,8 +1512,29 @@ def _encode(self, example: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, An inputs['input_ids'] = input_ids inputs['labels'] = labels + if is_deepspeed_enabled() and plain_text: + inputs['_data'] = {'plain_text': True, 'input_ids': torch.tensor(input_ids)[None]} return inputs, {} + def _post_encode(self, model, data: Any) -> Dict[str, Any]: + from PIL import Image + plain_text = data.pop('plain_text', False) + if is_deepspeed_enabled() and plain_text: + images = [Image.new('RGB', (32, 32), (0, 0, 0))] + processor = self.tokenizer.processor + media_inputs = processor.image_processor(images=images, videos=None, return_tensors='pt') + input_ids = data['input_ids'] + device = input_ids.device + pixel_values = media_inputs['pixel_values'].to(device) + _model = self.model.model + if not hasattr(_model, 'embed_tokens'): + _model = _model.model # LoRA + inputs_embeds = _model.embed_tokens(input_ids) + pixel_values = pixel_values.type(self.model.visual.get_dtype()) + image_embeds = self.model.visual(pixel_values, grid_thw=media_inputs['image_grid_thw']) + inputs_embeds += image_embeds.mean() * 0. + return {'inputs_embeds': inputs_embeds[0]} + def data_collator(self, batch: List[Dict[str, Any]], padding_to: Optional[int] = None) -> Dict[str, Any]: res = super().data_collator(batch, padding_to) for media_type in ['image', 'video']: From 810c64803a9b5c441da7890e772d9de4e9751b09 Mon Sep 17 00:00:00 2001 From: Jintao Huang Date: Tue, 24 Sep 2024 17:21:49 +0800 Subject: [PATCH 3/4] update --- swift/llm/utils/template.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/swift/llm/utils/template.py b/swift/llm/utils/template.py index c04e13b94b..37d1adfaa8 100644 --- a/swift/llm/utils/template.py +++ b/swift/llm/utils/template.py @@ -1484,7 +1484,7 @@ def _encode(self, example: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, An images = example.get('images') or [] videos = example.get('videos') or [] plain_text = False - if is_deepspeed_enabled() and not images and not videos: + if not images and not videos: plain_text = True for media_type in ['images', 'videos']: if locals()[media_type]: @@ -1526,12 +1526,12 @@ def _post_encode(self, model, data: Any) -> Dict[str, Any]: input_ids = data['input_ids'] device = input_ids.device pixel_values = media_inputs['pixel_values'].to(device) - _model = self.model.model + _model = model.model if not hasattr(_model, 'embed_tokens'): _model = _model.model # LoRA inputs_embeds = _model.embed_tokens(input_ids) - pixel_values = pixel_values.type(self.model.visual.get_dtype()) - image_embeds = self.model.visual(pixel_values, grid_thw=media_inputs['image_grid_thw']) + pixel_values = pixel_values.type(model.visual.get_dtype()) + image_embeds = model.visual(pixel_values, grid_thw=media_inputs['image_grid_thw']) inputs_embeds += image_embeds.mean() * 0. return {'inputs_embeds': inputs_embeds[0]} From 40403d720e8ce12928be9a37e0cbf58266d22758 Mon Sep 17 00:00:00 2001 From: "huangjintao.hjt" Date: Tue, 24 Sep 2024 17:33:46 +0800 Subject: [PATCH 4/4] update --- swift/llm/utils/template.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/swift/llm/utils/template.py b/swift/llm/utils/template.py index 37d1adfaa8..98fb23e293 100644 --- a/swift/llm/utils/template.py +++ b/swift/llm/utils/template.py @@ -1483,9 +1483,6 @@ def _encode(self, example: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, An labels = inputs['labels'] images = example.get('images') or [] videos = example.get('videos') or [] - plain_text = False - if not images and not videos: - plain_text = True for media_type in ['images', 'videos']: if locals()[media_type]: if media_type == 'images': @@ -1512,14 +1509,13 @@ def _encode(self, example: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, An inputs['input_ids'] = input_ids inputs['labels'] = labels - if is_deepspeed_enabled() and plain_text: - inputs['_data'] = {'plain_text': True, 'input_ids': torch.tensor(input_ids)[None]} + inputs['_data'] = {'plain_text': not images and not videos, 'input_ids': torch.tensor(input_ids)[None]} return inputs, {} def _post_encode(self, model, data: Any) -> Dict[str, Any]: - from PIL import Image plain_text = data.pop('plain_text', False) if is_deepspeed_enabled() and plain_text: + from PIL import Image images = [Image.new('RGB', (32, 32), (0, 0, 0))] processor = self.tokenizer.processor media_inputs = processor.image_processor(images=images, videos=None, return_tensors='pt') @@ -1533,7 +1529,8 @@ def _post_encode(self, model, data: Any) -> Dict[str, Any]: pixel_values = pixel_values.type(model.visual.get_dtype()) image_embeds = model.visual(pixel_values, grid_thw=media_inputs['image_grid_thw']) inputs_embeds += image_embeds.mean() * 0. - return {'inputs_embeds': inputs_embeds[0]} + return {'inputs_embeds': inputs_embeds[0]} + return {} def data_collator(self, batch: List[Dict[str, Any]], padding_to: Optional[int] = None) -> Dict[str, Any]: res = super().data_collator(batch, padding_to)