Inference problem, tokenizer shape mismatch

[JunlingWang0512]

I hope to do inference with my fine-tuned sd-3.5-large model. I followed this code: https://github.com/bghira/SimpleTuner/blob/main/documentation/LYCORIS.md and it works well for flux.1 model.
But when I adapt this code to sd-3.5-large, I met this error: RuntimeError: mat1 and mat2 shapes cannot be multiplied (2x157696 and 2048x2432)

This error is likely because of tokenizer mismatch, but I simply use tokenizer from downloaded model, and the mismatch happen at tokenizer and tokenizer_2:
text_encoder = CLIPTextModel.from_pretrained(bfl_repo, subfolder="text_encoder", torch_dtype=dtype)
tokenizer = CLIPTokenizer.from_pretrained(bfl_repo, subfolder="tokenizer", torch_dtype=dtype)

text_encoder_2 = CLIPTextModel.from_pretrained(bfl_repo, subfolder="text_encoder_2", torch_dtype=dtype)
tokenizer_2 = CLIPTokenizer.from_pretrained(bfl_repo, subfolder="tokenizer_2", torch_dtype=dtype)

text_encoder_3 = None
tokenizer_3 = None

Here is my complete code, and complete error, I wonder what might be the problem?

code:

import torch

from diffusers import FlowMatchEulerDiscreteScheduler, AutoencoderKL
# from diffusers.models.transformers.transformer_flux import FluxTransformer2DModel
from diffusers.models.transformers.transformer_sd3 import SD3Transformer2DModel
from diffusers import StableDiffusion3Pipeline
# from diffusers.pipelines.flux.pipeline_flux import FluxPipeline
from transformers import AutoModelForCausalLM, CLIPTextModel, CLIPTokenizer,T5EncoderModel, T5TokenizerFast

from lycoris import create_lycoris_from_weights

device = "cuda" if torch.cuda.is_available() else "cpu"
dtype = torch.bfloat16
bfl_repo = "model/models--stabilityai--stable-diffusion-3.5-large/snapshots/ceddf0a7fdf2064ea28e2213e3b84e4afa170a0f"
scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(bfl_repo, subfolder="scheduler")


text_encoder = CLIPTextModel.from_pretrained(bfl_repo, subfolder="text_encoder", torch_dtype=dtype)
tokenizer = CLIPTokenizer.from_pretrained(bfl_repo, subfolder="tokenizer", torch_dtype=dtype)


text_encoder_2 = CLIPTextModel.from_pretrained(bfl_repo, subfolder="text_encoder_2", torch_dtype=dtype)
tokenizer_2 = CLIPTokenizer.from_pretrained(bfl_repo, subfolder="tokenizer_2", torch_dtype=dtype)

text_encoder_3 = None
tokenizer_3 = None



vae = AutoencoderKL.from_pretrained(bfl_repo, subfolder="vae", torch_dtype=dtype)
transformer = SD3Transformer2DModel.from_pretrained(bfl_repo, subfolder="transformer")

lycoris_safetensors_path = 'pytorch_lora_weights.safetensors'
lycoris_strength = 1.0
wrapper, _ = create_lycoris_from_weights(lycoris_strength, lycoris_safetensors_path, transformer)
wrapper.merge_to() # using apply_to() will be slower.

transformer.to(device, dtype=dtype)

pipe = StableDiffusion3Pipeline(
    scheduler=scheduler,
    text_encoder=text_encoder,
    tokenizer=tokenizer,
    text_encoder_2=text_encoder_2,
    tokenizer_2=tokenizer_2,
    text_encoder_3=text_encoder_3,
    tokenizer_3=tokenizer_3,
    vae=vae,
    transformer=transformer,
)
# Define the generator
generator = torch.manual_seed(42)  # Reproducibility seed
pipe.enable_sequential_cpu_offload()

with torch.inference_mode():
    image = pipe(
        prompt="draw apples",
        width=1024,
        height=1024,
        num_inference_steps=28,
        generator=generator,
        guidance_scale=7,
    ).images[0]
image.save('image_new.png')

Error:

Traceback (most recent call last):
  File "SimpleTuner/stable-diffusion_inference7.py", line 62, in <module>
    image = pipe(
            ^^^^^
  File "anaconda3/envs/stable-diffusion/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "anaconda3/envs/stable-diffusion/lib/python3.11/site-packages/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py", line 952, in __call__
    noise_pred = self.transformer(
                 ^^^^^^^^^^^^^^^^^
  File "anaconda3/envs/stable-diffusion/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "anaconda3/envs/stable-diffusion/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "anaconda3/envs/stable-diffusion/lib/python3.11/site-packages/accelerate/hooks.py", line 170, in new_forward
    output = module._old_forward(*args, **kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "anaconda3/envs/stable-diffusion/lib/python3.11/site-packages/diffusers/models/transformers/transformer_sd3.py", line 390, in forward
    temb = self.time_text_embed(timestep, pooled_projections)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "anaconda3/envs/stable-diffusion/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "anaconda3/envs/stable-diffusion/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "anaconda3/envs/stable-diffusion/lib/python3.11/site-packages/diffusers/models/embeddings.py", line 1553, in forward
    pooled_projections = self.text_embedder(pooled_projection)
                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "anaconda3/envs/stable-diffusion/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "anaconda3/envs/stable-diffusion/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "anaconda3/envs/stable-diffusion/lib/python3.11/site-packages/diffusers/models/embeddings.py", line 2171, in forward
    hidden_states = self.linear_1(caption)
                    ^^^^^^^^^^^^^^^^^^^^^^
  File "anaconda3/envs/stable-diffusion/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "anaconda3/envs/stable-diffusion/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "anaconda3/envs/stable-diffusion/lib/python3.11/site-packages/accelerate/hooks.py", line 170, in new_forward
    output = module._old_forward(*args, **kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "anaconda3/envs/stable-diffusion/lib/python3.11/site-packages/torch/nn/modules/linear.py", line 125, in forward
    return F.linear(input, self.weight, self.bias)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: mat1 and mat2 shapes cannot be multiplied (2x157696 and 2048x2432)